From eec8451526bcaa87f35dad5677b5bae88ee6702a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:32:54 -0300 Subject: [PATCH 01/52] docs: start rank 12a dynamic supervisor design --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c63fe920..1dae3a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Design (not implementation) for rank 12a, the dynamic supervisor + (`DynamicSupervisor` / `simple_one_for_one`): + `docs/design/dynamic-supervisor.md`, the ethrex/ethlambda client mapping + and the `DynamicSupervisor` TLA+ model with negative controls. - Lockfile: `rustix` 1.1.2 -> 1.1.6 for RUSTSEC-2026-0486 (pulled in by the examples' `native-tls` dependency; spawned's crates are unaffected). - CI: a `sim` job runs the simulated test suites (`--cfg spawned_sim`) From d1d2a8c25b1950e1df4afe744c2eae0d629050fa Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:38:41 -0300 Subject: [PATCH 02/52] verify: DynamicSupervisor TLA+ model and MC configs (WIP) --- verification/tla/DynamicSupervisor.tla | 435 +++++++++++++++++++ verification/tla/MCDynamicSupervisor.cfg | 31 ++ verification/tla/MCDynamicSupervisor.tla | 15 + verification/tla/MCDynamicSupervisorDeep.cfg | 27 ++ verification/tla/MCDynamicSupervisorDeep.tla | 3 + 5 files changed, 511 insertions(+) create mode 100644 verification/tla/DynamicSupervisor.tla create mode 100644 verification/tla/MCDynamicSupervisor.cfg create mode 100644 verification/tla/MCDynamicSupervisor.tla create mode 100644 verification/tla/MCDynamicSupervisorDeep.cfg create mode 100644 verification/tla/MCDynamicSupervisorDeep.tla diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla new file mode 100644 index 00000000..fa3fc3a1 --- /dev/null +++ b/verification/tla/DynamicSupervisor.tla @@ -0,0 +1,435 @@ +------------------------- MODULE DynamicSupervisor ------------------------- +(***************************************************************************) +(* Rank 12a design model (not implemented): a dynamic supervisor, OTP's *) +(* simple_one_for_one / Elixir's DynamicSupervisor. Design contract: *) +(* docs/design/dynamic-supervisor.md. *) +(* *) +(* Slots 1..N are potential start_child calls; each is made at most once, *) +(* at any time, so starts race crashes, restarts, terminate_child, *) +(* max_children and shutdown. A slot that is admitted owns one child: *) +(* its first generation, then restarts with fresh generations (fresh *) +(* ActorIds) until the child is removed. Restart types are per slot so *) +(* one instance covers all three (a simple_one_for_one template has one *) +(* type; mixing them is the heterogeneous case and loses nothing). *) +(* *) +(* Abstractions, as in Supervision.tla: *) +(* - The admission gate and the supervisor's cancel are one boolean *) +(* `open`; shutdown (Close) and failure (Fail) both close it. *) +(* - Time is abstract. The rolling restart window is a counter released *) +(* by WindowExpire, at most MaxExpire times, which also bounds the *) +(* number of generations (no state constraint is needed). *) +(* - The shutdown_timeout -> kill escalation is the nondeterministic *) +(* reason of StopObserved (Shutdown, or Killed after the kill). A *) +(* cleanup that misses its second interval is CleanupStuck: the *) +(* supervisor fails, the generation stays alive until it really ends. *) +(* - Backoff is the "backoff" state; its sleep is not modelled. *) +(* - The budget is charged when a restart attempt is admitted; the *) +(* implementation's reserve-then-stamp protocol is checked in *) +(* Supervision.tla and shared code. *) +(* *) +(* Fault selects a negative control; negative_checks.py requires each to *) +(* violate its named invariant: *) +(* IgnoreMax (MaxChildrenBound) admission ignores the cap *) +(* UncountedRestart (MaxChildrenBound) a crashed child frees its *) +(* slot until it restarts *) +(* RestartTemporary (TemporaryNeverRestarted) *) +(* RestartTerminated (TerminatedNeverRestarted) terminate_child *) +(* does not cancel restart *) +(* AcceptDuringShutdown (NoStartAfterShutdown) start_child admitted and *) +(* committed after close *) +(* RestartAfterClose (NoGenerationAfterClose) a pending restart *) +(* proceeds after close *) +(* NoEscalate (ExhaustionCloses) budget exhaustion drops *) +(* only that child *) +(* EarlyDone (NoChildOutlivesSupervisor) done before every *) +(* child finished *) +(* Sequential (ConcurrentShutdown) children stopped one at a *) +(* time, as a static *) +(* supervisor does *) +(***************************************************************************) +EXTENDS Naturals, FiniteSets + +CONSTANTS + N, \* number of start_child calls (slots 1..N) + Mode, \* Mode[s] \in {"Permanent", "Transient", "Temporary"} + MaxChildren, \* max_children + MaxRestarts, \* restart intensity (max_restarts per window) + MaxExpire, \* how often the rolling window may release an entry + SelfExits, \* reasons a ready child may exit with on its own + StartFails, \* outcomes of a failed construction or readiness + Fault + +Slots == 1..N +Modes == {"Permanent", "Transient", "Temporary"} +\* "none": factory error or panic (no actor reason); "timeout": startup +\* deadline; otherwise the actor's exit reason (v0.8 reasons, design E5). +Reasons == {"none", "timeout", "Normal", "Shutdown", "Panic", "Kill", "Killed"} +Abnormal == {"none", "timeout", "Panic", "Kill", "Killed"} + +ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat \ {0} +ASSUME MaxRestarts \in Nat /\ MaxExpire \in Nat +ASSUME \A s \in Slots : Mode[s] \in Modes +ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons +ASSUME Fault \in {"None", "IgnoreMax", "UncountedRestart", "RestartTemporary", + "RestartTerminated", "AcceptDuringShutdown", "RestartAfterClose", + "NoEscalate", "EarlyDone", "Sequential"} + +\* Slot states. "idle": start_child not called yet. "starting": the first +\* generation is being constructed (the caller waits). "restarting": a +\* replacement is being constructed. "cleanup": the generation is exiting +\* and its owned cleanup has not finished. "decide": cleanup finished, +\* restart decision pending. "backoff": a restart is pending. +\* "removed" and "rejected" are terminal. +States == {"idle", "starting", "running", "cleanup", "decide", "backoff", + "restarting", "removed", "rejected"} +Active == {"starting", "running", "cleanup", "decide", "backoff", "restarting"} +Terminal == {"idle", "removed", "rejected"} +\* What start_child returned. "pending" while the caller waits. +Results == {"none", "pending", "ok", "failed", "max", "shutdown"} + +VARIABLES + open, \* admission open: neither shut down nor failed + failed, \* the supervisor failed (budget exhausted or cleanup stuck) + sphase, \* "running", "signal" (Sequential only), "waiting", "done" + budget, \* restart grants in the current window + expired, \* window releases used (bounds the model) + cs, \* slot state + gen, \* generations started in this slot + alive, \* live (not yet cleaned up) generations + pub, \* generation current() returns, 0 if none + reason, \* exit reason of the latest generation + readyGen, \* the latest generation became ready + everReady, \* the slot's first start succeeded + result, \* start_child's outcome + term, \* terminate_child was called for this child + stopSig, \* the supervisor's stop signal reached this slot + \* Ghosts for the invariants. + termGen, \* gen when terminate_child was called + closeGen, \* gen of each slot when admission closed, -1 while open + lateOk, \* a start_child returned ok after admission closed + exhausted \* a restart was refused for lack of budget + +sup == <> +slot == <> +ghost == <> +vars == <> + +TypeOK == + /\ open \in BOOLEAN /\ failed \in BOOLEAN + /\ sphase \in {"running", "signal", "waiting", "done"} + /\ budget \in 0..MaxRestarts /\ expired \in 0..MaxExpire + /\ cs \in [Slots -> States] + /\ gen \in [Slots -> Nat] + /\ alive \in [Slots -> SUBSET Nat] + /\ pub \in [Slots -> Nat] + /\ reason \in [Slots -> Reasons] + /\ readyGen \in [Slots -> BOOLEAN] /\ everReady \in [Slots -> BOOLEAN] + /\ result \in [Slots -> Results] + /\ term \in [Slots -> BOOLEAN] /\ stopSig \in [Slots -> BOOLEAN] + /\ termGen \in [Slots -> Nat] + /\ closeGen \in [Slots -> Nat \cup {-1}] + /\ lateOk \in [Slots -> BOOLEAN] + /\ exhausted \in BOOLEAN + +Init == + /\ open = TRUE /\ failed = FALSE /\ sphase = "running" + /\ budget = 0 /\ expired = 0 + /\ cs = [s \in Slots |-> "idle"] + /\ gen = [s \in Slots |-> 0] + /\ alive = [s \in Slots |-> {}] + /\ pub = [s \in Slots |-> 0] + /\ reason = [s \in Slots |-> "none"] + /\ readyGen = [s \in Slots |-> FALSE] + /\ everReady = [s \in Slots |-> FALSE] + /\ result = [s \in Slots |-> "none"] + /\ term = [s \in Slots |-> FALSE] + /\ stopSig = [s \in Slots |-> FALSE] + /\ termGen = [s \in Slots |-> 0] + /\ closeGen = [s \in Slots |-> -1] + /\ lateOk = [s \in Slots |-> FALSE] + /\ exhausted = FALSE + +\* Slots that count against max_children: every admitted child that was not +\* removed, including one that is restarting (in backoff or constructing its +\* replacement) or still cleaning up. The control counts only live, +\* published children. +Counted == + IF Fault = "UncountedRestart" + THEN {s \in Slots : cs[s] \in {"starting", "running"}} + ELSE {s \in Slots : cs[s] \in Active} + +\* Close admission (shutdown or failure) and record each slot's generation. +CloseGate == + /\ open' = FALSE + /\ closeGen' = IF open THEN gen ELSE closeGen + +Fail == failed' = TRUE /\ CloseGate + +Set(f, s, v) == [f EXCEPT ![s] = v] + +----------------------------------------------------------------------------- +(* Callers and environment *) + +\* start_child(args). Admission is atomic with the count and the gate. +Request(s) == + /\ cs[s] = "idle" + /\ IF ~open /\ Fault # "AcceptDuringShutdown" + THEN /\ cs' = Set(cs, s, "rejected") /\ result' = Set(result, s, "shutdown") + /\ UNCHANGED <> + ELSE IF Cardinality(Counted) >= MaxChildren /\ Fault # "IgnoreMax" + THEN /\ cs' = Set(cs, s, "rejected") /\ result' = Set(result, s, "max") + /\ UNCHANGED <> + ELSE /\ cs' = Set(cs, s, "starting") /\ result' = Set(result, s, "pending") + /\ gen' = Set(gen, s, 1) /\ alive' = Set(alive, s, {1}) + /\ reason' = Set(reason, s, "none") /\ readyGen' = Set(readyGen, s, FALSE) + /\ UNCHANGED <> + +\* Supervisor::stop / shutdown, system shutdown, or the parent stopping it. +Close == + /\ open + /\ CloseGate + /\ UNCHANGED <> + +\* terminate_child(id) on a child whose start returned. It never restarts +\* afterwards, whatever its restart type. After close it is a no-op (the +\* supervisor already stops every child), so it is only modelled while open. +TerminateChild(s) == + /\ open /\ everReady[s] /\ ~term[s] + /\ cs[s] \in {"running", "cleanup", "decide", "backoff", "restarting"} + /\ term' = Set(term, s, TRUE) + /\ stopSig' = Set(stopSig, s, TRUE) + /\ termGen' = Set(termGen, s, gen[s]) + /\ UNCHANGED <> + +\* The rolling window forgets its oldest grant. +WindowExpire == + /\ budget > 0 /\ expired < MaxExpire + /\ budget' = budget - 1 /\ expired' = expired + 1 + /\ UNCHANGED <> + +----------------------------------------------------------------------------- +(* One child slot *) + +\* started() returned while admission is open: readiness commits, and for +\* a first start start_child returns Ok(ref). +Ready(s) == + /\ cs[s] \in {"starting", "restarting"} + /\ open \/ Fault = "AcceptDuringShutdown" + /\ ~(term[s] /\ cs[s] = "restarting") + /\ cs' = Set(cs, s, "running") + /\ pub' = Set(pub, s, gen[s]) + /\ readyGen' = Set(readyGen, s, TRUE) + /\ everReady' = Set(everReady, s, TRUE) + /\ result' = IF cs[s] = "starting" THEN Set(result, s, "ok") ELSE result + /\ lateOk' = IF cs[s] = "starting" /\ ~open THEN Set(lateOk, s, TRUE) ELSE lateOk + /\ UNCHANGED <> + +\* Factory error or panic, panic or stop in started(), startup timeout. +StartFail(s, r) == + /\ cs[s] \in {"starting", "restarting"} + /\ r \in StartFails + /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) + /\ UNCHANGED <> + +\* Admission closed (or the child was terminated) during construction: the +\* attempt is cancelled and cleaned up; it never publishes. +AbortStart(s) == + /\ cs[s] \in {"starting", "restarting"} + /\ ~open \/ term[s] + /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, "Shutdown") + /\ UNCHANGED <> + +\* A ready child exits on its own. +SelfExit(s, r) == + /\ cs[s] = "running" /\ r \in SelfExits + /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) + /\ pub' = Set(pub, s, 0) + /\ UNCHANGED <> + +\* The supervisor's parent exit signal (Shutdown) reached the child, from +\* terminate_child or supervisor shutdown. It ends Shutdown, or Killed if +\* it missed shutdown_timeout and was killed (design E3). +StopObserved(s, r) == + /\ cs[s] = "running" /\ stopSig[s] + /\ r \in {"Shutdown", "Killed"} + /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) + /\ pub' = Set(pub, s, 0) + /\ UNCHANGED <> + +\* The generation reached publication and its owned cleanup joined. +CleanupDone(s) == + /\ cs[s] = "cleanup" + /\ alive' = Set(alive, s, {}) + /\ cs' = Set(cs, s, "decide") + /\ UNCHANGED <> + +\* Cleanup missed its deadline: the supervisor fails (and so stops every +\* other child); the generation stays alive until CleanupDone. +CleanupStuck(s) == + /\ cs[s] = "cleanup" /\ ~failed + /\ Fail + /\ UNCHANGED <> + +\* OTP's restart table, plus: a replacement that fails before readiness is a +\* failed restart attempt and is retried unless Temporary (V1). +Restarts(s) == + \/ Mode[s] = "Permanent" + \/ Mode[s] = "Transient" /\ reason[s] \in Abnormal + \/ Mode[s] # "Temporary" /\ ~readyGen[s] + \/ Mode[s] = "Temporary" /\ Fault = "RestartTemporary" + +Decide(s) == + /\ cs[s] = "decide" + /\ IF ~everReady[s] + \* First start failed: start_child returns the error; never restarted + \* and never charged to the budget (OTP start_child). + THEN /\ cs' = Set(cs, s, "removed") + /\ result' = Set(result, s, IF open THEN "failed" ELSE "shutdown") + ELSE IF ~open \/ (term[s] /\ Fault # "RestartTerminated") + THEN cs' = Set(cs, s, "removed") /\ UNCHANGED result + ELSE IF Restarts(s) + THEN cs' = Set(cs, s, "backoff") /\ UNCHANGED result + ELSE cs' = Set(cs, s, "removed") /\ UNCHANGED result + /\ UNCHANGED <> + +\* Backoff elapsed (or was cancelled). A restart needs a budget grant; +\* without one the supervisor fails and escalates. +BackoffDone(s) == + /\ cs[s] = "backoff" + /\ IF (~open /\ Fault # "RestartAfterClose") \/ (term[s] /\ Fault # "RestartTerminated") + THEN /\ cs' = Set(cs, s, "removed") + /\ UNCHANGED <> + ELSE IF budget < MaxRestarts + THEN /\ budget' = budget + 1 + /\ cs' = Set(cs, s, "restarting") + /\ gen' = Set(gen, s, gen[s] + 1) + /\ alive' = Set(alive, s, {gen[s] + 1}) + /\ reason' = Set(reason, s, "none") + /\ readyGen' = Set(readyGen, s, FALSE) + /\ UNCHANGED <> + ELSE /\ exhausted' = TRUE + /\ cs' = Set(cs, s, "removed") + /\ IF Fault = "NoEscalate" THEN UNCHANGED <> ELSE Fail + /\ UNCHANGED <> + /\ UNCHANGED <> + +----------------------------------------------------------------------------- +(* Supervisor shutdown: every child is signalled at once *) + +SupSignal == + /\ sphase = "running" /\ ~open /\ Fault # "Sequential" + /\ stopSig' = [s \in Slots |-> TRUE] + /\ sphase' = "waiting" + /\ UNCHANGED <> + +\* Control: one child at a time, the next only after the previous finished. +SupSignalOne == + /\ sphase \in {"running", "signal"} /\ ~open /\ Fault = "Sequential" + /\ \A s \in Slots : stopSig[s] => cs[s] \in Terminal + /\ LET todo == {s \in Slots : ~stopSig[s] /\ cs[s] \notin Terminal} + IN IF todo = {} + THEN sphase' = "waiting" /\ UNCHANGED stopSig + ELSE \E s \in todo : stopSig' = Set(stopSig, s, TRUE) /\ sphase' = "signal" + /\ UNCHANGED <> + +\* Done once every child finished (removed, rejected or never requested). +SupDone == + /\ sphase = "waiting" + /\ (\A s \in Slots : cs[s] \in Terminal) \/ Fault = "EarlyDone" + /\ sphase' = "done" + /\ UNCHANGED <> + +----------------------------------------------------------------------------- + +SlotStep(s) == + \/ Request(s) \/ TerminateChild(s) \/ Ready(s) + \/ \E r \in StartFails : StartFail(s, r) + \/ AbortStart(s) + \/ \E r \in SelfExits : SelfExit(s, r) + \/ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r) + \/ CleanupDone(s) \/ CleanupStuck(s) \/ Decide(s) \/ BackoffDone(s) + +Next == + \/ Close \/ WindowExpire + \/ SupSignal \/ SupSignalOne \/ SupDone + \/ \E s \in Slots : SlotStep(s) + +\* Progress: construction resolves, signalled children stop, cleanup ends +\* (CleanupStuck and crashes are not fair), the supervisor moves on. +Fairness == + /\ \A s \in Slots : + /\ WF_vars(Ready(s)) /\ WF_vars(AbortStart(s)) + /\ WF_vars(\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) + /\ WF_vars(CleanupDone(s)) /\ WF_vars(Decide(s)) /\ WF_vars(BackoffDone(s)) + /\ WF_vars(SupSignal) /\ WF_vars(SupSignalOne) /\ WF_vars(SupDone) + +Spec == Init /\ [][Next]_vars /\ Fairness + +----------------------------------------------------------------------------- +(* Safety *) + +\* A slot never has two live generations (V3). +AtMostOneGeneration == \A s \in Slots : Cardinality(alive[s]) <= 1 + +\* current() returns only the live, latest generation (V4). +PublishedIsLive == + \A s \in Slots : pub[s] # 0 => pub[s] = gen[s] /\ pub[s] \in alive[s] + +\* No child outlives the supervisor: done means every child finished. +NoChildOutlivesSupervisor == + sphase = "done" => \A s \in Slots : alive[s] = {} /\ cs[s] \in Terminal + +\* The live count, restarting children included, never exceeds the cap. +MaxChildrenBound == + Cardinality({s \in Slots : cs[s] \in Active}) <= MaxChildren + +TemporaryNeverRestarted == + \A s \in Slots : Mode[s] = "Temporary" => gen[s] <= 1 + +TerminatedNeverRestarted == + \A s \in Slots : term[s] => gen[s] = termGen[s] + +\* No start_child succeeds after shutdown begins. +NoStartAfterShutdown == \A s \in Slots : ~lateOk[s] + +\* No generation (first start or restart) begins after admission closed. +NoGenerationAfterClose == + \A s \in Slots : closeGen[s] # -1 => gen[s] <= closeGen[s] + +\* Budget exhaustion fails the supervisor, which closes it (and so stops +\* every child: ClosedLeadsToDone). +ExhaustionCloses == exhausted => failed /\ ~open + +BudgetBounded == budget <= MaxRestarts + +\* Shutdown is concurrent: once the supervisor started stopping, every +\* running child has its stop signal; none waits for a sibling. +ConcurrentShutdown == + sphase # "running" => \A s \in Slots : cs[s] = "running" => stopSig[s] + +\* start_child returned Ok only for a child that became ready. +OkMeansReady == \A s \in Slots : result[s] = "ok" => everReady[s] + +Safety == + /\ TypeOK /\ AtMostOneGeneration /\ PublishedIsLive + /\ NoChildOutlivesSupervisor /\ MaxChildrenBound + /\ TemporaryNeverRestarted /\ TerminatedNeverRestarted + /\ NoStartAfterShutdown /\ NoGenerationAfterClose + /\ ExhaustionCloses /\ BudgetBounded /\ ConcurrentShutdown /\ OkMeansReady + +NeverReopens == [][open' => open]_vars + +(* Liveness, assuming cleanup ends (CleanupStuck is not fair) *) + +\* Shutdown or failure ends with every child stopped. +ClosedLeadsToDone == (~open) ~> (sphase = "done") + +\* Every start_child call gets an answer. +StartAnswered == \A s \in Slots : (result[s] = "pending") ~> (result[s] # "pending") + +\* terminate_child completes: the child is removed. +TerminateCompletes == \A s \in Slots : term[s] ~> (cs[s] = "removed") +============================================================================= diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg new file mode 100644 index 00000000..bfd4599f --- /dev/null +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -0,0 +1,31 @@ +\* Fast instance (CI, every PR): three start_child calls (Permanent, +\* Transient, Temporary) against max_children 2, budget 1 with one window +\* release; safety, action property and liveness. Every DynamicSupervisor +\* negative control fails here on its named invariant. +SPECIFICATION Spec +CONSTANTS + N = 3 + Mode <- MCMode3 + MaxChildren = 2 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none", "Panic"} + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoStartAfterShutdown +INVARIANT NoGenerationAfterClose +INVARIANT ExhaustionCloses +INVARIANT BudgetBounded +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +PROPERTY NeverReopens +PROPERTY ClosedLeadsToDone +PROPERTY StartAnswered +PROPERTY TerminateCompletes diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla new file mode 100644 index 00000000..c2b3f681 --- /dev/null +++ b/verification/tla/MCDynamicSupervisor.tla @@ -0,0 +1,15 @@ +------------------------ MODULE MCDynamicSupervisor ------------------------ +(* Model-checking instances for DynamicSupervisor (rank 12a design). *) +EXTENDS DynamicSupervisor + +\* One start_child slot per restart type (fast, every PR). +MCMode3 == [s \in 1..3 |-> CASE s = 1 -> "Permanent" + [] s = 2 -> "Transient" + [] OTHER -> "Temporary"] +\* Four slots for the deep instance: two Permanent so two restarting +\* children can hold the cap together. +MCMode4 == [s \in 1..4 |-> CASE s = 1 -> "Permanent" + [] s = 2 -> "Transient" + [] s = 3 -> "Temporary" + [] OTHER -> "Permanent"] +============================================================================= diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg new file mode 100644 index 00000000..825f6fa4 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -0,0 +1,27 @@ +\* Deep instance (weekly, --deep): four start_child calls against +\* max_children 3, budget 2 with two window releases, every exit reason +\* and startup failure; safety and the action property. +SPECIFICATION Spec +CONSTANTS + N = 4 + Mode <- MCMode4 + MaxChildren = 3 + MaxRestarts = 2 + MaxExpire = 2 + SelfExits = {"Normal", "Shutdown", "Panic", "Kill"} + StartFails = {"none", "timeout", "Normal", "Panic"} + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoStartAfterShutdown +INVARIANT NoGenerationAfterClose +INVARIANT ExhaustionCloses +INVARIANT BudgetBounded +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +PROPERTY NeverReopens diff --git a/verification/tla/MCDynamicSupervisorDeep.tla b/verification/tla/MCDynamicSupervisorDeep.tla new file mode 100644 index 00000000..05e74666 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorDeep.tla @@ -0,0 +1,3 @@ +---------------------- MODULE MCDynamicSupervisorDeep ---------------------- +EXTENDS MCDynamicSupervisor +============================================================================= From ec5492f0e6de780291780af15ee35bf841c3cf4b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:39:00 -0300 Subject: [PATCH 03/52] verify: DynamicSupervisor without negative sentinels --- verification/tla/DynamicSupervisor.tla | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index fa3fc3a1..88518fc0 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -105,7 +105,7 @@ VARIABLES stopSig, \* the supervisor's stop signal reached this slot \* Ghosts for the invariants. termGen, \* gen when terminate_child was called - closeGen, \* gen of each slot when admission closed, -1 while open + closeGen, \* gen of each slot when admission closed (meaningful once closed) lateOk, \* a start_child returned ok after admission closed exhausted \* a restart was refused for lack of budget @@ -127,7 +127,7 @@ TypeOK == /\ result \in [Slots -> Results] /\ term \in [Slots -> BOOLEAN] /\ stopSig \in [Slots -> BOOLEAN] /\ termGen \in [Slots -> Nat] - /\ closeGen \in [Slots -> Nat \cup {-1}] + /\ closeGen \in [Slots -> Nat] /\ lateOk \in [Slots -> BOOLEAN] /\ exhausted \in BOOLEAN @@ -145,7 +145,7 @@ Init == /\ term = [s \in Slots |-> FALSE] /\ stopSig = [s \in Slots |-> FALSE] /\ termGen = [s \in Slots |-> 0] - /\ closeGen = [s \in Slots |-> -1] + /\ closeGen = [s \in Slots |-> 0] /\ lateOk = [s \in Slots |-> FALSE] /\ exhausted = FALSE @@ -397,7 +397,7 @@ NoStartAfterShutdown == \A s \in Slots : ~lateOk[s] \* No generation (first start or restart) begins after admission closed. NoGenerationAfterClose == - \A s \in Slots : closeGen[s] # -1 => gen[s] <= closeGen[s] + ~open => \A s \in Slots : gen[s] <= closeGen[s] \* Budget exhaustion fails the supervisor, which closes it (and so stops \* every child: ClosedLeadsToDone). From ddd86174f41576ba65ea18090742e8c06c73e4dd Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:42:02 -0300 Subject: [PATCH 04/52] verify: DynamicSupervisor negative controls (9 safety, 2 liveness) --- verification/tla/DynamicSupervisor.tla | 11 +++++++++-- verification/tla/negative_checks.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 88518fc0..9ac5f29b 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -46,6 +46,11 @@ (* Sequential (ConcurrentShutdown) children stopped one at a *) (* time, as a static *) (* supervisor does *) +(* and two liveness controls (named_temporal): *) +(* NoAbort (StartAnswered) a start racing shutdown *) +(* is not cancelled *) +(* StuckBackoff (ClosedLeadsToDone) a pending restart is not *) +(* cancelled by shutdown *) (***************************************************************************) EXTENDS Naturals, FiniteSets @@ -72,7 +77,8 @@ ASSUME \A s \in Slots : Mode[s] \in Modes ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons ASSUME Fault \in {"None", "IgnoreMax", "UncountedRestart", "RestartTemporary", "RestartTerminated", "AcceptDuringShutdown", "RestartAfterClose", - "NoEscalate", "EarlyDone", "Sequential"} + "NoEscalate", "EarlyDone", "Sequential", "NoAbort", + "StuckBackoff"} \* Slot states. "idle": start_child not called yet. "starting": the first \* generation is being constructed (the caller waits). "restarting": a @@ -236,7 +242,7 @@ StartFail(s, r) == \* attempt is cancelled and cleaned up; it never publishes. AbortStart(s) == /\ cs[s] \in {"starting", "restarting"} - /\ ~open \/ term[s] + /\ (~open /\ Fault # "NoAbort") \/ term[s] /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, "Shutdown") /\ UNCHANGED <> @@ -297,6 +303,7 @@ Decide(s) == \* without one the supervisor fails and escalates. BackoffDone(s) == /\ cs[s] = "backoff" + /\ open \/ term[s] \/ Fault # "StuckBackoff" /\ IF (~open /\ Fault # "RestartAfterClose") \/ (term[s] /\ Fault # "RestartTerminated") THEN /\ cs' = Set(cs, s, "removed") /\ UNCHANGED <> diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 33cb1017..70046a1d 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -180,6 +180,24 @@ def named_temporal(module, fault, prop, tlc, constants=None): ] LIFECYCLE_SMALL = {"Reqs": "{}", "MaxSignals": "0", "DropEnabled": "FALSE"} +# Rank 12a design model (docs/design/dynamic-supervisor.md): each mutation +# against the invariant it must break, on the per-PR configuration. +DYNAMIC_CONTROLS = [ + ("MCDynamicSupervisor", "IgnoreMax", "MaxChildrenBound"), + ("MCDynamicSupervisor", "UncountedRestart", "MaxChildrenBound"), + ("MCDynamicSupervisor", "RestartTemporary", "TemporaryNeverRestarted"), + ("MCDynamicSupervisor", "RestartTerminated", "TerminatedNeverRestarted"), + ("MCDynamicSupervisor", "AcceptDuringShutdown", "NoStartAfterShutdown"), + ("MCDynamicSupervisor", "RestartAfterClose", "NoGenerationAfterClose"), + ("MCDynamicSupervisor", "NoEscalate", "ExhaustionCloses"), + ("MCDynamicSupervisor", "EarlyDone", "NoChildOutlivesSupervisor"), + ("MCDynamicSupervisor", "Sequential", "ConcurrentShutdown"), +] +DYNAMIC_TEMPORAL_CONTROLS = [ + ("NoAbort", "StartAnswered"), + ("StuckBackoff", "ClosedLeadsToDone"), +] + DEFERRED_CONTROLS = [ ("MCDeferredReplies", "NoAutoReply", "NoOrphan"), ("MCDeferredReplies", "NoAtomicExpiry", "AtMostOneDischarge"), @@ -266,6 +284,10 @@ def fail_instead_of_retry(events): named_fault(module, fault, invariant, args.tlc) for fault, prop in LIFECYCLE_TEMPORAL_CONTROLS: named_temporal("MCLifecycleLive", fault, prop, args.tlc, LIFECYCLE_SMALL) + for module, fault, invariant in DYNAMIC_CONTROLS: + named_fault(module, fault, invariant, args.tlc) + for fault, prop in DYNAMIC_TEMPORAL_CONTROLS: + named_temporal("MCDynamicSupervisor", fault, prop, args.tlc) if __name__ == "__main__": From 30886bd00153c640c64ad96854c590696f1b02fe Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:46:25 -0300 Subject: [PATCH 05/52] docs: dynamic supervisor design contract (rank 12a) --- docs/design/dynamic-supervisor.md | 740 ++++++++++++++++++++++++++++++ 1 file changed, 740 insertions(+) create mode 100644 docs/design/dynamic-supervisor.md diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md new file mode 100644 index 00000000..0eeaf442 --- /dev/null +++ b/docs/design/dynamic-supervisor.md @@ -0,0 +1,740 @@ +# Dynamic supervisor: design contract + +**Status: design in review, not implemented.** ROADMAP rank 12a. Draft PR +[#245](https://github.com/lambdaclass/spawned/pull/245). + +- Covers OTP's `simple_one_for_one` and Elixir's `DynamicSupervisor`: children + started at runtime from one template, supervised one for one, terminated + individually, and all stopped concurrently when the supervisor stops. +- Builds on the v0.8 exit-signal contract ([otp-exit-signals.md](otp-exit-signals.md), + accepted, implementation in #240): the single death point D, `Killed`, + `NoProc`, the parent exit signal (E3), first-start failure as + `SupervisorError::FailedToStartChild` (S1, S2) and `stopped()` skipped for a + non-trapping child whose parent sends `Shutdown`. It also builds on rank 11 + ([deferred-replies.md](deferred-replies.md), #239) for what callers of a + dying child see. +- Model: [`DynamicSupervisor.tla`](../../verification/tla/DynamicSupervisor.tla); + see [What the model establishes](#what-the-model-establishes). +- Client mapping: [dynamic-supervisor-client-mapping.md](dynamic-supervisor-client-mapping.md) + (ethrex's per-peer connection actors; ethlambda has none). +- Illustrative API shapes are not copyable code. + +**Project rule.** Match Erlang/OTP. Deviate only where: + +- **R:** Rust forces it; +- **T:** Rust types add strictness; or +- **B:** Spawned adds a bound OTP lacks. + +Where this design keeps an existing Spawned behaviour that OTP does not have +(backoff, concurrent restarts), it says so and why; those are listed +separately from new deviations. + +## Contents + +- [Sources](#sources) +- [Goals and non-goals](#goals-and-non-goals) +- [API](#api) +- [Semantics](#semantics) +- [Erlang alignment](#erlang-alignment) +- [Non-guarantees](#non-guarantees) +- [Semantics contract: what this adds and touches](#semantics-contract-what-this-adds-and-touches) +- [Interaction with rank 12 (scoped registries)](#interaction-with-rank-12-scoped-registries) +- [What the model establishes](#what-the-model-establishes) +- [Alternatives considered](#alternatives-considered) +- [Open questions](#open-questions) + +## Sources + +Read for this design (line numbers refer to these revisions): + +- OTP `lib/stdlib/src/supervisor.erl` at master + [`186c0b94`](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl) + (OTP 29 development; compared with `OTP-28.5.0.8`, no behavioural difference + for dynamic children). +- Elixir `lib/elixir/lib/dynamic_supervisor.ex` at main + [`b57b70bd`](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex) + (compared with `v1.19.7`; only the shutdown code differs). + +What OTP and Elixir do, in short: + +| Topic | OTP `simple_one_for_one` | Elixir `DynamicSupervisor` | +|---|---|---| +| `start_child` | `gen_server:call(Sup, …, infinity)`; the child's `init` runs inside the supervisor, so starts are serialised with each other and with `'EXIT'` handling ([L652](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L652), [L1077-1092](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1077-L1092)) | same; "will block the `DynamicSupervisor` until the child initializes" ([L359](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L359)) | +| Arguments | `ExtraArgs` appended to the template's | `extra_arguments` (fixed per supervisor) prepended to each spec's args ([L752](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L752)) | +| Specs | one template | any child spec per call | +| Start result | `{ok, Pid}`, `{ok, Pid, Info}`, `{ok, undefined}` on `ignore` (nothing stored), `{error, Reason}` unwrapped | same, but `:ignore` bare | +| Start failure and the budget | not counted: `add_restart` runs only in `restart/2` | not counted | +| `terminate_child` | by pid; any restart type; removed, never restarted; an unknown dead pid returns `ok` ([L1108-1115](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1108-L1115), [L1861-1875](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1861-L1875)) | by pid; unknown pid `{:error, :not_found}` ([L712-721](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L712-L721)) | +| `restart_child`, `delete_child` | `{error, simple_one_for_one}` | not provided | +| `which_children` | `{undefined, Pid \| restarting, Type, Mods}`, unordered | same | +| `count_children` | `specs` always 1; `active` excludes restarting; `workers`/`supervisors` include them ([L1218-1229](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1218-L1229)) | `specs` = number of children; otherwise the same | +| Default restart type | `permanent` ([L449](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L449)) | `:permanent` ([L404-413](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L404-L413)); `Task.Supervisor` defaults to `:temporary` | +| Temporary children | start args not kept (`mapsets`), "it could become very costly" ([L1813-1817](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1813-L1817)) | args dropped (`mfa_for_restart`, [L784](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L784)) | +| Restart intensity | default 1 in 5 s; `add_restart` before every restart attempt, also each `try_again_restart` retry ([L1477-1531](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1477-L1531)) | default 3 in 5 s; each `$gen_restart` retry counts | +| Exhaustion | logs `reached_max_restart_intensity`, terminates every child, exits `shutdown` | same | +| `max_children` | none | `:infinity` by default; checked only in `start_child`, `map_size(children) < max_children`, which includes restarting entries; restarts never re-check ([L742](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L742)) | +| Shutdown | `terminate_dynamic_children`: monitor and `exit(P, shutdown)` to every child first, one timer for the template's shutdown value, then `kill` to every child still alive; order "is not defined" ([L1701-1799](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1701-L1799)) | same, one timer per distinct shutdown value | +| Static strategies, for contrast | children stopped one at a time in reverse start order, each waited for | | +| Restarted supervisor | starts with no children | same | +| `auto_shutdown`, significant children | rejected for `simple_one_for_one` | only `:never` | + +The ROADMAP entry says counting restarting children against `max_children` +"goes beyond Elixir". It does not: Elixir keeps a restarting child in its +`children` map under the old pid, so it counts. This design matches Elixir +there, and the ROADMAP row is corrected in this PR. + +## Goals and non-goals + +Goals: + +1. Start supervised children at runtime, each from a template plus per-call + arguments, with a synchronous `start_child` that returns the child or a + typed error. +2. OTP's restart types, supervisor-wide restart budget and escalation. +3. Terminate one child without restarting it. +4. Optional `max_children` that rejects at once, counting restarting + children. +5. Stop every child concurrently on shutdown, bounded by the per-child + shutdown timeout and the kill, not by the number of children. +6. Placement on one `ExecutionGroup`; ownership by an `ActorSystem`; + nesting under a static supervisor. + +Non-goals (v1): + +- Heterogeneous children (Elixir's any-spec `start_child`). See + [Alternatives](#alternatives-considered) and open question 2. +- A keyed registry owned by the supervisor (rank 12; see + [Interaction with rank 12](#interaction-with-rank-12-scoped-registries)). +- Supervisor-type dynamic children (a template that builds a nested + supervisor per call). Possible later through the same `SupervisorSpec` + adapter the static supervisor uses. +- `ignore` from a factory (the static supervisor does not have it either, + S1). +- `Task.Supervisor` (ranks 11 and 15), `PartitionSupervisor` (sharding), + significant children and `auto_shutdown` (OTP rejects them for + `simple_one_for_one`). +- Draining children on shutdown (rank 13), bounded mailboxes (rank 17). + +## API + +### Template + +```rust +/// One template for every child: placement, factory, restart type and limits. +/// The factory gets the per-call arguments (OTP's ExtraArgs). +pub struct DynamicChildSpec { /* name, group, factory, restart, timeouts */ } + +impl DynamicChildSpec { + /// Permanent, as OTP's default. Restartable children keep their + /// arguments for the next generation, hence `Clone`. + pub fn new(name: impl Into, group: &ExecutionGroup, factory: F) -> Self + where + F: Fn(FactoryContext, Args) -> U + Send + Sync + 'static, + U: Future> + Send + 'static, + E: std::fmt::Display; + pub fn restart(self, restart: RestartMode) -> Self; +} + +impl DynamicChildSpec { + /// A Temporary template. Arguments are moved into the only generation + /// and never kept (OTP stores no arguments for temporary children), so + /// they need not be `Clone`: a `TcpStream` or a semaphore permit works. + pub fn temporary(name: impl Into, group: &ExecutionGroup, factory: F) -> Self; +} + +impl DynamicChildSpec { + pub fn startup_timeout(self, limit: Duration) -> Self; // default 5 s + pub fn shutdown_timeout(self, limit: Duration) -> Self; // default 5 s +} +``` + +Two constructors instead of one is a Rust-forced shape (R): a restartable +child must be rebuilt from its arguments, so they must be cloneable, while a +temporary one must accept move-only values. `RestartMode::Temporary` passed +to `restart()` is still allowed (it simply never clones). The restart type is +per template, as in `simple_one_for_one`. + +### Starting the supervisor + +```rust +pub struct DynamicSupervisorConfig { + pub max_restarts: usize, // default 5 + pub restart_window: Duration, // default 30 s + pub max_children: Option, // default None (Elixir :infinity) + pub backoff: Backoff, // default as SupervisorConfig +} + +// Root, owned by the system (shutdown unit in admission order, S2): +let peers: DynamicSupervisor = + system.dynamic_supervisor(config, spec)?.start().await?; + +// Nested under a static supervisor, as `add_supervisor` (V8): +let slot: SupervisedDynamicSupervisor = + builder.add_dynamic_supervisor(DynamicSupervisorSpec::new("peers", &group, config, || spec()))?; +let peers = slot.current()?; // re-resolve after the parent restarts it +``` + +`start()` returns once the supervisor is admitted; it has no children to +start, so it cannot fail with `FailedToStartChild`. Configuration errors +(`max_children = Some(0)`, a zero window, `backoff.maximum < initial`, a +zero timeout, a foreign group) are `SupervisorError::Configuration` or +`System(ForeignGroup)` at construction, as for the static supervisor. The +`max_children <= 65536` ceiling of `SupervisorConfig` does not apply: the +static ceiling bounds a declaration list, while the dynamic count is the +user's explicit choice. + +### Operations + +```rust +impl DynamicSupervisor { + /// OTP start_child(Sup, ExtraArgs). Resolves when the first generation + /// is ready (started() returned) or failed. + pub async fn start_child(&self, args: Args) -> Result, StartChildError>; + /// OTP terminate_child(Sup, Pid). Resolves after the child finished. + pub async fn terminate_child(&self, id: ActorId) -> Result<(), TerminateChildError>; + pub fn which_children(&self) -> Vec; + pub fn count_children(&self) -> ChildCounts; + pub fn stop(&self); + pub async fn shutdown(&self, deadline: Duration) -> SupervisorReport; + pub async fn join(&self); + pub fn failure(&self) -> Option; +} + +pub struct Started { + pub actor: ActorRef, // the first generation, as OTP's Pid + pub child: DynamicChild, // the slot, across restarts +} + +/// Like `SupervisedChild` for a dynamic slot. +impl DynamicChild { + pub fn id(&self) -> DynamicChildId; // stable for the slot + pub fn current(&self) -> Result, ChildUnavailable>; + pub async fn wait_for_generation(&self, after: u64) -> Result, ChildUnavailable>; + pub fn snapshot(&self) -> ChildSnapshot; + /// terminate_child for whatever generation is current, including a + /// restart in backoff. + pub async fn terminate(&self) -> Result<(), TerminateChildError>; +} + +#[non_exhaustive] +pub enum StartChildError { + /// max_children reached (Elixir {:error, :max_children}). Nothing queued. + MaxChildren, + /// The supervisor is shutting down or has stopped (OTP: the caller's + /// call exits with the supervisor's reason, or noproc). + Stopped, + /// The first generation failed: factory error or panic, panic or stop + /// in started(), startup_timeout. The child is not kept and nothing is + /// charged to the budget (OTP {error, Reason}). + FailedToStart { cause: String }, + /// Placement: the group is closed or at capacity. + System(SystemError), +} + +#[non_exhaustive] +pub enum TerminateChildError { + /// Not a current child of this supervisor (Elixir {:error, :not_found}). + NotFound, + /// The supervisor is shutting down; it stops the child itself. + Stopped, + /// The child's cleanup missed its deadlines; the supervisor failed (V3). + CleanupStuck, +} + +pub struct DynamicChildInfo { + pub child: DynamicChildId, + pub actor: Option, // None while restarting (OTP `restarting`) + pub state: ChildState, + pub generation: u64, +} + +pub struct ChildCounts { + pub specs: usize, // 1, as OTP simple_one_for_one + pub active: usize, // children with a live, ready generation + pub supervisors: usize, // 0 in v1 (no supervisor-type children) + pub workers: usize, // every child, restarting ones included +} +``` + +`ChildState` gains no variant: a dynamic child is `Starting`, `Running`, +`Backoff` (restart pending), `Stopping`, then gone from `which_children`. +`which_children` is a snapshot in no particular order, as OTP's. + +### Restart types and defaults + +| | OTP | This design | +|---|---|---| +| Default restart type | `permanent` | `Permanent` (`DynamicChildSpec::new`); `temporary()` for one-shot children such as connections | +| Restart table | `do_restart/3` | unchanged V1 table: Permanent always; Transient on an abnormal reason (`Panic`, `Kill`, `Killed`, `NoProc`) or a cleanup failure (exit-signals design); Temporary never | +| A replacement failing before readiness | `try_again_restart`, retried | retried unless Temporary (V1), each attempt charged | +| Explicit `terminate_child` | never restarted | never restarted, any type | + +### Restart budget + +- **Scope:** one rolling window per supervisor, shared by all its children, + as OTP's intensity and period. +- **What counts:** every restart attempt: the factory call of every + generation after a child's first, including retries of a replacement that + failed before readiness (OTP calls `add_restart` before each restart and + each `try_again_restart`). A dynamic supervisor has no collateral + restarts, so v0.7's "every factory call" and rank 5f's "recovery rounds" + give the same count here. +- **What does not count:** a failed `start_child` (OTP), a Temporary child's + exit, an exit after `terminate_child`, exits during shutdown. +- **Defaults:** `max_restarts = 5`, `restart_window = 30 s`, the static + supervisor's defaults. OTP uses 1 in 5 s and Elixir 3 in 5 s. Spawned's + is looser because per-child exponential backoff (V6) already spreads + restarts: a crash-looping child still uses up 5 grants in well under a + second (backoff bounds 10, 20, 40, 80, 160 ms), so the looser window does + not let a loop run long, while a supervisor of many children tolerates + more unrelated crashes. Open question 3. +- **Exhaustion:** the supervisor fails (`failure()` records + "restart budget exhausted" with the child and its last reason), closes + admission, stops every child as in a shutdown and escalates: a nested + dynamic supervisor exits abnormally and its parent applies its own policy + (V8); a root supervisor reports it in `ShutdownReport::supervisor_failures`. + OTP exits `shutdown` here; see [alignment](#erlang-alignment) row 15. +- **Reservation:** as V5, capacity is reserved before backoff and refunded if + the backoff is cancelled (by `terminate_child` or shutdown). An exhausted + window fails at once without waiting for the backoff. + +### `max_children` + +- `None` by default (Elixir `:infinity`). `Some(n)` caps the number of + children the supervisor holds: starting, running, cleaning up, in backoff + or constructing a replacement. A slot is freed only when its child is + removed: its last generation finished cleanup (milestone 7 of the exit + phases) and no restart follows. +- Checked only by `start_child`, atomically with admission. A full + supervisor returns `Err(MaxChildren)` at once; nothing waits or queues. +- Restarts are never re-checked (Elixir) and never need to be: the slot they + use was never freed. So a new start cannot take the place of a child + that is restarting, and a restart cannot fail for want of a slot. +- Spawned frees the slot after cleanup, not at the exit signal (OTP removes + the child when it processes the `'EXIT'`). B: the cap then bounds what is + really held (state, sockets, helpers), not only live processes. + +### Placement and worker budgets + +- Every child runs on the template's `ExecutionGroup`; factories run there + too, as for the static supervisor. The group must belong to the + supervisor's `ActorSystem` (`ForeignGroup` otherwise). +- Async groups: children share the group's workers; `max_children` is the + bound on how many there are. A dynamic supervisor does not change worker + budgets; the supervisor's own bookkeeping runs on the system's control + runtime, outside every group budget (as today). +- `thread_group(n)` (one thread per live actor, `Capacity` beyond `n`) and + pinned groups: a child's slot holds its group lease from admission until + removal, across restarts, so a restart is never refused for capacity that + a newer `start_child` took. A start that finds the group full returns + `Err(System(Capacity))` and takes no slot. Implementation note: this needs + a lease that outlives one actor instance; today a lease is per live scope. +- A closed group (system shutdown) makes `start_child` return + `Err(System(Closed))` and fails restarts, which fails the supervisor (as + static supervisors treat group closure). + +### Nesting and the system + +- Root: `system.dynamic_supervisor(config, spec)` is a shutdown unit in + admission order, stopped in its turn by `ActorSystem::shutdown` (S2); its + children stop concurrently in that turn. +- Under a static supervisor: `add_dynamic_supervisor` uses the nested + adapter of `add_supervisor` (V8). The parent sees one child; it stops it + in reverse declaration order like any child; a restart by the parent + creates a new, empty dynamic supervisor (OTP: a restarted + `simple_one_for_one` supervisor starts with no children). Old + `DynamicSupervisor` handles keep pointing at the dead instance and return + `Stopped`; `SupervisedDynamicSupervisor::current()` resolves the new one. +- A dynamic supervisor's children are actors in v1. A child may itself start + a static supervisor in its factory only as unmanaged work, which is not + supported; nested supervisor templates are a follow-up. + +## Semantics + +### `start_child` + +1. **Admission** is one atomic step on the supervisor's gate: if the + supervisor is closed (shutdown, failure or exhaustion began), `Stopped`; + else if `max_children` is reached, `MaxChildren`; else the group lease is + reserved (`System(..)` if it cannot be) and a slot is created in + `Starting`, counted from this moment. +2. **Construction** runs the factory on the group and then `started()`, + bounded by `startup_timeout`, as a static child's first generation. + Concurrent `start_child` calls construct concurrently (see + [alignment](#erlang-alignment) row 2). +3. **Commit.** Readiness commits under the gate only while it is open. Then + `start_child` returns `Ok(Started)` and the child is supervised. +4. **First-start failure.** Factory error or panic, panic or stop in + `started()`, or `startup_timeout`: the generation is cleaned up, the slot + removed, `Err(FailedToStart { cause })` returned after that cleanup (S2's + "after rollback" rule, for one child). Never retried, never charged to the + budget, whatever the restart type (OTP returns `{error, Reason}`). +5. **Cancellation.** Dropping the `start_child` future does not cancel the + start (as R4: giving up never cancels). The child, once ready, is + supervised like any other and visible in `which_children`. OTP behaves the + same when the caller of `start_child` dies. + +### Races + +| Race | Outcome | +|---|---| +| `start_child` vs shutdown, admission first | the attempt is cancelled at shutdown: if `started()` had not begun, the state is dropped without hooks (L2); otherwise the generation gets the parent exit signal like a running child. `start_child` returns `Stopped`. It never returns `Ok` after shutdown began (model: `NoStartAfterShutdown`) | +| `start_child` vs shutdown, shutdown first | `Stopped` at admission; nothing is constructed | +| `start_child` vs a sibling's crash | the crashed child keeps its slot through cleanup, backoff and the replacement, so at the cap the new start gets `MaxChildren` (model: `MaxChildrenBound`, control `UncountedRestart`) | +| `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup; a start before that sees it counted | +| `start_child` vs budget exhaustion | exhaustion closes the gate; as shutdown | +| `terminate_child` vs a crash of that child | whichever is first under the slot lock wins. Crash first: if a restart is pending, `terminate_child` cancels it (refunding the budget) and removes the child; if the replacement is constructing, it is cancelled. Terminate first: the crash reason is recorded, no restart. Either way the child is removed and never restarted (model: `TerminatedNeverRestarted`) | +| `terminate_child` vs shutdown | after shutdown began, `Stopped`: the supervisor is already stopping every child | +| `terminate_child(id)` with an old generation's id | `NotFound` once that generation is not the slot's current one (a newer generation started); during backoff the last generation's id still names the slot. `DynamicChild::terminate()` targets the slot whatever its generation | + +### Restart identity + +- Every generation has a new `ActorId`, mailbox and state (M4). The slot's + `DynamicChildId` does not change; `DynamicChild::current()` returns the + latest ready generation, `wait_for_generation(n)` waits for a newer one + (V4). Old `ActorRef`s stay bound to their instance and get `NoProc` (v0.8). +- `which_children` shows the current `ActorId`, or `None` while restarting + (OTP's `restarting`). +- Callers that look children up by a domain key (a peer id) keep their own + map from key to `DynamicChild` until rank 12 provides keyed registration; + see [rank 12](#interaction-with-rank-12-scoped-registries). + +### Terminating one child + +`terminate_child` is OTP's `shutdown/2` for one child, under the exit-signal +design's E3 row for a supervisor stopping a child: + +1. Mark the slot terminated (no restart from now on), cancel a pending + backoff or replacement. +2. Send the parent exit signal `Shutdown`: a non-trapping child begins exit + after its current handler and skips `stopped()` (E2, decision 2); a + trapping child handles the messages queued before the signal, then runs + `stopped()` (E3). +3. If the child has not reached its death point `shutdown_timeout` after the + signal, kill it (`Killed`, E1); if it misses a second `shutdown_timeout` + the cleanup is stuck: the supervisor fails (V3) and `terminate_child` + returns `CleanupStuck`. +4. Resolve after the child's cleanup finished and the slot was removed. + +### Shutdown of all children + +When the supervisor closes (`stop`, `shutdown(deadline)`, system shutdown, +its parent stopping it, or a failure): + +1. Admission closes: `start_child` returns `Stopped`; pending restarts and + constructions are cancelled (budget refunded); starts in construction are + cancelled as in [Races](#races). +2. **Every live child gets the parent exit signal at once**, not in reverse + order: dynamic children have no declared order (OTP: "not defined") and + sequential stops would make shutdown take up to N × `shutdown_timeout`. + Model: `ConcurrentShutdown`, control `Sequential`. +3. Each child then follows the E3 deadlines from that one moment: kill after + `shutdown_timeout`, stuck after a second one. As every child shares the + template's timeout, this is OTP's single timer. Shutdown therefore takes + at most 2 × `shutdown_timeout` plus cleanup, whatever N is. +4. **Done** means every slot was removed: every generation reached + publication and its owned cleanup finished (V9 "done"). Model: + `NoChildOutlivesSupervisor`, `ClosedLeadsToDone`. +5. `shutdown(deadline)` keeps the static meaning: `deadline` is the report + deadline; at it the report is returned with `complete = false` if + children remain, and they stay owned by the system (S3). + +### In-flight requests and deferred replies of a dying child + +Nothing is specific to dynamic supervision; the child dies as any actor +(exit-signals design, phases of an exit, with rank 11): + +- At the child's death point D every queued request and every open + `ReplyTo` token is answered `ActorStopped(final reason)`: `Shutdown` after + a clean stop, `Killed` after the kill, the crash reason after a crash. A + request or registration after D gets `NoProc`. +- A non-trapping child stopped by its supervisor skips `stopped()`, so + deferred tokens it stored are answered at D, not by `stopped()` code. + Cleanup that must always run belongs in `Drop` or needs `trap_exit` (the + exit-signals migration note). +- Pipe jobs of the child are cancelled at exit begin and joined after + publication (rank 11 step 4), within the cleanup deadline; the slot is + freed only after that join. +- A replacement never receives the old generation's queue, tokens or jobs + (V4; rank 11 `ActorId` fencing). + +### Links and monitors + +- The supervisor is the parent of every child generation (E3), so its + `Shutdown` is a parent exit signal. It is not an actor: it observes + children through internal completion state, not links or a mailbox (V9), + so a child's backlog never delays its restart decision and a child cannot + unlink itself from supervision (OTP guards against such "naughty" children + with monitors; here there is nothing to unlink). +- Children can link to and monitor each other and outside actors as usual; + supervision does not add or remove those edges. A link between two + children propagates a crash under X3 and both are then handled by the + restart table independently (one for one). +- Outside actors that track children should monitor them + (`ctx.monitor(&started.actor.child_handle())`) rather than rely on the + child's `stopped()`: the `Down` arrives exactly once with the final reason, + also after a kill, a skipped `stopped()` or a runtime drop. This is what + replaces ethrex's self-reported removal ([client mapping](dynamic-supervisor-client-mapping.md)). + +### Budget exhaustion takes everything down + +Exhaustion is a supervisor failure: it closes admission and stops every +child exactly as a shutdown, then escalates (V8). Nothing restarts after the +failure (model: `ExhaustionCloses`, `NoGenerationAfterClose`, control +`NoEscalate`). A Temporary child never consumes budget, so a supervisor of +Temporary children (ethrex's case) fails only on a stuck cleanup or a closed +group. + +## Erlang alignment + +| # | Rule | OTP / Elixir | This design | Reason for any deviation | +|---|---|---|---|---| +| 1 | `start_child` is synchronous and returns the child | `{ok, Pid}` after `init` | `Ok(Started { actor, child })` after `started()` | aligned; `child` is an addition (row 9) | +| 2 | Starts are serialised in the supervisor | the supervisor runs each child's `init` and blocks meanwhile | admission is serialised (atomic count and gate); construction runs concurrently, bounded by `startup_timeout` | kept Spawned behaviour, not R/T/B: the supervisor is not a process, and static supervisors already restart children concurrently (v0.7). Observable difference is timing only: a slow `started()` does not delay other starts or restart decisions. Open question 1 | +| 3 | Template plus per-call args | `ExtraArgs` appended (Elixir: fixed `extra_arguments` prepended) | typed `Args` passed to the factory | T: one typed argument instead of a term list | +| 4 | `ignore` | `{ok, undefined}`, nothing stored | not supported | as the static supervisor (S1); a factory cannot return it | +| 5 | Start failure | `{error, Reason}`, not counted, not retried | `FailedToStart { cause }`, not counted, not retried, returned after cleanup | aligned (S2 ordering) | +| 6 | Default restart type | `permanent` | `Permanent` | aligned | +| 7 | Temporary children keep no args | `mapsets` | `temporary()` moves args once, no `Clone` | aligned; R for the separate constructor | +| 8 | `terminate_child` | by pid; unknown dead pid `ok` (OTP) or `not_found` (Elixir) | by `ActorId`; `NotFound` for anything not current | Elixir's answer: OTP's `ok` for a dead unknown pid is an artefact of `find_child` | +| 9 | Restart identity | new pid, found through names | new `ActorId`; stable `DynamicChildId` and `DynamicChild::current()` | addition, consistent with `SupervisedChild` (references are instance-bound, an accepted decision); no OTP behaviour changes | +| 10 | `restart_child`, `delete_child` | `{error, simple_one_for_one}` | not provided | aligned (absent at compile time: T) | +| 11 | `which_children` | pid or `restarting`, unordered | `Option`, unordered, with state and generation | aligned plus fields | +| 12 | `count_children` | `specs` 1, `active` excludes restarting | same | aligned with OTP (Elixir's `specs` differs) | +| 13 | `max_children` | Elixir: default infinity, start only, counts restarting | same; frees after cleanup | aligned with Elixir; B for freeing after cleanup | +| 14 | Restart budget | intensity/period, every restart attempt counts, start failures do not | same rule; defaults 5 in 30 s | defaults: kept Spawned behaviour (backoff, V6); open question 3 | +| 15 | Exhaustion | terminate children, supervisor exits `shutdown` | terminate children concurrently, fail, escalate; the nested adapter exits abnormally (as static nested supervisors today) | existing static-supervisor difference, not introduced here: a `Transient` OTP supervisor that exhausts its budget is not restarted by its parent, a Spawned one is. Open question 5 | +| 16 | Shutdown order | parallel, undefined order, one timer, then `kill` | parallel, per-child `shutdown_timeout` from one moment, then `kill`, then stuck | aligned; the stuck stage is R (Rust cannot kill a thread; exit-signals design) | +| 17 | Shutdown signal | `exit(Child, shutdown)`; trapping child runs `terminate` | parent exit signal: trapping child runs `stopped()` after its backlog; non-trapping skips `stopped()` | aligned (E2, E3) | +| 18 | `start_child` during shutdown | call queued, caller exits with the supervisor's reason or `noproc` | `Err(Stopped)` | R: Rust has no caller exit; a typed error is the counterpart | +| 19 | Restarted dynamic supervisor | empty | empty | aligned | +| 20 | Backoff | none: restart at once | per-child backoff (V6) | kept Spawned behaviour; budget refunded if cancelled | +| 21 | Shutdown admission | a stopping supervisor's children may start anything | the whole tree closes admission at once | B (exit-signals design) | +| 22 | Cleanup before a slot frees or a replacement starts | the process is gone at `'EXIT'` | after owned cleanup (V3) | R: destructors and owned helpers outlive the last hook | +| 23 | Restarted-child calls in flight | `'EXIT'`/monitor tells callers | `ActorStopped(reason)` at D, `NoProc` after (rank 11, v0.8) | aligned | +| 24 | Heterogeneous specs | Elixir: any spec per call | one typed template | T: `start_child` returns `ActorRef`; open question 2 | + +## Non-guarantees + +- **No order** among dynamic children, at start or shutdown, and none in + `which_children`. +- **No queueing** at `max_children`: a full supervisor rejects. Waiting for + a slot is rank 17's admission. +- **No exactly-once start under cancellation:** a dropped `start_child` + future may still produce a supervised child. +- **No state or queue carried across a restart** (V4); nothing replays + messages sent to an old generation. +- **No transparent reference:** `Started::actor` is the first generation + only. +- **No preemption:** a child stuck in a handler or `stopped()` is killed + cooperatively; a child whose cleanup never ends fails the supervisor and + keeps the shutdown report incomplete (S3), as today. +- **No fairness between `start_child` callers** beyond the atomic admission + step; under contention for the last slot any one may win. +- **`which_children`/`count_children` are snapshots**; they can be stale by + the time they are read (OTP's "likely to be very accurate"). +- **No bounded memory per child** (rank 18a) or per mailbox (rank 17). + +## Semantics contract: what this adds and touches + +New guarantees (section 6 of [semantics.md](semantics.md), ids proposed): + +| Id | Guarantee | Checked by (planned) | +|---|---|---| +| V10 | **Dynamic admission.** `start_child` is admitted atomically with the gate and the count; it never returns `Ok` after shutdown began; a full supervisor rejects with `MaxChildren`; a first-start failure returns `FailedToStart` after cleanup, uncharged and unretried. | `DynamicSupervisor!NoStartAfterShutdown`, `!MaxChildrenBound`, `!OkMeansReady`, `!StartAnswered` (liveness); tests | +| V11 | **Cap counts restarting children.** A child holds its slot from admission until removal, through cleanup, backoff and replacement. | `DynamicSupervisor!MaxChildrenBound` | +| V12 | **Terminated means removed.** After `terminate_child` a child is never restarted, whatever its type; a pending restart is cancelled and refunded. | `DynamicSupervisor!TerminatedNeverRestarted`, `!TerminateCompletes` | +| V13 | **Concurrent shutdown.** Every live child gets its stop signal at once; the supervisor is done only after every child finished. | `DynamicSupervisor!ConcurrentShutdown`, `!NoChildOutlivesSupervisor`, `!ClosedLeadsToDone` | + +Touched: + +- **V1** (restart table): applies per child; the Temporary rule also means no + retained arguments. `DynamicSupervisor!TemporaryNeverRestarted`. +- **V3, V4** (one live generation, publication): per slot. + `DynamicSupervisor!AtMostOneGeneration`, `!PublishedIsLive`. +- **V5** (budget): supervisor-wide, start failures excluded; exhaustion + stops all children concurrently instead of in reverse order. + `DynamicSupervisor!BudgetBounded`, `!ExhaustionCloses`. +- **V8** (escalation): a dynamic supervisor is a nestable subtree. +- **V9** (shutdown): concurrent instead of reverse declaration order for + dynamic children; "done" unchanged. `!NoGenerationAfterClose`. +- **S1, S2** (admission, ordered shutdown): the dynamic supervisor is a + shutdown unit; its own admission closes with the tree. +- **§10** gains the non-guarantees above; **§11** gains rows 2, 14, 15, 20 + of the alignment table. + +No guarantee in sections 1-5 changes. + +## Interaction with rank 12 (scoped registries) + +**Decision: registry-owned lookup is deferred to rank 12.** Rank 12a ships +`start_child` returning a handle and nothing keyed. Reasons: + +- The two are independent: a dynamic supervisor is useful without keys (a + pool of workers, one actor per accepted socket counted by `max_children`), + and a keyed registry is useful without supervision (rank 12's own client + evidence is a peer map whose entries need generation guards). +- What a key means on conflict is domain policy. ethrex keeps the first + connection and rejects a duplicate ("Peer already connected, don't + replace it"); another user may want replace-and-stop-old. That belongs in + the registry's design, not in the supervisor's. +- Folding keys into 12a would make the dynamic supervisor depend on rank 12's + scoping (system-scoped instead of the process-global registry) before it + is designed. + +Until rank 12, the pattern is the application map plus a monitor: + +```rust +// In the owner (a peer table actor): +let started = peers.start_child(args).await?; +let mref = ctx.monitor(&started.actor.child_handle()); +self.by_key.insert(key, Entry { child: started.child, monitor: mref }); +// Handler: remove the entry only if it still holds that monitor. +``` + +The `Down`'s `MonitorRef` (or the `ActorId`, never reused, M4) is the +generation guard that ethrex's `ConnectionId` hand-builds, and the `Down` is +delivered exactly once whatever ends the child (X1), unlike a message sent +from `stopped()`. + +**How they compose once rank 12 lands:** + +- A keyed start is `start_child` plus a generation-owned registration made + by the child (in `started()`, as `ctx.register_name` already does, S5) or + by the supervisor on readiness. The registration is released when that + generation's cleanup finishes, so an old cleanup can never remove a + replacement (S5, the ethrex #7435 bug). +- For restartable children, each generation re-registers under the same key + with a new generation; lookups between the crash and the replacement's + readiness find nothing (instance-bound references, V4). +- Duplicate policy (reject or replace) is the registry's; `max_children` and + the budget stay the supervisor's. +- The rank 12 model (`an old entry's cleanup never removes its replacement`) + composes with `DynamicSupervisor.tla` through the slot's generations. + +## What the model establishes + +[`DynamicSupervisor.tla`](../../verification/tla/DynamicSupervisor.tla) +models one dynamic supervisor and N `start_child` calls, each made at most +once at any moment, so starts race crashes, restarts, `terminate_child`, +`max_children`, budget exhaustion and shutdown. Restart types are per slot so +one instance covers all three. Time is abstract: the window is a counter +released at most `MaxExpire` times, which also bounds the generations (no +state constraint); the kill escalation is the nondeterministic reason +(`Shutdown` or `Killed`) of a stopped child; a cleanup can get stuck +(`CleanupStuck`, not fair), which fails the supervisor. + +Invariants: + +| Invariant | Meaning | +|---|---| +| `AtMostOneGeneration` | a slot never has two live generations | +| `PublishedIsLive` | `current()` returns only the live, latest generation | +| `NoChildOutlivesSupervisor` | done means every child finished and was removed | +| `MaxChildrenBound` | children held (restarting and cleaning up included) never exceed `max_children` | +| `TemporaryNeverRestarted` | a Temporary child has one generation | +| `TerminatedNeverRestarted` | no generation starts after `terminate_child` | +| `NoStartAfterShutdown` | no `start_child` returns `Ok` after shutdown began | +| `NoGenerationAfterClose` | no generation (start or restart) begins after admission closed | +| `ExhaustionCloses` | budget exhaustion fails and closes the supervisor | +| `BudgetBounded` | grants in the window stay within `max_restarts` | +| `ConcurrentShutdown` | once stopping, every running child has its stop signal | +| `OkMeansReady` | `Ok` only for a child that became ready | + +Action property `NeverReopens`. Liveness (fair construction, stop, cleanup +and supervisor steps; crashes and stuck cleanup not fair): +`ClosedLeadsToDone`, `StartAnswered`, `TerminateCompletes`. + +TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): + +| Configuration | Instance | Distinct states | Depth | Time | +|---|---|---|---|---| +| `MCDynamicSupervisor` (every PR; safety, action property, liveness) | 3 slots (Permanent, Transient, Temporary), `max_children` 2, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Panic` | RESULTS_FAST | | | +| `MCDynamicSupervisorDeep` (weekly; safety, action property) | 4 slots (P, T, Temp, P), `max_children` 3, budget 2, two releases, every exit reason and startup failure | RESULTS_DEEP | | | + +Negative controls (`negative_checks.py`; each checks `TypeOK` and only the +named property, on `MCDynamicSupervisor`): + +| `Fault` | Mutation | Fails | Result | +|---|---|---|---| +| `IgnoreMax` | admission ignores the cap | `MaxChildrenBound` | counterexample | +| `UncountedRestart` | a crashed child frees its slot until it restarts (counts only starting and running children) | `MaxChildrenBound` | counterexample | +| `RestartTemporary` | Temporary children follow the Permanent rule | `TemporaryNeverRestarted` | counterexample | +| `RestartTerminated` | `terminate_child` does not cancel a restart | `TerminatedNeverRestarted` | counterexample | +| `AcceptDuringShutdown` | `start_child` admitted and committed after close | `NoStartAfterShutdown` | counterexample | +| `RestartAfterClose` | a pending restart proceeds after close | `NoGenerationAfterClose` | counterexample | +| `NoEscalate` | budget exhaustion drops only that child | `ExhaustionCloses` | counterexample | +| `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | counterexample | +| `Sequential` | children stopped one at a time (static order) | `ConcurrentShutdown` | counterexample | +| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` (liveness) | counterexample | +| `StuckBackoff` | shutdown does not cancel a pending restart | `ClosedLeadsToDone` (liveness) | counterexample | + +All eleven produce their counterexample in under 5 s each. + +Not modelled: handler bodies, mailboxes and requests (the child's own death +is `Lifecycle.tla`'s; replies are rank 11's models); real time (backoff +delays, the window's clock); group capacity and leases; nesting (a failed +dynamic supervisor is one abnormal child of `NestedSupervision.tla`'s +parent). Trace validation of real runs against this model, as +`TraceSupervision.tla` does for OneForOne, is part of the implementation +(verification rule). Apalache inductive invariants are follow-up work: the +model needs type annotations and an `IndInv` with helper lemmas like +`SupervisionInd.tla`. + +## Alternatives considered + +1. **Elixir-style heterogeneous children as the primary API** + (`start_child(ChildSpec)` on an untyped supervisor). Rejected for v1: + every call site would carry its own spec and restart type, which the one + client does not need, and a typed template gives `ActorRef` directly. + It can be added later as a separate untyped handle without changing this + contract (the model already allows per-slot restart types). Open + question 2. +2. **Serialise starts like OTP.** Rejected (open question 1): one slow + handshake in `started()` would block every other start and every restart + decision, and static supervisors already restart concurrently. The OTP + idiom for this is `handle_continue`, which Spawned users get by keeping + `started()` short and doing the handshake in a handler. +3. **Make `max_children` wait instead of reject.** Rejected: Elixir rejects, + and waiting is rank 17's admission with its own deadlines and credits. +4. **Free the slot at the exit signal, as OTP removes the child at + `'EXIT'`.** Rejected: owned cleanup (helpers, destructors) still holds + resources, and the cap is meant to bound them (B). +5. **Stop children in reverse start order, as the static supervisor.** + Rejected: OTP does not order them, and N sequential stops multiply the + shutdown time (control `Sequential`). +6. **Terminate by slot only (no `ActorId` variant).** Rejected: OTP and + Elixir terminate by pid, and a stale id must not hit a newer generation + silently; the slot variant (`DynamicChild::terminate`) is offered too. +7. **Return only the `ActorRef` from `start_child`.** Rejected: restartable + children need a way to find the next generation without a registry + (rank 12); the `DynamicChild` handle is that way, as `SupervisedChild` is + for static children. +8. **Put keyed registration in 12a.** Rejected; see + [rank 12](#interaction-with-rank-12-scoped-registries). +9. **Add `terminate_child` to static supervisors in the same change.** The + ROADMAP research note asks for it (today an individually stopped + Permanent child restarts). It is the same mechanism (mark removed, parent + exit, kill, remove) but changes the static API and `Supervision.tla`; + proposed as a follow-up, open question 6. + +## Open questions + +1. **Concurrent construction** (alignment row 2). Keep concurrent starts + (proposed), or serialise factory plus `started()` per supervisor as OTP + does? +2. **Heterogeneous children.** Is the typed template enough for v1, or + should an untyped `start_child(ChildSpec)` ship with it? +3. **Default budget and cap.** Keep the static defaults (5 per 30 s, + `max_children = None`), or use Elixir's 3 per 5 s? Should the dynamic + supervisor default to a finite `max_children` (B), given that every + other Spawned bound is opt-in today? +4. **Group leases across restarts.** Is holding a thread-group lease from + admission to removal acceptable, or should `max_children` simply be + required to fit the group's capacity? +5. **Exit reason of an exhausted supervisor.** OTP exits `shutdown` (so a + Transient parent does not restart it); Spawned's nested adapter exits + with a panic reason. Align both static and dynamic supervisors with OTP + (a separate change), or keep it? +6. **`terminate_child` on static supervisors.** Add it in the same release + (ROADMAP research note), or later? From 4c5e297b294b7292c706dea1df1c701e669c0e5c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:48:21 -0300 Subject: [PATCH 06/52] docs: dynamic supervisor client mapping (ethrex, ethlambda) --- .../dynamic-supervisor-client-mapping.md | 297 ++++++++++++++++++ docs/design/dynamic-supervisor.md | 4 + 2 files changed, 301 insertions(+) create mode 100644 docs/design/dynamic-supervisor-client-mapping.md diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md new file mode 100644 index 00000000..1ea6d8a1 --- /dev/null +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -0,0 +1,297 @@ +# Dynamic supervisor: client mapping + +Companion to the [dynamic supervisor design](dynamic-supervisor.md) (ROADMAP +rank 12a). Maps the one runtime-spawned actor population in the clients, +ethrex's per-peer connection actors, onto the proposed API, and records that +ethlambda has none. Sketches are illustrative, not patches; nothing in either +client was changed. + +Revisions read (source reading only, nothing built or run): + +| Repository | Ref | Commit | spawned | +|---|---|---|---| +| ethrex | `main` | [`fd5c57dd`](https://github.com/lambdaclass/ethrex/tree/fd5c57ddada097952660f6b749c739119e4de852) | 0.5.0 | +| ethrex | `spawned-v0.7` | [`0dd4c105`](https://github.com/lambdaclass/ethrex/tree/0dd4c105f65d544b0fb5e92d8634a1699127f826) | 0.7.3 | +| ethrex | PR [#7435](https://github.com/lambdaclass/ethrex/pull/7435) (`ConnectionId` guard) | merge [`fc44a9fb`](https://github.com/lambdaclass/ethrex/commit/fc44a9fb25451a17656e88e8e15b4443f747f661) | | +| ethlambda | `main` | [`c3f5f566`](https://github.com/lambdaclass/ethlambda/tree/c3f5f566cb3a61e3d388c66ce21794c18a6618ce) | 0.5.0 | +| ethlambda | `spawned-v0.7` | [`6a12bc75`](https://github.com/lambdaclass/ethlambda/tree/6a12bc753ae18c9d63f3dbf930edc700c4c4704c) | 0.7.3 | + +`crates/networking/p2p/rlpx/connection/server.rs` and `handshake.rs` are +identical on both ethrex branches; their links below use `spawned-v0.7`. +Short forms: **V** = `https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826`, +**M** = `https://github.com/lambdaclass/ethrex/blob/fd5c57ddada097952660f6b749c739119e4de852`. + +## ethrex today + +### Who starts connections + +One `PeerConnectionServer` actor per TCP connection, started with a bare +`.start()` (unmanaged, S6) and owned by nothing: + +- **Accept:** `PeerConnection::spawn_as_receiver(context, peer_addr, stream, permit)` + ([V server.rs:191-209](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L191-L209)), + called from the accept loop `serve_p2p_requests`, which discards the result + ([V network.rs:253-290](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/network.rs#L253-L290)). +- **Dial:** `PeerConnection::spawn_as_initiator(context, &node)` + ([V server.rs:211-224](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L211-L224)), + called by the `RLPxInitiator` actor + ([V initiator.rs:91-120](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/initiator.rs#L91-L120)), + result also discarded. + +```rust +// server.rs:191-224 (both branches) +pub fn spawn_as_receiver(context: P2PContext, peer_addr: SocketAddr, stream: TcpStream, + admission_permit: tokio::sync::OwnedSemaphorePermit) -> PeerConnection { + let state = ConnectionState::Receiver(Receiver { context, peer_addr, stream: Arc::new(stream) }); + let connection = PeerConnectionServer { state, _admission_permit: Some(admission_permit) }; + Self { handle: connection.start() } +} +pub fn spawn_as_initiator(context: P2PContext, node: &Node) -> PeerConnection { + let state = ConnectionState::Initiator(Initiator { context, node: node.clone() }); + // Outbound dials are not admission-capped (we initiate them); inbound is the attack surface. + let connection = PeerConnectionServer { state, _admission_permit: None }; + Self { handle: connection.start() } +} +``` + +On `spawned-v0.7` the singletons (peer table, initiator, broadcaster, +subscription manager) moved onto an `ActorSystem` group and discovery under a +OneForOne supervisor, but connections did not; the comment says why: +"Per-connection actors (`PeerConnectionServer`) are not admitted: they are +created and dropped per peer, and the system only owns a fixed set" +([V initializers.rs:1041-1042](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/cmd/ethrex/initializers.rs#L1041-L1042)). +So connections run on whatever runtime is current at the call site, and +system shutdown does not stop them. + +### How they are tracked + +After the handshake the connection registers itself with the peer table, +`IndexMap` +([V peer_table.rs:546](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/peer_table.rs#L546)), +passing its own `ActorRef` and a `ConnectionId` +([server.rs:1074-1101](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L1074-L1101)). +The first connection to a node wins; a duplicate gets `false`, sends +Disconnect(AlreadyConnected) and stops. + +`ConnectionId` is a process-global counter added by #7435 after a duplicate +connection's cleanup removed the live entry +([server.rs:176-188](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L176-L188)): + +```rust +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ConnectionId(u64); +impl ConnectionId { + pub(crate) fn next() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + Self(NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) + } +} +``` + +### How they are cleaned up + +Nothing monitors or links a connection. It reports its own death from its +`stopped()` hook +([server.rs:623-671](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L623-L671)): + +```rust +// The peer table removes the node only if it lists this connection for it, so +// a duplicate connection to an already-connected node leaves the live one +// listed. Sent even when registration failed: a registration the peer table +// applied after this connection stopped waiting for it is undone here. +if let Err(e) = established_state.peer_table.remove_peer( + established_state.node.node_id(), + established_state.connection_id, +) { debug!("Failed to remove peer from table: {e}"); } +if established_state.registered_in_peer_table + && let Err(e) = established_state.tx_broadcaster.remove_peer(established_state.node.node_id()) +{ debug!("Failed to remove peer from tx broadcaster: {e}"); } +established_state.teardown().await; +``` + +and the table checks the guard +([V peer_table.rs:693-711](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/peer_table.rs#L693-L711)): + +```rust +if let Some(peer) = self.peers.get(&msg.node_id) { + if peer.connection_id != Some(msg.connection_id) { return; } + self.peers.swap_remove(&msg.node_id); +} +self.sessions.remove(&msg.node_id); +``` + +Gaps in this scheme: + +- If `stopped()` does not run, the entry is never removed. Today that is a + panic in `started()`; under v0.8 it is also a kill, an untrapped exit + signal and the supervisor's or system's shutdown of a non-trapping actor + (exit-signals design, decision 2). +- A registration that timed out on the connection side but was applied by + the table is undone only because `stopped()` sends `remove_peer` + unconditionally; the comment above exists for this case. +- At node shutdown nothing stops connections; their `remove_peer` sends fail + silently once the table is gone ([V shutdown.rs:81-153](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/cmd/ethrex/shutdown.rs#L81-L153) + stops the system's actors only). + +### Limits + +- Inbound: a semaphore of `MAX_INBOUND_CONNECTIONS = 150` + ([V network.rs:68](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/network.rs#L68)), + `try_acquire_owned()` in the accept loop; when full the stream is dropped + before any actor exists. The permit lives in actor state + (`_admission_permit`) and is released when the state drops. +- Outbound dials are not capped by it; `target_peers` (100) gates dialing and + registration (`TooManyPeers`). + +### Restarts + +None. A dead connection is removed; the node may be re-dialed later through +discovery. In supervision terms every connection is **Temporary**. + +## Mapping + +| ethrex today | With the dynamic supervisor | +|---|---| +| `.start()`, unmanaged, on the caller's runtime | `start_child(args)` on a `DynamicSupervisor`, placed on the `p2p` group | +| no owner; shutdown does not stop connections | the supervisor is a shutdown unit of the existing `ActorSystem`; admitted after the peer table, so in reverse admission order connections stop before the table, all at once, each killed after its `shutdown_timeout` | +| `Semaphore(150)` plus `_admission_permit` in actor state | `max_children = Some(150)` on an inbound supervisor; `Err(MaxChildren)` instead of `try_acquire_owned()` failing. The slot is freed after the actor's cleanup, as the permit is freed when the state drops | +| outbound uncapped | a second supervisor for dials without `max_children` (or one supervisor if a shared cap is wanted; that would change behaviour) | +| `ConnectionId` (process-global counter) | the connection's `ActorId` (never reused, M4) or the table's `MonitorRef` | +| `stopped()` sends `remove_peer(node_id, connection_id)` | the peer table monitors the connection when it registers it; `Handler` removes the entry whose monitor matches. `Down` arrives exactly once whatever ends the actor (X1), including kills and skipped `stopped()` | +| `handle_remove_peer` guard | the monitor match in `Handler` | +| timed-out registration undone by an unconditional `remove_peer` | the table's monitor fires for that registration too; a monitor installed after the connection died gets `Down { NoProc }` at once (E4) | +| connections never restarted | `DynamicChildSpec::temporary(..)`: no restarts, no budget use, args (the `TcpStream`, the `Node`) moved without `Clone` | +| first-wins on duplicates (`new_connected_peer` returns `false`) | unchanged: domain policy in the table (rank 12 may later own it) | +| `teardown()` mempool cleanup, `tx_broadcaster.remove_peer` | unchanged code, but see the note on `stopped()` below | + +### After (sketch) + +```rust +// Wiring (initializers.rs), after the peer table so connections stop first. +let inbound = system + .dynamic_supervisor( + DynamicSupervisorConfig { max_children: Some(MAX_INBOUND_CONNECTIONS), ..Default::default() }, + DynamicChildSpec::temporary("rlpx-inbound", &p2p_group, |_ctx, args: ConnArgs| async move { + Ok::<_, Infallible>(PeerConnectionServer::new(args)) + }) + .shutdown_timeout(Duration::from_secs(2)), + )? + .start() + .await?; +let outbound = system + .dynamic_supervisor(DynamicSupervisorConfig::default(), /* same template */)? + .start() + .await?; + +// Accept loop (network.rs): the semaphore and the permit plumbing go away. +match context.inbound.start_child(ConnArgs::receiver(context.clone(), peer_addr, stream)).await { + Ok(_) => {} + Err(StartChildError::MaxChildren) => debug!(peer = %peer_addr, "Inbound connection cap reached"), + Err(e) => debug!(peer = %peer_addr, "Inbound connection not started: {e}"), +} + +// Peer table (peer_table.rs): registration installs the monitor... +async fn handle_new_connected_peer(&mut self, msg: NewConnectedPeer, ctx: &Context) -> bool { + let id = msg.node.node_id(); + if self.peers.contains_key(&id) { return false; } + let monitor = ctx.monitor(&msg.connection.handle.child_handle()); + self.by_monitor.insert(monitor, id); + self.peers.insert(id, PeerData::new(/* … */).with_monitor(monitor)); + true +} +// ...and removal is the Down, matched by monitor, not reported by the peer. +impl Handler for PeerTableServer { + async fn handle(&mut self, down: Down, _ctx: &Context) { + if let Some(id) = self.by_monitor.remove(&down.monitor_ref) { + self.peers.swap_remove(&id); + self.sessions.remove(&id); + } + } +} +``` + +`start_child` resolves when the connection's `started()` returns. ethrex's +`started()` runs the whole RLPx handshake today, under its own +`HANDSHAKE_TIMEOUT`, and calls `ctx.stop()` when it fails +([server.rs:537-620](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L537-L620)). +Under the dynamic supervisor that is a first-start failure: `start_child` +returns `FailedToStart`, the slot is freed, nothing is restarted or charged, +which is the behaviour ethrex wants. Two consequences: the accept loop would +wait for the handshake, and `startup_timeout` (default 5 s) must exceed +`HANDSHAKE_TIMEOUT` or the supervisor cuts handshakes short. Either the +accept loop spawns the `start_child` future (it does not need the result), +or the handshake moves from `started()` into a first self-sent message, the +Spawned counterpart of OTP's `handle_continue`. The second is better: +`startup_timeout` then bounds only construction, and a slow peer never holds +a start; a failed handshake then becomes an ordinary Temporary exit. + +### What disappears + +About 45-60 of the ~135 hand-written lines (counted on `main`): + +| Code | Lines | Fate | +|---|---|---| +| `ConnectionId` type and `next()` ([server.rs:176-188](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L176-L188)) | 13 | removed | +| `connection_id` in `Established`, `PeerData` and the handshake ([server.rs:432-437](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L432-L437), [handshake.rs:177](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/handshake.rs#L177), [M peer_table.rs:324-326](https://github.com/lambdaclass/ethrex/blob/fd5c57ddada097952660f6b749c739119e4de852/crates/networking/p2p/peer_table.rs#L324-L326)) | ~8 | removed (replaced by the monitor) | +| `remove_peer` protocol method and `handle_remove_peer` with its guard ([V peer_table.rs:693-711](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/peer_table.rs#L693-L711)) | ~22 | replaced by a ~10-line `Handler` | +| `remove_peer` send in `stopped()` | ~6 | removed | +| inbound semaphore, `try_acquire_owned`, `_admission_permit` field and parameter | ~15 | replaced by `max_children` | +| "connections are not admitted" comment and unmanaged `.start()` | | connections owned by the system | + +What stays: first-wins registration (domain policy), `teardown()`'s mempool +cleanup, the tx-broadcaster index, `target_peers`. #7435's integration test +(171 lines) stays as a regression test, now exercising the monitor match. + +### Bugs and gaps it closes + +- **Stale entries when `stopped()` does not run** (panic in `started()` + today; kills and supervisor shutdown under v0.8): the `Down` still comes. +- **The #7435 class** (an old connection's cleanup removing a new entry): + impossible by construction, because removal is keyed by the monitor of the + registered connection, not by node id. +- **Unowned connections at shutdown**: they are stopped, within the system + deadline, before the peer table, instead of dying with the process. +- **Placement**: connections run on the `p2p` group instead of whichever + runtime started them. + +### Things to watch in a migration + +- **`stopped()` is skipped on supervisor shutdown under v0.8** for a + non-trapping actor. `teardown()` and `tx_broadcaster.remove_peer` live in + `stopped()`; at node shutdown that is harmless (everything is going away), + but on a `kill` during normal operation (a connection stuck past its + `shutdown_timeout`) the broadcaster index would leak. Fix: the broadcaster + also monitors connections, or the peer table forwards removal from its + `Down` handler. Same reasoning as the exit-signals migration note. +- **Caps change meaning slightly**: the semaphore counted live actor states; + `max_children` counts slots until owned cleanup ends (helpers included). + For these actors (no helpers) that is the same. +- **Rejected arguments are dropped**: on `MaxChildren` the `TcpStream` inside + `ConnArgs` is dropped, closing the socket without an RLPx Disconnect, + exactly as today. Returning the arguments with the error is open question + 7 of the design. +- **Budget**: Temporary children never consume it, so the supervisor fails + only on a stuck cleanup or a closed group. A connection whose cleanup + hangs past twice its `shutdown_timeout` would fail the whole connection + supervisor; set `shutdown_timeout` with that in mind. + +## ethlambda + +No dynamic per-X actors on either branch. + +- `main`: two singletons, `BlockChainServer` (`start_with_backend(Backend::Thread)`, + [crates/blockchain/src/lib.rs:434](https://github.com/lambdaclass/ethlambda/blob/c3f5f566cb3a61e3d388c66ce21794c18a6618ce/crates/blockchain/src/lib.rs#L434)) + and `P2PServer` (`start()`, + [crates/net/p2p/src/lib.rs:1271](https://github.com/lambdaclass/ethlambda/blob/c3f5f566cb3a61e3d388c66ce21794c18a6618ce/crates/net/p2p/src/lib.rs#L1271)), + plus one `PeerTableServer` and one `DiscoveryServer` from ethrex. Peers + are libp2p connections handled inside the swarm, not actors. No maps of + `ActorRef`. +- `spawned-v0.7`: the same singletons placed with `start_on(group)` + ([lib.rs:1268](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/crates/net/p2p/src/lib.rs#L1268), + [blockchain lib.rs:449](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/crates/blockchain/src/lib.rs#L449)) + and the HTTP server under a static supervisor + ([main.rs:977-989](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/bin/ethlambda/src/main.rs#L977-L989)). + +Per-request and per-check concurrency is plain `tokio::spawn`/`spawn_blocking` +(gossip checks, aggregation), which belongs to ranks 11b and 15, not to a +dynamic supervisor. diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 0eeaf442..3d4ffbd3 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -738,3 +738,7 @@ model needs type annotations and an `IndInv` with helper lemmas like (a separate change), or keep it? 6. **`terminate_child` on static supervisors.** Add it in the same release (ROADMAP research note), or later? +7. **Rejected arguments.** `MaxChildren`, `Stopped` and `System` errors drop + the arguments (ethrex's `TcpStream` closes without an RLPx Disconnect, as + today). Should these errors hand the arguments back (`Err((error, args))`), + or should a `try_reserve()` permit precede `start_child`? From 55770028806c0f4f9345107a2083e786f74e0f2e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 10:49:06 -0300 Subject: [PATCH 07/52] docs: wire DynamicSupervisor into README, TESTING, ROADMAP, semantics and the weekly deep matrix --- .github/workflows/verification.yml | 1 + CHANGELOG.md | 9 ++-- ROADMAP.md | 2 +- docs/TESTING.md | 13 +++++- docs/design/README.md | 7 +++ .../dynamic-supervisor-client-mapping.md | 2 +- docs/design/semantics.md | 5 +++ verification/tla/README.md | 45 +++++++++++++++++++ 8 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index f2d1512d..e64516c8 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -112,6 +112,7 @@ jobs: - MCLifecycleWideDeep - MCLifecycleLiveDeep - MCLifecycleDeep + - MCDynamicSupervisorDeep steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v16 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dae3a4d..947258ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,6 @@ ## Unreleased -- Design (not implementation) for rank 12a, the dynamic supervisor - (`DynamicSupervisor` / `simple_one_for_one`): - `docs/design/dynamic-supervisor.md`, the ethrex/ethlambda client mapping - and the `DynamicSupervisor` TLA+ model with negative controls. - Lockfile: `rustix` 1.1.2 -> 1.1.6 for RUSTSEC-2026-0486 (pulled in by the examples' `native-tls` dependency; spawned's crates are unaffected). - CI: a `sim` job runs the simulated test suites (`--cfg spawned_sim`) @@ -53,6 +49,11 @@ it is computed from, so unrelated models no longer invalidate it, and is regenerated (same keys and state counts). `check.sh` gains `--only` and `--exclude`. +- Design (not implementation) for rank 12a, the dynamic supervisor + (`DynamicSupervisor` / `simple_one_for_one`, PR #245): + `docs/design/dynamic-supervisor.md`, the ethrex/ethlambda client mapping + and the `DynamicSupervisor` TLA+ model (per-PR and weekly configurations) + with 11 named negative controls. ## 0.7.3 — 2026-11-10 diff --git a/ROADMAP.md b/ROADMAP.md index 94d6e6d4..35071e66 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -555,7 +555,7 @@ the first start (semantics §11; found reviewing #233). Behaviour change: extend | 11a | Design inputs to rank 11: promise pipelining and transactional turns | Decide within the rank 11 design, not as a separate API: (1) whether a pipelined send ("on reply to R, send M to the target R names") is expressible without an extra handler turn, with M failing like R rather than being dropped (E/Cap'n Proto/Goblins pipelining; the main latency tool if rank 44 ever ships); (2) whether deferred replies and completions are sent at the end of a turn or immediately, and an opt-in Goblins-style transactional turn that holds a handler's sends until it returns and discards them on panic (state rollback only for snapshot-able state; external effects are never rolled back); (3) whether per-turn snapshots can serve ranks 21 and 30 too. The rank 11 TLA+ model must cover a reply sent by a turn that then panics. See [request semantics references](docs/research/request-semantics-references.md); STM's "effects only on commit" is the same idea restricted to one actor, while its `retry` is selective receive and stays out ([STM note](docs/research/concurrency-models.md#5-software-transactional-memory)). | | 11b | Managed compute pool (CPU offload with pipe-to-self) | **Client evidence:** ethrex's four hand-built rayon pools, 50 `spawn_blocking` calls and the L2 import deadlock at 0% CPU (535647e45); synchronous cell serving that occupied every runtime worker (d958dcba4); ethlambda's hand-owned aggregation worker and semaphore-bounded, `catch_unwind`-guarded gossip checks ([evidence](docs/research/concurrency-models.md#who-uses-it)). One bounded CPU pool per `ActorSystem` or group, never on Tokio workers; `ctx.compute(job, map)` (name illustrative) delivers the result to the submitting actor as a message; per-actor and per-pool admission caps that reject at once or follow rank 17; cancellation tied to owner stop, restart and scope close (rank 15a) with generation fencing; a panic becomes an error message, exactly once; metrics for queue depth, wait, run time and rejections. Document that jobs must not wait on their own pool and gang jobs need a dedicated pool. Rayon behind a feature first; a fixed native pool as the alternative. Built on rank 11's completion contract; shares admission design with rank 34. Prior art: Akka dispatchers, Orleans `Task.Run`, Ray tasks, BEAM dirty schedulers ([concurrency models](docs/research/concurrency-models.md#1-cpu-parallel-compute-pools-integrated-with-actors)). | | 12 | Scoped, lifecycle-owned registries | **Client evidence:** ethrex's per-peer connection map and its `ConnectionId` guard (#7435). Keyed registration that owns the actor's lifetime, with generations so an old cleanup cannot remove its replacement. Also scope the registry to an `ActorSystem` instead of the process-global static, so parallel tests and multi-node simulations do not share names. Prerequisite for groups/pub-sub. Orleans keeps the same one-live-instance-per-key invariant ([virtual actors](docs/research/virtual-actors.md#what-it-means-for-spawned)). | -| 12a | Dynamic supervisor (`DynamicSupervisor` / `simple_one_for_one`) | **Client evidence:** ethrex starts one connection actor per peer at runtime and cleans it up by hand (see [per-key actor maps](#client-search-results)); today's `Supervisor` only owns the fixed child list declared at build time. Proposed contract: children are started at runtime from one `ChildSpec` template with per-call arguments, sharing its restart mode (Elixir-style heterogeneous specs only if a user needs them). The restart budget is supervisor-wide, as in OTP: exhausting it fails the supervisor and escalates to its parent, so a flapping child cannot restart forever. `max_children` counts restarting children, so a new start cannot take a slot mid-restart, and rejects with a typed error rather than queueing. `start_child` after shutdown begins is rejected, never dropped or leaked. Explicitly stopping a child does not restart it. Shutdown stops all live children concurrently under one supervisor deadline (OTP does not order dynamic children; sequential stops would multiply per-child timeouts). Usable inside a nested tree. Independent of rank 12 (`start_child` returns a handle); when combined, a restarted child re-registers under a new generation so an old cleanup cannot remove it. Planned after v0.7, not part of it; extend the TLA+ model (start/stop racing restart and shutdown, cap including restarting children) before release, as for OneForAll/RestForOne. The other OTP variants stay out: `Task.Supervisor` folds into ranks 11/15, `PartitionSupervisor` is sharding, and significant children/`auto_shutdown` wait for a user. **Research ([supervision models](docs/research/supervision-models.md)):** counting restarting children against `max_children` goes beyond Elixir; add `terminate_child` (stop without restart; today an individually stopped Permanent child is restarted) and consider it on static supervisors too. Significant children/`auto_shutdown` would be a trigger for draining (rank 13). Document why Spawned's default budget (5 restarts in 30 s) is looser than OTP's (1 in 5 s): per-child backoff already spreads restarts. Non-goals: an Akka-style Resume directive (keep state after a failure) and buffering messages during backoff, because references are instance-bound. | +| 12a | Dynamic supervisor (`DynamicSupervisor` / `simple_one_for_one`) | **Design in review** (PR #245, not implemented): [design contract](docs/design/dynamic-supervisor.md), [client mapping](docs/design/dynamic-supervisor-client-mapping.md), TLA+ model `DynamicSupervisor.tla` (starts racing crashes, restarts, `terminate_child`, `max_children`, budget exhaustion and shutdown) with 11 negative controls. Registry-owned lookup stays in rank 12. **Client evidence:** ethrex starts one connection actor per peer at runtime and cleans it up by hand (see [per-key actor maps](#client-search-results)); today's `Supervisor` only owns the fixed child list declared at build time. Proposed contract: children are started at runtime from one `ChildSpec` template with per-call arguments, sharing its restart mode (Elixir-style heterogeneous specs only if a user needs them). The restart budget is supervisor-wide, as in OTP: exhausting it fails the supervisor and escalates to its parent, so a flapping child cannot restart forever. `max_children` counts restarting children, so a new start cannot take a slot mid-restart, and rejects with a typed error rather than queueing. `start_child` after shutdown begins is rejected, never dropped or leaked. Explicitly stopping a child does not restart it. Shutdown stops all live children concurrently under one supervisor deadline (OTP does not order dynamic children; sequential stops would multiply per-child timeouts). Usable inside a nested tree. Independent of rank 12 (`start_child` returns a handle); when combined, a restarted child re-registers under a new generation so an old cleanup cannot remove it. Planned after v0.7, not part of it; extend the TLA+ model (start/stop racing restart and shutdown, cap including restarting children) before release, as for OneForAll/RestForOne. The other OTP variants stay out: `Task.Supervisor` folds into ranks 11/15, `PartitionSupervisor` is sharding, and significant children/`auto_shutdown` wait for a user. **Research ([supervision models](docs/research/supervision-models.md)):** counting restarting children against `max_children` matches Elixir, which keeps a restarting child in its map (corrected by the design; this said it went beyond Elixir); add `terminate_child` (stop without restart; today an individually stopped Permanent child is restarted) and consider it on static supervisors too. Significant children/`auto_shutdown` would be a trigger for draining (rank 13). Document why Spawned's default budget (5 restarts in 30 s) is looser than OTP's (1 in 5 s): per-child backoff already spreads restarts. Non-goals: an Akka-style Resume directive (keep state after a failure) and buffering messages during backoff, because references are instance-bound. | | 13 | Coordinated shutdown and bounded draining | Ordered stop exists; add explicit draining as a policy: close admission, handle queued work by policy, join helpers and report unfinished work. Standalone scopes currently use reverse admission order; explore declared dependencies with cycle/unknown-edge rejection and model-checked deadline propagation. Keep current stop/discard as the default. Design with rank 15a: draining is closing the actor's scope under its deadline. Keep "stuck cleanup fails the tree" instead of OTP's `brutal_kill`; add only reporting of unfinished work ([supervision models](docs/research/supervision-models.md)). | | 14 | Separate fair control path | Part of supervision/shutdown design; defer a generic public priority API. Non-trapping actors already receive abnormal exits through cancellation; queued `ExitEnvelope` only matters for trapping actors. Specify boundedness and per-sender ordering for startup, monitor and trapped-exit events. Priority cannot preempt a running handler or starve user work. | | 15 | Actor-owned helper completion scopes | **Client evidence:** ethlambda hand-owns its aggregation worker; add cancel-previous, generation fencing and join-with-deadline. Managed helpers require ActorSystem; explore standalone ownership without detached fallback or changing actor-join guarantees implicitly. Gate new helper registration, cancel, then explicitly join; helpers depending on their stopped owner can deadlock. | diff --git a/docs/TESTING.md b/docs/TESTING.md index 4d360749..bb9905e2 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -109,7 +109,7 @@ Status of each layer. "Planned" items are tracked in the | Layer | Tool | Catches | Status | |---|---|---|---| -| Design | TLA+ and TLC | Holes in lifecycle, restart and shutdown rules | `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` models in `verification/tla`, run in CI | +| Design | TLA+ and TLC | Holes in lifecycle, restart and shutdown rules | `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` models in `verification/tla`, run in CI; design models `Lifecycle` (v0.8 exit signals), `DeferredReplies`/`DeferredJobs` (rank 11) and `DynamicSupervisor` (rank 12a), each with named negative controls | | Design, unbounded depth | Apalache inductive invariants | An invariant that a state TLC never reached would break | Safety invariants of all four models proven inductive at fixed sizes, with negative controls; `tla-apalache` CI job | | Design, any size | TLAPS or Veil | An invariant that fails only for larger configurations | Planned ([verification tools](research/verification-tools.md)) | | Spec and code | Trace validation | The implementation doing something the model forbids | Real OneForOne, cohort and nested traces plus invalid-trace and model-mutation controls | @@ -924,6 +924,17 @@ model. Hand-edited traces are rejected at the edited event. Trace validation found one model gap (a running child may exit with `Shutdown` while the supervisor is open, via `ChildHandle::shutdown`); no implementation deviation. +`verification/tla/DynamicSupervisor.tla` models the rank 12a design (not +implemented): `start_child` racing crashes, restarts, `terminate_child`, +`max_children`, budget exhaustion and shutdown. TLC checks twelve safety +invariants (no child outlives the supervisor, the cap counts restarting +children, Temporary and terminated children are never restarted, no start +succeeds after shutdown begins, exhaustion closes the supervisor, shutdown +signals every child at once), `NeverReopens` and three liveness properties; +`MCDynamicSupervisor` runs on every PR and `MCDynamicSupervisorDeep` weekly. +Eleven named negative controls (nine safety, two liveness) each produce a +counterexample. See the [design](design/dynamic-supervisor.md#what-the-model-establishes). + Apalache checks that the safety invariants of `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` are inductive (`Init => IndInv`, `IndInv /\ Next => IndInv'`), so they hold at any depth, for the configured diff --git a/docs/design/README.md b/docs/design/README.md index 0cdf4eea..a3564537 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -19,6 +19,13 @@ Active design contracts. Usage is documented in the [guide](../GUIDE.md). implemented) for exit signals, `Killed`/`NoProc`, supervisor startup and runtime-dropped actors as in OTP (ROADMAP ranks 5e, 5f, 5g), with the base actor lifecycle model `Lifecycle.tla`. +- **[dynamic-supervisor.md](dynamic-supervisor.md)** (rank 12a, design in + review, not implemented): OTP's `simple_one_for_one` / Elixir's + `DynamicSupervisor`: runtime `start_child` from a typed template, + `terminate_child`, `max_children` counting restarting children, a + supervisor-wide budget and concurrent shutdown. Model: + `verification/tla/DynamicSupervisor.tla`. Client mapping (ethrex's + per-peer connections): [dynamic-supervisor-client-mapping.md](dynamic-supervisor-client-mapping.md). Earlier designs (the v0.5 API redesign alternatives, the framework comparison and superseded roadmap plans) are archived in diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index 1ea6d8a1..cfac87c1 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -218,7 +218,7 @@ Under the dynamic supervisor that is a first-start failure: `start_child` returns `FailedToStart`, the slot is freed, nothing is restarted or charged, which is the behaviour ethrex wants. Two consequences: the accept loop would wait for the handshake, and `startup_timeout` (default 5 s) must exceed -`HANDSHAKE_TIMEOUT` or the supervisor cuts handshakes short. Either the +`HANDSHAKE_TIMEOUT` (10 s) or the supervisor cuts handshakes short. Either the accept loop spawns the `start_child` future (it does not need the result), or the handshake moves from `started()` into a first self-sent message, the Spawned counterpart of OTP's `handle_continue`. The second is better: diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 528ea7d4..9075cf8e 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -551,6 +551,11 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table - Rank 11, deferred typed replies, forwarding and pipe-to-self: a reply may leave the handler's turn; adds exactly-one completion on panic and timeout. +- Rank 12a, dynamic supervisor (design in review, + [dynamic-supervisor.md](dynamic-supervisor.md)): adds V10-V13 (dynamic + admission, a cap that counts restarting children, terminated means + removed, concurrent shutdown of dynamic children); touches V1, V3-V5, V8, + V9, S1, S2. - Rank 13, bounded draining: an opt-in stop that handles queued work by policy instead of discarding it (D4 remains the default). - Rank 14, fair control path: lifecycle signals not stuck behind a backlog (X7). diff --git a/verification/tla/README.md b/verification/tla/README.md index 82c47470..d3ef5aff 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,6 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (4 slots, every reason). | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -579,3 +580,47 @@ Measured 2026-11-10 (TLC 2.19, fede): | `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit each; safety) | 5,876,658 | 12,792,039 | 23 | 25 s, 12 workers | All 22 named controls find their counterexample, each in a few seconds. + +## Dynamic supervisor (rank 12a design) + +`DynamicSupervisor.tla` backs the [dynamic supervisor design](../../docs/design/dynamic-supervisor.md), +written before any implementation. Slots 1..N are `start_child` calls, each +made at most once at any moment, so starts race everything else. Restart +types are per slot (one instance covers all three). The rolling window is a +counter released at most `MaxExpire` times, which also bounds the number of +generations, so no state constraint is needed and the liveness properties +are checked on the full graph. + +Invariants: `AtMostOneGeneration`, `PublishedIsLive`, +`NoChildOutlivesSupervisor`, `MaxChildrenBound`, `TemporaryNeverRestarted`, +`TerminatedNeverRestarted`, `NoStartAfterShutdown`, `NoGenerationAfterClose`, +`ExhaustionCloses`, `BudgetBounded`, `ConcurrentShutdown`, `OkMeansReady`. +Action property `NeverReopens`; liveness `ClosedLeadsToDone`, +`StartAnswered`, `TerminateCompletes` (crashes and stuck cleanup are not +fair). + +| Configuration | Mutation | Breaks | +|---|---|---| +| `MCDynamicSupervisor` | `IgnoreMax` | `MaxChildrenBound` | +| `MCDynamicSupervisor` | `UncountedRestart` (a crashed child frees its slot until it restarts) | `MaxChildrenBound` | +| `MCDynamicSupervisor` | `RestartTemporary` | `TemporaryNeverRestarted` | +| `MCDynamicSupervisor` | `RestartTerminated` | `TerminatedNeverRestarted` | +| `MCDynamicSupervisor` | `AcceptDuringShutdown` | `NoStartAfterShutdown` | +| `MCDynamicSupervisor` | `RestartAfterClose` | `NoGenerationAfterClose` | +| `MCDynamicSupervisor` | `NoEscalate` (exhaustion drops only that child) | `ExhaustionCloses` | +| `MCDynamicSupervisor` | `EarlyDone` | `NoChildOutlivesSupervisor` | +| `MCDynamicSupervisor` | `Sequential` (static reverse-order stops) | `ConcurrentShutdown` | +| `MCDynamicSupervisor` (liveness) | `NoAbort` (a start racing shutdown is not cancelled) | `StartAnswered` | +| `MCDynamicSupervisor` (liveness) | `StuckBackoff` (shutdown does not cancel a pending restart) | `ClosedLeadsToDone` | + +Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`): + +| Configuration | Distinct states | Generated | Depth | Time | +|---|---|---|---|---| +RESULTS_ROWS + +All eleven controls find their counterexample in under 5 s each. Not yet: +Apalache inductive invariants (needs type annotations and an `IndInv`, as +`SupervisionInd.tla`) and trace validation, which come with the +implementation. + From 467d150fc815cc496a300586c1dc187cecb35a86 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:02:57 -0300 Subject: [PATCH 08/52] verify: smaller weekly DynamicSupervisor instance (one reason per class) --- verification/tla/MCDynamicSupervisorDeep.cfg | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 825f6fa4..76cc378e 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,6 +1,10 @@ \* Deep instance (weekly, --deep): four start_child calls against -\* max_children 3, budget 2 with two window releases, every exit reason -\* and startup failure; safety and the action property. +\* max_children 3, budget 2 with two window releases; safety and the action +\* property. Reasons are one per class: the model's decisions depend only on +\* Normal-class (Normal, Shutdown) versus abnormal (Panic, Kill, Killed), and +\* every startup failure takes the same path, so more reasons only multiply +\* states (every reason for every operation made this instance exceed 12 GB +\* of TLC state on fede without finishing). SPECIFICATION Spec CONSTANTS N = 4 @@ -8,8 +12,8 @@ CONSTANTS MaxChildren = 3 MaxRestarts = 2 MaxExpire = 2 - SelfExits = {"Normal", "Shutdown", "Panic", "Kill"} - StartFails = {"none", "timeout", "Normal", "Panic"} + SelfExits = {"Normal", "Panic"} + StartFails = {"none"} Fault = "None" INVARIANT TypeOK INVARIANT AtMostOneGeneration From 37a050fa189cc9853cc782f24bd1854e1c2b4970 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:33:44 -0300 Subject: [PATCH 09/52] verify: weekly DynamicSupervisor instance with deeper budget on three slots --- docs/design/dynamic-supervisor.md | 2 +- verification/tla/MCDynamicSupervisorDeep.cfg | 23 ++++++++++---------- verification/tla/README.md | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 3d4ffbd3..8a254bd0 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -650,7 +650,7 @@ TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| | `MCDynamicSupervisor` (every PR; safety, action property, liveness) | 3 slots (Permanent, Transient, Temporary), `max_children` 2, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Panic` | RESULTS_FAST | | | -| `MCDynamicSupervisorDeep` (weekly; safety, action property) | 4 slots (P, T, Temp, P), `max_children` 3, budget 2, two releases, every exit reason and startup failure | RESULTS_DEEP | | | +| `MCDynamicSupervisorDeep` (weekly; safety, action property) | 4 slots (P, T, Temp, P), `max_children` 3, budget 2, two releases, one reason per class (the decisions depend only on normal versus abnormal; with every reason the instance passed 12 GB of TLC state without finishing) | RESULTS_DEEP | | | Negative controls (`negative_checks.py`; each checks `TypeOK` and only the named property, on `MCDynamicSupervisor`): diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 76cc378e..44162816 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,19 +1,20 @@ -\* Deep instance (weekly, --deep): four start_child calls against -\* max_children 3, budget 2 with two window releases; safety and the action -\* property. Reasons are one per class: the model's decisions depend only on -\* Normal-class (Normal, Shutdown) versus abnormal (Panic, Kill, Killed), and -\* every startup failure takes the same path, so more reasons only multiply -\* states (every reason for every operation made this instance exceed 12 GB -\* of TLC state on fede without finishing). +\* Deep instance (weekly, --deep): the fast instance's three start_child +\* calls (Permanent, Transient, Temporary) with a deeper budget and window +\* (budget 2, two window releases, so up to four restarts) and both slots of +\* max_children 2 contested; safety and the action property. Reasons are +\* one per class: the decisions depend only on Normal-class (Normal, +\* Shutdown) versus abnormal (Panic, Kill, Killed). Four slots do not fit: +\* with one reason per class they passed 222M distinct states in 30 minutes +\* on fede without finishing. SPECIFICATION Spec CONSTANTS - N = 4 - Mode <- MCMode4 - MaxChildren = 3 + N = 3 + Mode <- MCMode3 + MaxChildren = 2 MaxRestarts = 2 MaxExpire = 2 SelfExits = {"Normal", "Panic"} - StartFails = {"none"} + StartFails = {"none", "Panic"} Fault = "None" INVARIANT TypeOK INVARIANT AtMostOneGeneration diff --git a/verification/tla/README.md b/verification/tla/README.md index d3ef5aff..2fa4f1ce 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (4 slots, every reason). | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (4 slots, `max_children` 3, budget 2, one reason per class). | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | From dd8ba39555a9ea1fd6654fa548dee803f07e43da Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:34:45 -0300 Subject: [PATCH 10/52] docs: TLC results for the DynamicSupervisor configurations --- docs/design/dynamic-supervisor.md | 4 ++-- verification/tla/MCDynamicSupervisor.tla | 6 ------ verification/tla/README.md | 5 +++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 8a254bd0..b6d10ff3 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -649,8 +649,8 @@ TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (every PR; safety, action property, liveness) | 3 slots (Permanent, Transient, Temporary), `max_children` 2, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Panic` | RESULTS_FAST | | | -| `MCDynamicSupervisorDeep` (weekly; safety, action property) | 4 slots (P, T, Temp, P), `max_children` 3, budget 2, two releases, one reason per class (the decisions depend only on normal versus abnormal; with every reason the instance passed 12 GB of TLC state without finishing) | RESULTS_DEEP | | | +| `MCDynamicSupervisor` (every PR; safety, action property, liveness) | 3 slots (Permanent, Transient, Temporary), `max_children` 2, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Panic` | 1,601,979 (5,010,572 generated) | 33 | 2m16s | +| `MCDynamicSupervisorDeep` (weekly; safety, action property) | the same 3 slots and cap with budget 2 and two window releases (up to four restarts); one reason per class, since the decisions depend only on normal versus abnormal. Four slots did not fit: 222M distinct states after 30 min without finishing | 5,617,958 (18,853,746 generated) | 42 | 41 s | Negative controls (`negative_checks.py`; each checks `TypeOK` and only the named property, on `MCDynamicSupervisor`): diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla index c2b3f681..e5c82bef 100644 --- a/verification/tla/MCDynamicSupervisor.tla +++ b/verification/tla/MCDynamicSupervisor.tla @@ -6,10 +6,4 @@ EXTENDS DynamicSupervisor MCMode3 == [s \in 1..3 |-> CASE s = 1 -> "Permanent" [] s = 2 -> "Transient" [] OTHER -> "Temporary"] -\* Four slots for the deep instance: two Permanent so two restarting -\* children can hold the cap together. -MCMode4 == [s \in 1..4 |-> CASE s = 1 -> "Permanent" - [] s = 2 -> "Transient" - [] s = 3 -> "Temporary" - [] OTHER -> "Permanent"] ============================================================================= diff --git a/verification/tla/README.md b/verification/tla/README.md index 2fa4f1ce..2d7652d1 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (4 slots, `max_children` 3, budget 2, one reason per class). | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (the same 3 slots with budget 2 and two window releases). | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -617,7 +617,8 @@ Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -RESULTS_ROWS +| `MCDynamicSupervisor` (fast: 3 slots, `max_children` 2, budget 1, one window release; safety, `NeverReopens`, liveness) | 1,601,979 | 5,010,572 | 33 | 2m16s | +| `MCDynamicSupervisorDeep` (budget 2, two releases; safety, `NeverReopens`) | 5,617,958 | 18,853,746 | 42 | 41 s | All eleven controls find their counterexample in under 5 s each. Not yet: Apalache inductive invariants (needs type annotations and an `IndInv`, as From 0f12f0b59b6763cdfd45442a941eb49d9e4276ef Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:57 -0300 Subject: [PATCH 11/52] verify: DynamicSupervisor revision 2 (serialized loop, reservations, history invariants, stuck isolation, terminate resolution) --- verification/tla/DynamicSupervisor.tla | 772 +++++++++++-------- verification/tla/MCDynamicSupervisor.cfg | 31 +- verification/tla/MCDynamicSupervisorDeep.cfg | 32 +- verification/tla/MCDynamicSupervisorZero.cfg | 41 + verification/tla/MCDynamicSupervisorZero.tla | 3 + 5 files changed, 534 insertions(+), 345 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorZero.cfg create mode 100644 verification/tla/MCDynamicSupervisorZero.tla diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 9ac5f29b..6a610594 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -2,55 +2,41 @@ (***************************************************************************) (* Rank 12a design model (not implemented): a dynamic supervisor, OTP's *) (* simple_one_for_one / Elixir's DynamicSupervisor. Design contract: *) -(* docs/design/dynamic-supervisor.md. *) +(* docs/design/dynamic-supervisor.md (revision 2, after review). *) (* *) (* Slots 1..N are potential start_child calls; each is made at most once, *) -(* at any time, so starts race crashes, restarts, terminate_child, *) -(* max_children and shutdown. A slot that is admitted owns one child: *) -(* its first generation, then restarts with fresh generations (fresh *) -(* ActorIds) until the child is removed. Restart types are per slot so *) -(* one instance covers all three (a simple_one_for_one template has one *) -(* type; mixing them is the heterogeneous case and loses nothing). *) +(* at any time, and each started child can get one terminate_child call. *) +(* As in OTP, one control loop (`op`) serialises the supervisor's *) +(* operations: admitting a start and constructing its first generation, *) +(* a restart decision and the restart attempt, and a terminate_child, *) +(* each runs to completion before the next. Children run, exit and clean *) +(* up concurrently. Shutdown (Close) and failure close admission at once *) +(* and cancel the operation in progress (B: tree-wide admission closure). *) (* *) -(* Abstractions, as in Supervision.tla: *) -(* - The admission gate and the supervisor's cancel are one boolean *) -(* `open`; shutdown (Close) and failure (Fail) both close it. *) -(* - Time is abstract. The rolling restart window is a counter released *) -(* by WindowExpire, at most MaxExpire times, which also bounds the *) -(* number of generations (no state constraint is needed). *) +(* Restart types are per slot so one instance covers all three (a *) +(* simple_one_for_one template has one type; mixing them loses nothing). *) +(* *) +(* Abstractions: *) +(* - `open` is the admission gate and the supervisor's cancel together. *) +(* - Time is abstract. The rolling window is `windowed` (stamped grants *) +(* still in the window), released by WindowExpire at most MaxExpire *) +(* times, which also bounds the generations. Backoff is the "backoff" *) +(* state (zero-length when backoff is disabled, the default). *) (* - The shutdown_timeout -> kill escalation is the nondeterministic *) (* reason of StopObserved (Shutdown, or Killed after the kill). A *) -(* cleanup that misses its second interval is CleanupStuck: the *) -(* supervisor fails, the generation stays alive until it really ends. *) -(* - Backoff is the "backoff" state; its sleep is not modelled. *) -(* - The budget is charged when a restart attempt is admitted; the *) -(* implementation's reserve-then-stamp protocol is checked in *) -(* Supervision.tla and shared code. *) +(* child that misses its deadlines, before its death point (signalled *) +(* but still running) or after it (owned cleanup), is CleanupStuck. *) +(* - The shutdown fan-out is one step (the implementation sends every *) +(* signal without awaiting between sends; each child's intervals run *) +(* from its own signal, design E3). *) +(* *) +(* Invariants that name an event (a late commit, a late generation, a *) +(* policy violation) read history variables that GhostUpdate computes *) +(* from every step's state change, independently of the actions, so a *) +(* mutated action cannot hide its own violation. *) (* *) (* Fault selects a negative control; negative_checks.py requires each to *) -(* violate its named invariant: *) -(* IgnoreMax (MaxChildrenBound) admission ignores the cap *) -(* UncountedRestart (MaxChildrenBound) a crashed child frees its *) -(* slot until it restarts *) -(* RestartTemporary (TemporaryNeverRestarted) *) -(* RestartTerminated (TerminatedNeverRestarted) terminate_child *) -(* does not cancel restart *) -(* AcceptDuringShutdown (NoStartAfterShutdown) start_child admitted and *) -(* committed after close *) -(* RestartAfterClose (NoGenerationAfterClose) a pending restart *) -(* proceeds after close *) -(* NoEscalate (ExhaustionCloses) budget exhaustion drops *) -(* only that child *) -(* EarlyDone (NoChildOutlivesSupervisor) done before every *) -(* child finished *) -(* Sequential (ConcurrentShutdown) children stopped one at a *) -(* time, as a static *) -(* supervisor does *) -(* and two liveness controls (named_temporal): *) -(* NoAbort (StartAnswered) a start racing shutdown *) -(* is not cancelled *) -(* StuckBackoff (ClosedLeadsToDone) a pending restart is not *) -(* cancelled by shutdown *) +(* violate its named property (see DYNAMIC_CONTROLS there). *) (***************************************************************************) EXTENDS Naturals, FiniteSets @@ -58,385 +44,525 @@ CONSTANTS N, \* number of start_child calls (slots 1..N) Mode, \* Mode[s] \in {"Permanent", "Transient", "Temporary"} MaxChildren, \* max_children - MaxRestarts, \* restart intensity (max_restarts per window) - MaxExpire, \* how often the rolling window may release an entry + MaxRestarts, \* restart intensity (grants per window) + MaxExpire, \* how often the rolling window may release a grant SelfExits, \* reasons a ready child may exit with on its own StartFails, \* outcomes of a failed construction or readiness + CleanupFails, \* {FALSE} or {FALSE, TRUE}: cleanup may record a failure Fault -Slots == 1..N -Modes == {"Permanent", "Transient", "Temporary"} +Slots == 1..N +Modes == {"Permanent", "Transient", "Temporary"} \* "none": factory error or panic (no actor reason); "timeout": startup \* deadline; otherwise the actor's exit reason (v0.8 reasons, design E5). Reasons == {"none", "timeout", "Normal", "Shutdown", "Panic", "Kill", "Killed"} Abnormal == {"none", "timeout", "Panic", "Kill", "Killed"} - -ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat \ {0} +Faults == {"None", + \* revision 1 controls + "IgnoreMax", "UncountedRestart", "RestartTemporary", "RestartTerminated", + "AcceptDuringShutdown", "RestartAfterClose", "NoEscalate", "EarlyDone", + "Sequential", "NoAbort", "StuckBackoff", + \* revision 2 controls (review findings) + "ConcurrentStart", "StuckFailsAll", "PublishAfterClose", "OverlapGeneration", + "PublishBeforeReady", "PrematureOk", "UnchargedRetry", "ChargeFirstStart", + "TransientNormalRestart", "PermanentRetires", "RetryFirstStart", + "EarlyFailedReply", "NoRefund", "ExpireReserved", "TermReplyEarly", + "LateFailedOverwrite"} + +ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat ASSUME MaxRestarts \in Nat /\ MaxExpire \in Nat ASSUME \A s \in Slots : Mode[s] \in Modes ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons -ASSUME Fault \in {"None", "IgnoreMax", "UncountedRestart", "RestartTemporary", - "RestartTerminated", "AcceptDuringShutdown", "RestartAfterClose", - "NoEscalate", "EarlyDone", "Sequential", "NoAbort", - "StuckBackoff"} - -\* Slot states. "idle": start_child not called yet. "starting": the first -\* generation is being constructed (the caller waits). "restarting": a -\* replacement is being constructed. "cleanup": the generation is exiting -\* and its owned cleanup has not finished. "decide": cleanup finished, -\* restart decision pending. "backoff": a restart is pending. -\* "removed" and "rejected" are terminal. -States == {"idle", "starting", "running", "cleanup", "decide", "backoff", - "restarting", "removed", "rejected"} -Active == {"starting", "running", "cleanup", "decide", "backoff", "restarting"} +ASSUME CleanupFails \subseteq BOOLEAN /\ FALSE \in CleanupFails +ASSUME Fault \in Faults + +\* Slot states. "idle": start_child not called. "queued": the call waits +\* for the control loop. "starting": the first generation is constructed. +\* "restarting": a replacement is constructed. "cleanup": the generation is +\* exiting or its owned cleanup has not finished. "decide": cleanup done, +\* the loop has not handled the exit yet. "backoff": a restart is pending, +\* holding a budget reservation. "stuck": cleanup missed its deadline; the +\* slot stays counted until it really ends. "removed"/"rejected": terminal. +States == {"idle", "queued", "starting", "running", "cleanup", "decide", + "backoff", "restarting", "stuck", "removed", "rejected"} +Active == {"starting", "running", "cleanup", "decide", "backoff", + "restarting", "stuck"} Terminal == {"idle", "removed", "rejected"} -\* What start_child returned. "pending" while the caller waits. -Results == {"none", "pending", "ok", "failed", "max", "shutdown"} +\* start_child: "stopped" and "max" are pre-admission rejections (the +\* arguments are returned); "aborted" is shutdown during construction; +\* "failed" is FailedToStart. +Results == {"none", "pending", "ok", "failed", "aborted", "max", "stopped"} +\* terminate_child: "queued" waits for the loop, "pending" for removal. +TermResults == {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped"} + +Idle == [k |-> "idle", s |-> 0] VARIABLES open, \* admission open: neither shut down nor failed - failed, \* the supervisor failed (budget exhausted or cleanup stuck) + failed, \* the supervisor failed + exhausted, \* a restart was refused for lack of budget sphase, \* "running", "signal" (Sequential only), "waiting", "done" - budget, \* restart grants in the current window + windowed, \* stamped grants still in the window expired, \* window releases used (bounds the model) - cs, \* slot state - gen, \* generations started in this slot - alive, \* live (not yet cleaned up) generations - pub, \* generation current() returns, 0 if none - reason, \* exit reason of the latest generation - readyGen, \* the latest generation became ready - everReady, \* the slot's first start succeeded - result, \* start_child's outcome - term, \* terminate_child was called for this child - stopSig, \* the supervisor's stop signal reached this slot - \* Ghosts for the invariants. - termGen, \* gen when terminate_child was called - closeGen, \* gen of each slot when admission closed (meaningful once closed) - lateOk, \* a start_child returned ok after admission closed - exhausted \* a restart was refused for lack of budget - -sup == <> -slot == <> -ghost == <> -vars == <> + reserved, \* slots holding a reservation (pending restarts) + stamped, \* grants ever stamped (charged restart attempts) + op, \* the control loop's current operation [k, s] + sl, \* per-slot record, see SlotInit + \* History, written only by GhostUpdate. + lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, + restarts + +sup == <> +ghost == <> +vars == <> + +SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, + reason |-> "none", cfail |-> FALSE, readyGen |-> FALSE, + everReady |-> FALSE, result |-> "none", cause |-> "none", + term |-> FALSE, termRes |-> "none", stopSig |-> FALSE, + stuck |-> "none"] TypeOK == - /\ open \in BOOLEAN /\ failed \in BOOLEAN + /\ open \in BOOLEAN /\ failed \in BOOLEAN /\ exhausted \in BOOLEAN /\ sphase \in {"running", "signal", "waiting", "done"} - /\ budget \in 0..MaxRestarts /\ expired \in 0..MaxExpire - /\ cs \in [Slots -> States] - /\ gen \in [Slots -> Nat] - /\ alive \in [Slots -> SUBSET Nat] - /\ pub \in [Slots -> Nat] - /\ reason \in [Slots -> Reasons] - /\ readyGen \in [Slots -> BOOLEAN] /\ everReady \in [Slots -> BOOLEAN] - /\ result \in [Slots -> Results] - /\ term \in [Slots -> BOOLEAN] /\ stopSig \in [Slots -> BOOLEAN] - /\ termGen \in [Slots -> Nat] - /\ closeGen \in [Slots -> Nat] - /\ lateOk \in [Slots -> BOOLEAN] - /\ exhausted \in BOOLEAN + /\ windowed \in Nat /\ expired \in 0..MaxExpire + /\ reserved \subseteq Slots /\ stamped \in Nat + /\ op.k \in {"idle", "start", "restart", "term"} /\ op.s \in 0..N + /\ \A s \in Slots : + /\ sl[s].cs \in States /\ sl[s].gen \in Nat /\ sl[s].alive \subseteq Nat + /\ sl[s].pub \in Nat /\ sl[s].reason \in Reasons + /\ sl[s].result \in Results /\ sl[s].termRes \in TermResults + /\ sl[s].cause \in {"none", "failed", "aborted"} + /\ sl[s].stuck \in {"none", "retire", "restart", "closing"} Init == - /\ open = TRUE /\ failed = FALSE /\ sphase = "running" - /\ budget = 0 /\ expired = 0 - /\ cs = [s \in Slots |-> "idle"] - /\ gen = [s \in Slots |-> 0] - /\ alive = [s \in Slots |-> {}] - /\ pub = [s \in Slots |-> 0] - /\ reason = [s \in Slots |-> "none"] - /\ readyGen = [s \in Slots |-> FALSE] - /\ everReady = [s \in Slots |-> FALSE] - /\ result = [s \in Slots |-> "none"] - /\ term = [s \in Slots |-> FALSE] - /\ stopSig = [s \in Slots |-> FALSE] - /\ termGen = [s \in Slots |-> 0] - /\ closeGen = [s \in Slots |-> 0] - /\ lateOk = [s \in Slots |-> FALSE] - /\ exhausted = FALSE - -\* Slots that count against max_children: every admitted child that was not -\* removed, including one that is restarting (in backoff or constructing its -\* replacement) or still cleaning up. The control counts only live, -\* published children. + /\ open = TRUE /\ failed = FALSE /\ exhausted = FALSE /\ sphase = "running" + /\ windowed = 0 /\ expired = 0 /\ reserved = {} /\ stamped = 0 /\ op = Idle + /\ sl = [s \in Slots |-> SlotInit] + /\ lateCommit = [s \in Slots |-> FALSE] /\ latePub = [s \in Slots |-> FALSE] + /\ lateGenClose = [s \in Slots |-> FALSE] /\ lateGenTerm = [s \in Slots |-> FALSE] + /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] + /\ restarts = 0 + +\* Children held: counted against max_children. Counted == IF Fault = "UncountedRestart" - THEN {s \in Slots : cs[s] \in {"starting", "running"}} - ELSE {s \in Slots : cs[s] \in Active} - -\* Close admission (shutdown or failure) and record each slot's generation. -CloseGate == - /\ open' = FALSE - /\ closeGen' = IF open THEN gen ELSE closeGen - -Fail == failed' = TRUE /\ CloseGate - -Set(f, s, v) == [f EXCEPT ![s] = v] + THEN {s \in Slots : sl[s].cs \in {"starting", "running"}} + ELSE {s \in Slots : sl[s].cs \in Active} + +LoopFree(s) == op.k = "idle" \/ op.s = s \/ Fault = "ConcurrentStart" +Release(s) == op' = IF op.s = s THEN Idle ELSE op +Take(k, s) == op' = IF Fault = "ConcurrentStart" /\ k # "term" THEN op ELSE [k |-> k, s |-> s] +Budget == windowed + Cardinality(reserved) + +\* The design's restart table, stated independently of the actions. +DesignAllowsRestart(m, r, cf) == + CASE m = "Temporary" -> FALSE + [] m = "Permanent" -> TRUE + [] m = "Transient" -> r \in {"Panic", "Kill", "Killed"} \/ cf +\* A restart of slot s is allowed now: a ready-then-exited generation under +\* the table, or a failed replacement (never ready) of a restartable child. +Allowed(s) == + /\ sl[s].everReady + /\ \/ DesignAllowsRestart(Mode[s], sl[s].reason, sl[s].cfail) + \/ ~sl[s].readyGen /\ Mode[s] # "Temporary" + +\* What the supervisor decides (with the policy faults). +WouldRestart(s, cf) == + /\ sl[s].everReady \/ Fault = "RetryFirstStart" + /\ \/ Mode[s] = "Permanent" /\ Fault # "PermanentRetires" + \/ Mode[s] = "Transient" + /\ (sl[s].reason \in Abnormal \/ cf \/ Fault = "TransientNormalRestart") + \/ Mode[s] # "Temporary" /\ ~sl[s].readyGen + \/ Mode[s] = "Temporary" /\ Fault = "RestartTemporary" + +CloseGate == open' = FALSE +Fail == failed' = TRUE /\ CloseGate ----------------------------------------------------------------------------- (* Callers and environment *) -\* start_child(args). Admission is atomic with the count and the gate. +\* start_child(args): queued for the loop, or rejected at once if closed. Request(s) == - /\ cs[s] = "idle" - /\ IF ~open /\ Fault # "AcceptDuringShutdown" - THEN /\ cs' = Set(cs, s, "rejected") /\ result' = Set(result, s, "shutdown") - /\ UNCHANGED <> - ELSE IF Cardinality(Counted) >= MaxChildren /\ Fault # "IgnoreMax" - THEN /\ cs' = Set(cs, s, "rejected") /\ result' = Set(result, s, "max") - /\ UNCHANGED <> - ELSE /\ cs' = Set(cs, s, "starting") /\ result' = Set(result, s, "pending") - /\ gen' = Set(gen, s, 1) /\ alive' = Set(alive, s, {1}) - /\ reason' = Set(reason, s, "none") /\ readyGen' = Set(readyGen, s, FALSE) - /\ UNCHANGED <> + /\ sl[s].cs = "idle" + /\ sl' = IF ~open + THEN [sl EXCEPT ![s].cs = "rejected", ![s].result = "stopped"] + ELSE [sl EXCEPT ![s].cs = "queued", ![s].result = "pending"] + /\ UNCHANGED sup + +\* The loop admits a queued start: cap check, then the first generation. +LoopAdmit(s) == + /\ sl[s].cs = "queued" /\ LoopFree(s) + /\ open \/ Fault = "AcceptDuringShutdown" + /\ IF Cardinality(Counted) >= MaxChildren /\ Fault # "IgnoreMax" + THEN /\ sl' = [sl EXCEPT ![s].cs = "rejected", ![s].result = "max"] + /\ UNCHANGED <> + ELSE /\ sl' = [sl EXCEPT ![s].cs = "starting", ![s].gen = 1, ![s].alive = {1}, + ![s].result = IF Fault = "PrematureOk" THEN "ok" ELSE "pending"] + /\ Take("start", s) + /\ IF Fault = "ChargeFirstStart" + THEN stamped' = stamped + 1 /\ windowed' = windowed + 1 + ELSE UNCHANGED <> + /\ UNCHANGED <> + +\* Shutdown rejects queued starts before admission (arguments returned). +RejectQueued(s) == + /\ sl[s].cs = "queued" /\ ~open + /\ sl' = [sl EXCEPT ![s].cs = "rejected", ![s].result = "stopped"] + /\ UNCHANGED sup \* Supervisor::stop / shutdown, system shutdown, or the parent stopping it. Close == - /\ open - /\ CloseGate - /\ UNCHANGED <> - -\* terminate_child(id) on a child whose start returned. It never restarts -\* afterwards, whatever its restart type. After close it is a no-op (the -\* supervisor already stops every child), so it is only modelled while open. -TerminateChild(s) == - /\ open /\ everReady[s] /\ ~term[s] - /\ cs[s] \in {"running", "cleanup", "decide", "backoff", "restarting"} - /\ term' = Set(term, s, TRUE) - /\ stopSig' = Set(stopSig, s, TRUE) - /\ termGen' = Set(termGen, s, gen[s]) - /\ UNCHANGED <> - -\* The rolling window forgets its oldest grant. + /\ open /\ CloseGate + /\ UNCHANGED <> + +\* The rolling window forgets its oldest stamped grant; reservations stay. WindowExpire == - /\ budget > 0 /\ expired < MaxExpire - /\ budget' = budget - 1 /\ expired' = expired + 1 - /\ UNCHANGED <> + /\ expired < MaxExpire + /\ IF windowed > 0 + THEN windowed' = windowed - 1 /\ UNCHANGED reserved + ELSE /\ Fault = "ExpireReserved" /\ reserved # {} + /\ \E r \in reserved : reserved' = reserved \ {r} + /\ UNCHANGED windowed + /\ expired' = expired + 1 + /\ UNCHANGED <> + +\* terminate_child(id) by a caller holding the child's last published id. +TerminateRequest(s) == + /\ sl[s].termRes = "none" /\ sl[s].everReady + /\ sl' = [sl EXCEPT ![s].termRes = IF open THEN "queued" ELSE "stopped"] + /\ UNCHANGED sup + +\* The loop handles it: mark terminated, signal, and hold the loop until +\* the child is removed (OTP terminate_child blocks the supervisor). +LoopTerm(s) == + /\ sl[s].termRes = "queued" /\ op.k = "idle" /\ open + /\ IF sl[s].cs = "removed" + THEN sl' = [sl EXCEPT ![s].termRes = "notfound"] /\ UNCHANGED op + ELSE IF sl[s].cs = "stuck" + THEN sl' = [sl EXCEPT ![s].termRes = "stuck"] /\ UNCHANGED op + ELSE /\ sl' = [sl EXCEPT ![s].term = TRUE, ![s].stopSig = TRUE, + ![s].termRes = "pending"] + /\ op' = [k |-> "term", s |-> s] + /\ UNCHANGED <> + +\* A terminate queued when shutdown began resolves with the removal. +TermQueuedClosed(s) == + /\ sl[s].termRes = "queued" /\ ~open + /\ sl' = [sl EXCEPT ![s].termRes = "pending"] + /\ UNCHANGED sup + +TermResolve(s) == + /\ sl[s].termRes = "pending" + /\ sl[s].cs \in {"removed", "stuck"} \/ Fault = "TermReplyEarly" + /\ sl' = [sl EXCEPT ![s].termRes = IF sl[s].cs = "stuck" THEN "stuck" ELSE "ok"] + /\ IF op = [k |-> "term", s |-> s] THEN op' = Idle ELSE UNCHANGED op + /\ UNCHANGED <> ----------------------------------------------------------------------------- (* One child slot *) -\* started() returned while admission is open: readiness commits, and for -\* a first start start_child returns Ok(ref). +\* started() returned while admission is open: readiness commits; for a +\* first start the caller's Ok is committed (historical from now on). Ready(s) == - /\ cs[s] \in {"starting", "restarting"} - /\ open \/ Fault = "AcceptDuringShutdown" - /\ ~(term[s] /\ cs[s] = "restarting") - /\ cs' = Set(cs, s, "running") - /\ pub' = Set(pub, s, gen[s]) - /\ readyGen' = Set(readyGen, s, TRUE) - /\ everReady' = Set(everReady, s, TRUE) - /\ result' = IF cs[s] = "starting" THEN Set(result, s, "ok") ELSE result - /\ lateOk' = IF cs[s] = "starting" /\ ~open THEN Set(lateOk, s, TRUE) ELSE lateOk - /\ UNCHANGED <> + /\ sl[s].cs \in {"starting", "restarting"} + /\ \/ open + \/ Fault = "AcceptDuringShutdown" /\ sl[s].cs = "starting" + \/ Fault = "PublishAfterClose" /\ sl[s].cs = "restarting" + /\ sl' = [sl EXCEPT ![s].cs = "running", ![s].pub = sl[s].gen, + ![s].readyGen = TRUE, ![s].everReady = TRUE, + ![s].result = IF sl[s].cs = "starting" THEN "ok" ELSE sl[s].result] + /\ Release(s) + /\ UNCHANGED <> \* Factory error or panic, panic or stop in started(), startup timeout. StartFail(s, r) == - /\ cs[s] \in {"starting", "restarting"} - /\ r \in StartFails - /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) - /\ UNCHANGED <> - -\* Admission closed (or the child was terminated) during construction: the -\* attempt is cancelled and cleaned up; it never publishes. + /\ sl[s].cs \in {"starting", "restarting"} /\ r \in StartFails + /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0, + ![s].cause = IF sl[s].cs = "starting" THEN "failed" ELSE sl[s].cause, + ![s].result = IF Fault = "EarlyFailedReply" /\ sl[s].cs = "starting" + THEN "failed" ELSE sl[s].result] + /\ UNCHANGED sup + +\* Admission closed during construction: the attempt is cancelled. AbortStart(s) == - /\ cs[s] \in {"starting", "restarting"} - /\ (~open /\ Fault # "NoAbort") \/ term[s] - /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, "Shutdown") - /\ UNCHANGED <> + /\ sl[s].cs \in {"starting", "restarting"} /\ ~open /\ Fault # "NoAbort" + /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = "Shutdown", ![s].pub = 0, + ![s].cause = IF sl[s].cs = "starting" THEN "aborted" ELSE sl[s].cause] + /\ UNCHANGED sup -\* A ready child exits on its own. SelfExit(s, r) == - /\ cs[s] = "running" /\ r \in SelfExits - /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) - /\ pub' = Set(pub, s, 0) - /\ UNCHANGED <> - -\* The supervisor's parent exit signal (Shutdown) reached the child, from -\* terminate_child or supervisor shutdown. It ends Shutdown, or Killed if -\* it missed shutdown_timeout and was killed (design E3). + /\ sl[s].cs = "running" /\ r \in SelfExits + /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0] + /\ UNCHANGED sup + +\* The parent exit signal (terminate_child or shutdown) reached the child: +\* it ends Shutdown, or Killed after missing shutdown_timeout (design E3). StopObserved(s, r) == - /\ cs[s] = "running" /\ stopSig[s] - /\ r \in {"Shutdown", "Killed"} - /\ cs' = Set(cs, s, "cleanup") /\ reason' = Set(reason, s, r) - /\ pub' = Set(pub, s, 0) - /\ UNCHANGED <> - -\* The generation reached publication and its owned cleanup joined. -CleanupDone(s) == - /\ cs[s] = "cleanup" - /\ alive' = Set(alive, s, {}) - /\ cs' = Set(cs, s, "decide") - /\ UNCHANGED <> - -\* Cleanup missed its deadline: the supervisor fails (and so stops every -\* other child); the generation stays alive until CleanupDone. + /\ sl[s].cs = "running" /\ sl[s].stopSig /\ r \in {"Shutdown", "Killed"} + /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0] + /\ UNCHANGED sup + +\* Publication and owned cleanup finished; a destructor or helper failure +\* may be recorded (a Transient child then restarts). +CleanupDone(s, f) == + /\ sl[s].cs = "cleanup" /\ f \in CleanupFails + /\ sl' = [sl EXCEPT ![s].cs = "decide", ![s].cfail = f, + ![s].alive = sl[s].alive \ {sl[s].gen}] + /\ UNCHANGED sup + +\* The child missed its deadlines: still running after its stop signal and +\* the kill, or its cleanup is stuck. The slot stays counted and alive. +\* It fails the supervisor only when the child would have been restarted +\* (V3: no replacement while the old generation lives) or while closing; +\* a child that was retiring anyway fails only itself. CleanupStuck(s) == - /\ cs[s] = "cleanup" /\ ~failed - /\ Fail - /\ UNCHANGED <> - -\* OTP's restart table, plus: a replacement that fails before readiness is a -\* failed restart attempt and is retried unless Temporary (V1). -Restarts(s) == - \/ Mode[s] = "Permanent" - \/ Mode[s] = "Transient" /\ reason[s] \in Abnormal - \/ Mode[s] # "Temporary" /\ ~readyGen[s] - \/ Mode[s] = "Temporary" /\ Fault = "RestartTemporary" - + /\ sl[s].stuck = "none" + /\ \/ sl[s].cs = "cleanup" + \/ sl[s].cs = "running" /\ sl[s].stopSig + /\ LET kind == IF ~open THEN "closing" + ELSE IF ~sl[s].term /\ sl[s].cs = "cleanup" /\ WouldRestart(s, TRUE) + THEN "restart" ELSE "retire" + IN /\ sl' = [sl EXCEPT ![s].cs = "stuck", ![s].stuck = kind, ![s].pub = 0, + ![s].result = IF sl[s].result = "pending" + THEN sl[s].cause ELSE sl[s].result] + /\ IF kind # "retire" \/ Fault = "StuckFailsAll" + THEN Fail ELSE UNCHANGED <> + /\ IF op.s = s /\ op.k # "term" THEN op' = Idle ELSE UNCHANGED op + /\ UNCHANGED <> + +\* A stuck child finally ends (assumed for liveness only). +StuckDone(s) == + /\ sl[s].cs = "stuck" + /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].alive = {}] + /\ UNCHANGED sup + +\* The loop handles the exit: first-start failure, removal, or a restart +\* that reserves budget first (failing at once if none is left). Decide(s) == - /\ cs[s] = "decide" - /\ IF ~everReady[s] - \* First start failed: start_child returns the error; never restarted - \* and never charged to the budget (OTP start_child). - THEN /\ cs' = Set(cs, s, "removed") - /\ result' = Set(result, s, IF open THEN "failed" ELSE "shutdown") - ELSE IF ~open \/ (term[s] /\ Fault # "RestartTerminated") - THEN cs' = Set(cs, s, "removed") /\ UNCHANGED result - ELSE IF Restarts(s) - THEN cs' = Set(cs, s, "backoff") /\ UNCHANGED result - ELSE cs' = Set(cs, s, "removed") /\ UNCHANGED result - /\ UNCHANGED <> - -\* Backoff elapsed (or was cancelled). A restart needs a budget grant; -\* without one the supervisor fails and escalates. + /\ \/ sl[s].cs = "decide" + \/ Fault = "OverlapGeneration" /\ sl[s].cs = "cleanup" + /\ LoopFree(s) + /\ IF ~sl[s].everReady /\ Fault # "RetryFirstStart" + THEN /\ sl' = [sl EXCEPT ![s].cs = "removed", + ![s].result = IF Fault = "LateFailedOverwrite" /\ ~open + THEN "aborted" ELSE sl[s].cause] + /\ UNCHANGED <> + ELSE IF ~open \/ (sl[s].term /\ Fault # "RestartTerminated") + THEN /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ UNCHANGED <> + ELSE IF WouldRestart(s, sl[s].cfail) + THEN IF Budget < MaxRestarts + THEN /\ sl' = [sl EXCEPT ![s].cs = "backoff"] + /\ reserved' = reserved \cup {s} + /\ UNCHANGED <> + ELSE /\ exhausted' = TRUE + /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ IF Fault = "NoEscalate" THEN UNCHANGED <> ELSE Fail + /\ UNCHANGED reserved + ELSE /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ UNCHANGED <> + /\ Release(s) + /\ UNCHANGED <> + +\* Backoff elapsed: stamp the reservation and construct the replacement. BackoffDone(s) == - /\ cs[s] = "backoff" - /\ open \/ term[s] \/ Fault # "StuckBackoff" - /\ IF (~open /\ Fault # "RestartAfterClose") \/ (term[s] /\ Fault # "RestartTerminated") - THEN /\ cs' = Set(cs, s, "removed") - /\ UNCHANGED <> - ELSE IF budget < MaxRestarts - THEN /\ budget' = budget + 1 - /\ cs' = Set(cs, s, "restarting") - /\ gen' = Set(gen, s, gen[s] + 1) - /\ alive' = Set(alive, s, {gen[s] + 1}) - /\ reason' = Set(reason, s, "none") - /\ readyGen' = Set(readyGen, s, FALSE) - /\ UNCHANGED <> - ELSE /\ exhausted' = TRUE - /\ cs' = Set(cs, s, "removed") - /\ IF Fault = "NoEscalate" THEN UNCHANGED <> ELSE Fail - /\ UNCHANGED <> - /\ UNCHANGED <> + /\ sl[s].cs = "backoff" /\ LoopFree(s) + /\ open \/ Fault = "RestartAfterClose" + /\ ~sl[s].term \/ Fault = "RestartTerminated" + /\ LET g == sl[s].gen + 1 + charge == ~(Fault = "UnchargedRetry" /\ ~sl[s].readyGen) + IN /\ sl' = [sl EXCEPT ![s].cs = "restarting", ![s].gen = g, + ![s].alive = sl[s].alive \cup {g}, + ![s].pub = IF Fault = "PublishBeforeReady" THEN g ELSE 0, + ![s].reason = "none", ![s].cfail = FALSE, + ![s].readyGen = FALSE] + /\ reserved' = reserved \ {s} + /\ IF charge THEN stamped' = stamped + 1 /\ windowed' = windowed + 1 + ELSE UNCHANGED <> + /\ Take("restart", s) + /\ UNCHANGED <> + +\* Shutdown or terminate_child cancels a pending restart and refunds it. +CancelBackoff(s) == + /\ sl[s].cs = "backoff" + /\ \/ ~open /\ Fault # "StuckBackoff" + \/ sl[s].term /\ Fault # "RestartTerminated" + /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ reserved' = IF Fault = "NoRefund" THEN reserved ELSE reserved \ {s} + /\ UNCHANGED <> ----------------------------------------------------------------------------- -(* Supervisor shutdown: every child is signalled at once *) +(* Supervisor shutdown: every child is signalled without awaiting another *) SupSignal == /\ sphase = "running" /\ ~open /\ Fault # "Sequential" - /\ stopSig' = [s \in Slots |-> TRUE] + /\ sl' = [s \in Slots |-> [sl[s] EXCEPT !.stopSig = TRUE]] /\ sphase' = "waiting" - /\ UNCHANGED <> + /\ UNCHANGED <> -\* Control: one child at a time, the next only after the previous finished. +\* Control: one child at a time, the next only after the previous ended. SupSignalOne == /\ sphase \in {"running", "signal"} /\ ~open /\ Fault = "Sequential" - /\ \A s \in Slots : stopSig[s] => cs[s] \in Terminal - /\ LET todo == {s \in Slots : ~stopSig[s] /\ cs[s] \notin Terminal} + /\ \A s \in Slots : sl[s].stopSig => sl[s].cs \in Terminal \cup {"stuck"} + /\ LET todo == {s \in Slots : ~sl[s].stopSig /\ sl[s].cs \notin Terminal} IN IF todo = {} - THEN sphase' = "waiting" /\ UNCHANGED stopSig - ELSE \E s \in todo : stopSig' = Set(stopSig, s, TRUE) /\ sphase' = "signal" - /\ UNCHANGED <> + THEN sphase' = "waiting" /\ UNCHANGED sl + ELSE \E s \in todo : sl' = [sl EXCEPT ![s].stopSig = TRUE] /\ sphase' = "signal" + /\ UNCHANGED <> -\* Done once every child finished (removed, rejected or never requested). SupDone == /\ sphase = "waiting" - /\ (\A s \in Slots : cs[s] \in Terminal) \/ Fault = "EarlyDone" + /\ (\A s \in Slots : sl[s].cs \in Terminal) \/ Fault = "EarlyDone" /\ sphase' = "done" - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- +Grew(s) == sl'[s].gen > sl[s].gen + +\* History, computed from each step's effect, not from the action taken. +GhostUpdate == + /\ lateCommit' = [s \in Slots |-> lateCommit[s] \/ + (sl'[s].result = "ok" /\ sl[s].result # "ok" /\ ~open)] + /\ latePub' = [s \in Slots |-> latePub[s] \/ + (sl'[s].pub # 0 /\ sl[s].pub = 0 /\ (~open \/ sl[s].term))] + /\ lateGenClose' = [s \in Slots |-> lateGenClose[s] \/ (Grew(s) /\ ~open)] + /\ lateGenTerm' = [s \in Slots |-> lateGenTerm[s] \/ (Grew(s) /\ sl[s].term)] + /\ badRestart' = [s \in Slots |-> badRestart[s] \/ + (Grew(s) /\ sl[s].gen > 0 /\ ~Allowed(s))] + /\ badRetire' = [s \in Slots |-> badRetire[s] \/ + (sl[s].cs = "decide" /\ sl'[s].cs = "removed" /\ open' + /\ ~sl[s].term /\ Allowed(s))] + /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) + SlotStep(s) == - \/ Request(s) \/ TerminateChild(s) \/ Ready(s) - \/ \E r \in StartFails : StartFail(s, r) - \/ AbortStart(s) + \/ Request(s) \/ LoopAdmit(s) \/ RejectQueued(s) + \/ TerminateRequest(s) \/ LoopTerm(s) \/ TermQueuedClosed(s) \/ TermResolve(s) + \/ Ready(s) \/ \E r \in StartFails : StartFail(s, r) \/ AbortStart(s) \/ \E r \in SelfExits : SelfExit(s, r) \/ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r) - \/ CleanupDone(s) \/ CleanupStuck(s) \/ Decide(s) \/ BackoffDone(s) + \/ \E f \in CleanupFails : CleanupDone(s, f) + \/ CleanupStuck(s) \/ StuckDone(s) + \/ Decide(s) \/ BackoffDone(s) \/ CancelBackoff(s) -Next == - \/ Close \/ WindowExpire - \/ SupSignal \/ SupSignalOne \/ SupDone +Act == + \/ Close \/ WindowExpire \/ SupSignal \/ SupSignalOne \/ SupDone \/ \E s \in Slots : SlotStep(s) -\* Progress: construction resolves, signalled children stop, cleanup ends -\* (CleanupStuck and crashes are not fair), the supervisor moves on. +Next == Act /\ GhostUpdate + +\* Progress: the loop, construction, stops, cleanup and stuck children +\* eventually move (crashes, start failures and getting stuck are not fair). Fairness == /\ \A s \in Slots : - /\ WF_vars(Ready(s)) /\ WF_vars(AbortStart(s)) - /\ WF_vars(\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) - /\ WF_vars(CleanupDone(s)) /\ WF_vars(Decide(s)) /\ WF_vars(BackoffDone(s)) - /\ WF_vars(SupSignal) /\ WF_vars(SupSignalOne) /\ WF_vars(SupDone) + /\ WF_vars(LoopAdmit(s) /\ GhostUpdate) /\ WF_vars(RejectQueued(s) /\ GhostUpdate) + /\ WF_vars(LoopTerm(s) /\ GhostUpdate) /\ WF_vars(TermQueuedClosed(s) /\ GhostUpdate) + /\ WF_vars(TermResolve(s) /\ GhostUpdate) + /\ WF_vars(Ready(s) /\ GhostUpdate) /\ WF_vars(AbortStart(s) /\ GhostUpdate) + /\ WF_vars((\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) /\ GhostUpdate) + /\ WF_vars((\E f \in CleanupFails : CleanupDone(s, f)) /\ GhostUpdate) + /\ WF_vars(StuckDone(s) /\ GhostUpdate) + /\ WF_vars(Decide(s) /\ GhostUpdate) /\ WF_vars(BackoffDone(s) /\ GhostUpdate) + /\ WF_vars(CancelBackoff(s) /\ GhostUpdate) + /\ WF_vars(SupSignal /\ GhostUpdate) /\ WF_vars(SupSignalOne /\ GhostUpdate) + /\ WF_vars(SupDone /\ GhostUpdate) Spec == Init /\ [][Next]_vars /\ Fairness ----------------------------------------------------------------------------- (* Safety *) -\* A slot never has two live generations (V3). -AtMostOneGeneration == \A s \in Slots : Cardinality(alive[s]) <= 1 +\* A slot never has two live generations (V3); alive accumulates and only +\* the generation whose cleanup finished is removed. +AtMostOneGeneration == \A s \in Slots : Cardinality(sl[s].alive) <= 1 -\* current() returns only the live, latest generation (V4). +\* current() returns only the live, latest, ready generation (V4). PublishedIsLive == - \A s \in Slots : pub[s] # 0 => pub[s] = gen[s] /\ pub[s] \in alive[s] + \A s \in Slots : sl[s].pub # 0 => + sl[s].pub = sl[s].gen /\ sl[s].pub \in sl[s].alive /\ sl[s].readyGen -\* No child outlives the supervisor: done means every child finished. NoChildOutlivesSupervisor == - sphase = "done" => \A s \in Slots : alive[s] = {} /\ cs[s] \in Terminal + sphase = "done" => \A s \in Slots : sl[s].alive = {} /\ sl[s].cs \in Terminal + +\* Children held, restarting and stuck ones included, never exceed the cap. +MaxChildrenBound == Cardinality({s \in Slots : sl[s].cs \in Active}) <= MaxChildren + +\* The control loop constructs one generation at a time (OTP). +SerialConstruction == + Cardinality({s \in Slots : sl[s].cs \in {"starting", "restarting"}}) <= 1 + +TemporaryNeverRestarted == \A s \in Slots : Mode[s] = "Temporary" => sl[s].gen <= 1 + +\* No generation begins after terminate_child, or after admission closed. +TerminatedNeverRestarted == \A s \in Slots : ~lateGenTerm[s] +NoGenerationAfterClose == \A s \in Slots : ~lateGenClose[s] + +\* No generation is published after terminate_child or close. +NoPublishAfterStop == \A s \in Slots : ~latePub[s] -\* The live count, restarting children included, never exceeds the cap. -MaxChildrenBound == - Cardinality({s \in Slots : cs[s] \in Active}) <= MaxChildren +\* A start_child success is committed only while admission is open. +NoStartAfterShutdown == \A s \in Slots : ~lateCommit[s] -TemporaryNeverRestarted == - \A s \in Slots : Mode[s] = "Temporary" => gen[s] <= 1 +\* Restarts follow the table; a restartable exit is never retired while open. +RestartFollowsPolicy == \A s \in Slots : ~badRestart[s] /\ ~badRetire[s] -TerminatedNeverRestarted == - \A s \in Slots : term[s] => gen[s] = termGen[s] +\* A failed first start is never retried. +FirstStartNeverRetried == \A s \in Slots : sl[s].gen > 1 => sl[s].everReady -\* No start_child succeeds after shutdown begins. -NoStartAfterShutdown == \A s \in Slots : ~lateOk[s] +\* FailedToStart / aborted is returned only after cleanup, or at the stuck +\* deadline (the slot then stays counted). +FailedAfterCleanup == + \A s \in Slots : sl[s].result \in {"failed", "aborted"} => + sl[s].cs \in {"removed", "stuck"} -\* No generation (first start or restart) begins after admission closed. -NoGenerationAfterClose == - ~open => \A s \in Slots : gen[s] <= closeGen[s] +\* The error names what happened to the attempt, not the gate afterwards. +FailureCauseKept == + \A s \in Slots : sl[s].result \in {"failed", "aborted"} => sl[s].result = sl[s].cause + +\* Every restart attempt is charged exactly once; first starts never. +ChargedRestarts == restarts = stamped + +\* Reservations are exactly the pending restarts. +ReservationsMatch == reserved = {s \in Slots : sl[s].cs = "backoff"} + +BudgetBounded == Budget <= MaxRestarts -\* Budget exhaustion fails the supervisor, which closes it (and so stops -\* every child: ClosedLeadsToDone). ExhaustionCloses == exhausted => failed /\ ~open -BudgetBounded == budget <= MaxRestarts +\* A child stuck while retiring fails only itself. +StuckIsolated == + failed => exhausted \/ \E s \in Slots : sl[s].stuck \in {"restart", "closing"} -\* Shutdown is concurrent: once the supervisor started stopping, every -\* running child has its stop signal; none waits for a sibling. ConcurrentShutdown == - sphase # "running" => \A s \in Slots : cs[s] = "running" => stopSig[s] + sphase # "running" => \A s \in Slots : sl[s].cs = "running" => sl[s].stopSig + +OkMeansReady == \A s \in Slots : sl[s].result = "ok" => sl[s].everReady -\* start_child returned Ok only for a child that became ready. -OkMeansReady == \A s \in Slots : result[s] = "ok" => everReady[s] +\* terminate_child returns Ok only once the child was removed. +TermOkMeansRemoved == + \A s \in Slots : sl[s].termRes = "ok" => sl[s].cs = "removed" /\ sl[s].alive = {} Safety == - /\ TypeOK /\ AtMostOneGeneration /\ PublishedIsLive - /\ NoChildOutlivesSupervisor /\ MaxChildrenBound - /\ TemporaryNeverRestarted /\ TerminatedNeverRestarted - /\ NoStartAfterShutdown /\ NoGenerationAfterClose - /\ ExhaustionCloses /\ BudgetBounded /\ ConcurrentShutdown /\ OkMeansReady + /\ TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ NoChildOutlivesSupervisor + /\ MaxChildrenBound /\ SerialConstruction /\ TemporaryNeverRestarted + /\ TerminatedNeverRestarted /\ NoGenerationAfterClose /\ NoPublishAfterStop + /\ NoStartAfterShutdown /\ RestartFollowsPolicy /\ FirstStartNeverRetried + /\ FailedAfterCleanup /\ FailureCauseKept /\ ChargedRestarts /\ ReservationsMatch + /\ BudgetBounded /\ ExhaustionCloses /\ StuckIsolated /\ ConcurrentShutdown + /\ OkMeansReady /\ TermOkMeansRemoved NeverReopens == [][open' => open]_vars -(* Liveness, assuming cleanup ends (CleanupStuck is not fair) *) - -\* Shutdown or failure ends with every child stopped. +(* Liveness (assumes stuck children eventually end) *) ClosedLeadsToDone == (~open) ~> (sphase = "done") - -\* Every start_child call gets an answer. -StartAnswered == \A s \in Slots : (result[s] = "pending") ~> (result[s] # "pending") - -\* terminate_child completes: the child is removed. -TerminateCompletes == \A s \in Slots : term[s] ~> (cs[s] = "removed") +StartAnswered == \A s \in Slots : + (sl[s].result = "pending") ~> (sl[s].result # "pending") +TerminateAnswered == \A s \in Slots : + (sl[s].termRes \in {"queued", "pending"}) ~> (sl[s].termRes \notin {"queued", "pending"}) +\* A restartable exit is restarted, or superseded by terminate, shutdown, +\* exhaustion or a stuck cleanup. +RecoveryProgress == \A s \in Slots : + (sl[s].cs \in {"decide", "backoff", "restarting"} /\ sl[s].everReady) + ~> (sl[s].cs \in {"running", "removed", "stuck"}) ============================================================================= diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index bfd4599f..747c1aa0 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -1,31 +1,44 @@ \* Fast instance (CI, every PR): three start_child calls (Permanent, -\* Transient, Temporary) against max_children 2, budget 1 with one window -\* release; safety, action property and liveness. Every DynamicSupervisor -\* negative control fails here on its named invariant. +\* Transient, Temporary), each terminable, against max_children 2; budget 1 +\* with one window release; exits Normal/Panic, start failures factory and +\* Normal-before-readiness, cleanup failures. Safety, NeverReopens and +\* liveness. Every DynamicSupervisor negative control runs here. SPECIFICATION Spec CONSTANTS N = 3 Mode <- MCMode3 MaxChildren = 2 - MaxRestarts = 1 - MaxExpire = 1 SelfExits = {"Normal", "Panic"} - StartFails = {"none", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} Fault = "None" + MaxRestarts = 1 + MaxExpire = 1 INVARIANT TypeOK INVARIANT AtMostOneGeneration INVARIANT PublishedIsLive INVARIANT NoChildOutlivesSupervisor INVARIANT MaxChildrenBound +INVARIANT SerialConstruction INVARIANT TemporaryNeverRestarted INVARIANT TerminatedNeverRestarted -INVARIANT NoStartAfterShutdown INVARIANT NoGenerationAfterClose -INVARIANT ExhaustionCloses +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady +INVARIANT TermOkMeansRemoved PROPERTY NeverReopens PROPERTY ClosedLeadsToDone PROPERTY StartAnswered -PROPERTY TerminateCompletes +PROPERTY TerminateAnswered +PROPERTY RecoveryProgress diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 44162816..17bd39e8 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,32 +1,38 @@ -\* Deep instance (weekly, --deep): the fast instance's three start_child -\* calls (Permanent, Transient, Temporary) with a deeper budget and window -\* (budget 2, two window releases, so up to four restarts) and both slots of -\* max_children 2 contested; safety and the action property. Reasons are -\* one per class: the decisions depend only on Normal-class (Normal, -\* Shutdown) versus abnormal (Panic, Kill, Killed). Four slots do not fit: -\* with one reason per class they passed 222M distinct states in 30 minutes -\* on fede without finishing. +\* Deep instance (weekly, --deep): as the fast instance with budget 2 and +\* two window releases (up to four restarts). Safety and NeverReopens. +\* Four slots do not fit: revision 1 passed 222M distinct states in 30 min. SPECIFICATION Spec CONSTANTS N = 3 Mode <- MCMode3 MaxChildren = 2 - MaxRestarts = 2 - MaxExpire = 2 SelfExits = {"Normal", "Panic"} - StartFails = {"none", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} Fault = "None" + MaxRestarts = 2 + MaxExpire = 2 INVARIANT TypeOK INVARIANT AtMostOneGeneration INVARIANT PublishedIsLive INVARIANT NoChildOutlivesSupervisor INVARIANT MaxChildrenBound +INVARIANT SerialConstruction INVARIANT TemporaryNeverRestarted INVARIANT TerminatedNeverRestarted -INVARIANT NoStartAfterShutdown INVARIANT NoGenerationAfterClose -INVARIANT ExhaustionCloses +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady +INVARIANT TermOkMeansRemoved PROPERTY NeverReopens diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg new file mode 100644 index 00000000..3bf9141a --- /dev/null +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -0,0 +1,41 @@ +\* Zero budget (CI, every PR): max_restarts 0, so the first restartable +\* exit fails the supervisor at once, without entering backoff. +SPECIFICATION Spec +CONSTANTS + N = 3 + Mode <- MCMode3 + MaxChildren = 2 + SelfExits = {"Normal", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} + Fault = "None" + MaxRestarts = 0 + MaxExpire = 0 +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermOkMeansRemoved +PROPERTY NeverReopens +PROPERTY ClosedLeadsToDone +PROPERTY StartAnswered +PROPERTY TerminateAnswered +PROPERTY RecoveryProgress diff --git a/verification/tla/MCDynamicSupervisorZero.tla b/verification/tla/MCDynamicSupervisorZero.tla new file mode 100644 index 00000000..ee809910 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorZero.tla @@ -0,0 +1,3 @@ +----------------------- MODULE MCDynamicSupervisorZero ----------------------- +EXTENDS MCDynamicSupervisor +============================================================================= From 97f5da5b39d0486f9f9ce0b564eb5299e1c313bc Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:10:50 -0300 Subject: [PATCH 12/52] docs: dynamic supervisor design revision 2 (decisions Q1-Q7, review resolutions) --- docs/design/dynamic-supervisor.md | 1040 +++++++++++++++++------------ 1 file changed, 617 insertions(+), 423 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index b6d10ff3..c0d286e4 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -1,7 +1,10 @@ # Dynamic supervisor: design contract -**Status: design in review, not implemented.** ROADMAP rank 12a. Draft PR -[#245](https://github.com/lambdaclass/spawned/pull/245). +**Status: design in review (revision 2), not implemented.** ROADMAP rank 12a. +Draft PR [#245](https://github.com/lambdaclass/spawned/pull/245). Revision 2 +applies the maintainer's decisions on the seven open questions of revision 1 +and resolves the first independent reviews (Codex and agy); see +[Decisions](#decisions) and [Review resolutions](#review-resolutions). - Covers OTP's `simple_one_for_one` and Elixir's `DynamicSupervisor`: children started at runtime from one template, supervised one for one, terminated @@ -25,14 +28,11 @@ - **T:** Rust types add strictness; or - **B:** Spawned adds a bound OTP lacks. -Where this design keeps an existing Spawned behaviour that OTP does not have -(backoff, concurrent restarts), it says so and why; those are listed -separately from new deviations. - ## Contents - [Sources](#sources) - [Goals and non-goals](#goals-and-non-goals) +- [Decisions](#decisions) - [API](#api) - [Semantics](#semantics) - [Erlang alignment](#erlang-alignment) @@ -41,7 +41,8 @@ separately from new deviations. - [Interaction with rank 12 (scoped registries)](#interaction-with-rank-12-scoped-registries) - [What the model establishes](#what-the-model-establishes) - [Alternatives considered](#alternatives-considered) -- [Open questions](#open-questions) +- [Follow-ups](#follow-ups) +- [Review resolutions](#review-resolutions) ## Sources @@ -59,7 +60,7 @@ What OTP and Elixir do, in short: | Topic | OTP `simple_one_for_one` | Elixir `DynamicSupervisor` | |---|---|---| -| `start_child` | `gen_server:call(Sup, …, infinity)`; the child's `init` runs inside the supervisor, so starts are serialised with each other and with `'EXIT'` handling ([L652](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L652), [L1077-1092](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1077-L1092)) | same; "will block the `DynamicSupervisor` until the child initializes" ([L359](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L359)) | +| `start_child` | `gen_server:call(Sup, …, infinity)`; the supervisor runs the start function, which normally waits for the child's `init` (proc_lib), so starts are serialised with each other and with `'EXIT'` handling ([L652](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L652), [L1077-1092](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1077-L1092)) | same; "will block the `DynamicSupervisor` until the child initializes" ([L359](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L359)) | | Arguments | `ExtraArgs` appended to the template's | `extra_arguments` (fixed per supervisor) prepended to each spec's args ([L752](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L752)) | | Specs | one template | any child spec per call | | Start result | `{ok, Pid}`, `{ok, Pid, Info}`, `{ok, undefined}` on `ignore` (nothing stored), `{error, Reason}` unwrapped | same, but `:ignore` bare | @@ -73,6 +74,7 @@ What OTP and Elixir do, in short: | Restart intensity | default 1 in 5 s; `add_restart` before every restart attempt, also each `try_again_restart` retry ([L1477-1531](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1477-L1531)) | default 3 in 5 s; each `$gen_restart` retry counts | | Exhaustion | logs `reached_max_restart_intensity`, terminates every child, exits `shutdown` | same | | `max_children` | none | `:infinity` by default; checked only in `start_child`, `map_size(children) < max_children`, which includes restarting entries; restarts never re-check ([L742](https://github.com/elixir-lang/elixir/blob/b57b70bd96e2e76d66cb81eba6b39c172fa8e8e4/lib/elixir/lib/dynamic_supervisor.ex#L742)) | +| `terminate_child` duration | blocks the supervisor until the child is gone: `shutdown` then `kill` after the timeout, or forever with `infinity` (the default for supervisor children) | same | | Shutdown | `terminate_dynamic_children`: monitor and `exit(P, shutdown)` to every child first, one timer for the template's shutdown value, then `kill` to every child still alive; order "is not defined" ([L1701-1799](https://github.com/erlang/otp/blob/186c0b94da565e4cd234685307b3c3ea41c4af1c/lib/stdlib/src/supervisor.erl#L1701-L1799)) | same, one timer per distinct shutdown value | | Static strategies, for contrast | children stopped one at a time in reverse start order, each waited for | | | Restarted supervisor | starts with no children | same | @@ -90,43 +92,61 @@ Goals: 1. Start supervised children at runtime, each from a template plus per-call arguments, with a synchronous `start_child` that returns the child or a typed error. -2. OTP's restart types, supervisor-wide restart budget and escalation. -3. Terminate one child without restarting it. -4. Optional `max_children` that rejects at once, counting restarting +2. OTP's operation order: one control loop serialises starts, restart + decisions and terminations, as the supervisor process does. +3. OTP's restart types, supervisor-wide restart budget and escalation. +4. Terminate one child without restarting it. +5. Optional `max_children` that rejects at once, counting restarting children. -5. Stop every child concurrently on shutdown, bounded by the per-child +6. Stop every child concurrently on shutdown, bounded by the per-child shutdown timeout and the kill, not by the number of children. -6. Placement on one `ExecutionGroup`; ownership by an `ActorSystem`; +7. One child that cannot finish its cleanup never takes healthy siblings + down unless OTP would also stop them. +8. Placement on one `ExecutionGroup`; ownership by an `ActorSystem`; nesting under a static supervisor. Non-goals (v1): -- Heterogeneous children (Elixir's any-spec `start_child`). See - [Alternatives](#alternatives-considered) and open question 2. +- Heterogeneous children (Elixir's any-spec `start_child`): decision Q2. - A keyed registry owned by the supervisor (rank 12; see [Interaction with rank 12](#interaction-with-rank-12-scoped-registries)). - Supervisor-type dynamic children (a template that builds a nested - supervisor per call). Possible later through the same `SupervisorSpec` - adapter the static supervisor uses. + supervisor per call). Possible later through the nested adapter. - `ignore` from a factory (the static supervisor does not have it either, S1). - `Task.Supervisor` (ranks 11 and 15), `PartitionSupervisor` (sharding), significant children and `auto_shutdown` (OTP rejects them for `simple_one_for_one`). - Draining children on shutdown (rank 13), bounded mailboxes (rank 17). +- `terminate_child` on static supervisors (decision Q6, a follow-up). + +## Decisions + +The maintainer settled revision 1's open questions under the match-Erlang +rule: + +| # | Question | Decision | +|---|---|---| +| Q1 | Concurrent or serialised construction | **Serialised, as OTP.** One control loop runs each start (admission, factory, `started()`) to completion before the next operation. Factories still run on the target group. Long work (ethrex's RLPx handshake) moves out of `started()` into the first message, OTP's `handle_continue` idiom | +| Q2 | Heterogeneous children | The typed template is enough for v1 | +| Q3 | Defaults | Elixir's: `max_restarts = 3`, `restart_window = 5 s`; backoff **opt-in** (`None` by default); `max_children = None` | +| Q4 | Group capacity across restarts | The slot holds a **transferable native-capacity reservation** from admission to removal, separate from any generation's scope lease | +| Q5 | Exit reason of an exhausted supervisor | `Shutdown`, as OTP, keeping the diagnostic in `failure()`. Static nested supervisors need the same change: a separate follow-up PR | +| Q6 | `terminate_child` on static supervisors | Separate follow-up | +| Q7 | Arguments on rejection | Returned on pre-admission rejection (`MaxChildren`, `Stopped`, placement); a distinct error for a start aborted after construction began | ## API ### Template ```rust -/// One template for every child: placement, factory, restart type and limits. +/// One template for every child: placement, factory, restart type, limits. /// The factory gets the per-call arguments (OTP's ExtraArgs). pub struct DynamicChildSpec { /* name, group, factory, restart, timeouts */ } -impl DynamicChildSpec { - /// Permanent, as OTP's default. Restartable children keep their - /// arguments for the next generation, hence `Clone`. +impl DynamicChildSpec { + /// Permanent, as OTP's default. A restartable child is rebuilt from its + /// arguments, hence `Clone`. pub fn new(name: impl Into, group: &ExecutionGroup, factory: F) -> Self where F: Fn(FactoryContext, Args) -> U + Send + Sync + 'static, @@ -138,7 +158,7 @@ impl DynamicChildSpec { impl DynamicChildSpec { /// A Temporary template. Arguments are moved into the only generation /// and never kept (OTP stores no arguments for temporary children), so - /// they need not be `Clone`: a `TcpStream` or a semaphore permit works. + /// they need not be `Clone`: a `TcpStream` works. pub fn temporary(name: impl Into, group: &ExecutionGroup, factory: F) -> Self; } @@ -148,50 +168,64 @@ impl DynamicChildSpec { } ``` -Two constructors instead of one is a Rust-forced shape (R): a restartable -child must be rebuilt from its arguments, so they must be cloneable, while a -temporary one must accept move-only values. `RestartMode::Temporary` passed -to `restart()` is still allowed (it simply never clones). The restart type is -per template, as in `simple_one_for_one`. +- Two constructors is a Rust-forced shape (R): a restartable child must be + rebuilt from its arguments, a temporary one must accept move-only values. + `RestartMode::Temporary` passed to `restart()` is allowed and never + clones. The restart type is per template, as in `simple_one_for_one`. +- `Args` needs `Send + 'static`, plus `Clone` for restartable templates; not + `Sync` (one owner holds the retained copy). +- `DynamicChildSpec`, `DynamicSupervisor`, `SupervisedDynamicSupervisor` and + `DynamicChild` implement `Clone` without requiring `A: Clone` or + `Args: Clone` (shared handles), so a supervisor of move-only arguments can + be stored in a cloneable context. ### Starting the supervisor ```rust pub struct DynamicSupervisorConfig { - pub max_restarts: usize, // default 5 - pub restart_window: Duration, // default 30 s - pub max_children: Option, // default None (Elixir :infinity) - pub backoff: Backoff, // default as SupervisorConfig + pub max_restarts: usize, // default 3 (Elixir) + pub restart_window: Duration, // default 5 s (Elixir) + pub max_children: Option, // default None (Elixir :infinity); Some(0) allowed + pub backoff: Option, // default None: restart at once, as OTP } -// Root, owned by the system (shutdown unit in admission order, S2): +// Root, owned by the system (a shutdown unit in admission order, S2): let peers: DynamicSupervisor = system.dynamic_supervisor(config, spec)?.start().await?; // Nested under a static supervisor, as `add_supervisor` (V8): let slot: SupervisedDynamicSupervisor = - builder.add_dynamic_supervisor(DynamicSupervisorSpec::new("peers", &group, config, || spec()))?; + builder.add_dynamic_supervisor(DynamicSupervisorSpec::new("peers", config, spec))?; let peers = slot.current()?; // re-resolve after the parent restarts it ``` -`start()` returns once the supervisor is admitted; it has no children to -start, so it cannot fail with `FailedToStartChild`. Configuration errors -(`max_children = Some(0)`, a zero window, `backoff.maximum < initial`, a -zero timeout, a foreign group) are `SupervisorError::Configuration` or -`System(ForeignGroup)` at construction, as for the static supervisor. The -`max_children <= 65536` ceiling of `SupervisorConfig` does not apply: the -static ceiling bounds a declaration list, while the dynamic count is the -user's explicit choice. +- `start()` returns once the supervisor is admitted; it has no children to + start, so it cannot fail with `FailedToStartChild`. +- Configuration errors (zero window, `backoff.maximum < initial`, zero + timeouts, a foreign group) are `SupervisorError::Configuration` or + `System(ForeignGroup)` at construction. `max_children = Some(0)` is valid + and rejects every start, as Elixir's `max_children: 0`. The static + supervisor's `max_children <= 65536` ceiling does not apply. +- `DynamicSupervisorSpec::new(name, config, spec)` takes the template; the + adapter runs on the template's group. Its `shutdown_timeout` (how long the + parent waits for the whole dynamic supervisor) defaults to + **2 × the child `shutdown_timeout` + 1 s**, so the parent never gives up + before the inner kill stage ran (agy 6). OTP defaults supervisor children + to `infinity` for the same reason; Spawned keeps a finite default so a + stuck subtree is still reported (B). It can be raised with + `.shutdown_timeout()`. ### Operations ```rust impl DynamicSupervisor { - /// OTP start_child(Sup, ExtraArgs). Resolves when the first generation - /// is ready (started() returned) or failed. - pub async fn start_child(&self, args: Args) -> Result, StartChildError>; - /// OTP terminate_child(Sup, Pid). Resolves after the child finished. + /// OTP start_child(Sup, ExtraArgs). Queued on the control loop; resolves + /// when the first generation is ready (started() returned) or failed. + pub async fn start_child(&self, args: Args) -> Result, StartChildError>; + /// OTP terminate_child(Sup, Pid): by the child's last published ActorId. pub async fn terminate_child(&self, id: ActorId) -> Result<(), TerminateChildError>; + /// The same by slot, for a child seen in which_children (any state). + pub async fn terminate_slot(&self, id: DynamicChildId) -> Result<(), TerminateChildError>; pub fn which_children(&self) -> Vec; pub fn count_children(&self) -> ChildCounts; pub fn stop(&self); @@ -205,45 +239,46 @@ pub struct Started { pub child: DynamicChild, // the slot, across restarts } -/// Like `SupervisedChild` for a dynamic slot. +/// Like `SupervisedChild` for a dynamic slot. Holds no arguments and no +/// scope; after removal it keeps only terminal metadata. impl DynamicChild { - pub fn id(&self) -> DynamicChildId; // stable for the slot + pub fn id(&self) -> DynamicChildId; pub fn current(&self) -> Result, ChildUnavailable>; pub async fn wait_for_generation(&self, after: u64) -> Result, ChildUnavailable>; pub fn snapshot(&self) -> ChildSnapshot; - /// terminate_child for whatever generation is current, including a - /// restart in backoff. - pub async fn terminate(&self) -> Result<(), TerminateChildError>; + pub async fn terminate(&self) -> Result<(), TerminateChildError>; // = terminate_slot } #[non_exhaustive] -pub enum StartChildError { - /// max_children reached (Elixir {:error, :max_children}). Nothing queued. - MaxChildren, - /// The supervisor is shutting down or has stopped (OTP: the caller's - /// call exits with the supervisor's reason, or noproc). - Stopped, +pub enum StartChildError { + /// Not admitted: nothing was constructed and the arguments come back. + Rejected { reason: RejectReason, args: Args }, + /// Admitted, then the supervisor began shutting down before readiness + /// committed. The arguments were consumed by the factory. + Aborted, /// The first generation failed: factory error or panic, panic or stop - /// in started(), startup_timeout. The child is not kept and nothing is - /// charged to the budget (OTP {error, Reason}). - FailedToStart { cause: String }, - /// Placement: the group is closed or at capacity. - System(SystemError), + /// in started(), or startup_timeout. Not retried, not charged. + FailedToStart { cause: String, exit: Option, cleanup_stuck: bool }, +} + +#[non_exhaustive] +pub enum RejectReason { + MaxChildren, // Elixir {:error, :max_children} + Stopped, // shutting down or stopped (OTP: the call exits) + Placement(SystemError),// the group is closed or out of native capacity } #[non_exhaustive] pub enum TerminateChildError { - /// Not a current child of this supervisor (Elixir {:error, :not_found}). - NotFound, - /// The supervisor is shutting down; it stops the child itself. - Stopped, - /// The child's cleanup missed its deadlines; the supervisor failed (V3). - CleanupStuck, + NotFound, // not a current child (Elixir {:error, :not_found}) + Stopped, // called after shutdown began: the supervisor stops it + CleanupStuck, // the child missed its deadlines; see "Stuck children" } pub struct DynamicChildInfo { pub child: DynamicChildId, - pub actor: Option, // None while restarting (OTP `restarting`) + pub actor: Option, // last published id; None before the first readiness + pub ready: bool, // false while restarting (OTP `restarting`) pub state: ChildState, pub generation: u64, } @@ -251,87 +286,96 @@ pub struct DynamicChildInfo { pub struct ChildCounts { pub specs: usize, // 1, as OTP simple_one_for_one pub active: usize, // children with a live, ready generation - pub supervisors: usize, // 0 in v1 (no supervisor-type children) - pub workers: usize, // every child, restarting ones included + pub supervisors: usize, // 0 in v1 + pub workers: usize, // every child held, restarting and stuck included } ``` -`ChildState` gains no variant: a dynamic child is `Starting`, `Running`, -`Backoff` (restart pending), `Stopping`, then gone from `which_children`. -`which_children` is a snapshot in no particular order, as OTP's. - -### Restart types and defaults +- `StartChildError` implements `Error` when `Args: Debug`; + `into_args()` returns the arguments of a `Rejected`, and `without_args()` + maps to `StartChildError<()>` for `?` in code that does not need them. +- `FailedToStart` carries the generation's `ExitReason` when an actor + existed (`Panic` from `started()`, `Normal` from `ctx.stop()` in + `started()`), so callers need not match strings (agy 9). +- `ChildState` gains no variant: a dynamic child is `Starting`, `Running`, + `Backoff` (restart pending), `Stopping` (including stuck), then leaves + `which_children`. `which_children` is a snapshot in no particular order. +- Removed slots are evicted from the supervisor's table at removal, after + their retained arguments are dropped: memory is O(children held), not + O(starts). + +### Restart types | | OTP | This design | |---|---|---| -| Default restart type | `permanent` | `Permanent` (`DynamicChildSpec::new`); `temporary()` for one-shot children such as connections | -| Restart table | `do_restart/3` | unchanged V1 table: Permanent always; Transient on an abnormal reason (`Panic`, `Kill`, `Killed`, `NoProc`) or a cleanup failure (exit-signals design); Temporary never | -| A replacement failing before readiness | `try_again_restart`, retried | retried unless Temporary (V1), each attempt charged | -| Explicit `terminate_child` | never restarted | never restarted, any type | +| Default | `permanent` | `Permanent` (`DynamicChildSpec::new`); `temporary()` for one-shot children | +| Table | `do_restart/3` | V1 with the v0.8 rule: Permanent always; Transient on an abnormal reason (`Panic`, `Kill`, `Killed`, `NoProc`) or a recorded cleanup failure; Temporary never | +| A replacement failing before readiness | `try_again_restart`, retried | retried unless Temporary, each attempt charged | +| `terminate_child` | never restarted | never restarted, any type | ### Restart budget -- **Scope:** one rolling window per supervisor, shared by all its children, - as OTP's intensity and period. +- **Scope:** one rolling window per supervisor, as OTP's intensity and + period. Defaults 3 per 5 s (Elixir). - **What counts:** every restart attempt: the factory call of every generation after a child's first, including retries of a replacement that - failed before readiness (OTP calls `add_restart` before each restart and - each `try_again_restart`). A dynamic supervisor has no collateral - restarts, so v0.7's "every factory call" and rank 5f's "recovery rounds" - give the same count here. -- **What does not count:** a failed `start_child` (OTP), a Temporary child's - exit, an exit after `terminate_child`, exits during shutdown. -- **Defaults:** `max_restarts = 5`, `restart_window = 30 s`, the static - supervisor's defaults. OTP uses 1 in 5 s and Elixir 3 in 5 s. Spawned's - is looser because per-child exponential backoff (V6) already spreads - restarts: a crash-looping child still uses up 5 grants in well under a - second (backoff bounds 10, 20, 40, 80, 160 ms), so the looser window does - not let a loop run long, while a supervisor of many children tolerates - more unrelated crashes. Open question 3. -- **Exhaustion:** the supervisor fails (`failure()` records - "restart budget exhausted" with the child and its last reason), closes - admission, stops every child as in a shutdown and escalates: a nested - dynamic supervisor exits abnormally and its parent applies its own policy - (V8); a root supervisor reports it in `ShutdownReport::supervisor_failures`. - OTP exits `shutdown` here; see [alignment](#erlang-alignment) row 15. -- **Reservation:** as V5, capacity is reserved before backoff and refunded if - the backoff is cancelled (by `terminate_child` or shutdown). An exhausted - window fails at once without waiting for the backoff. + failed before readiness (OTP's `add_restart` before each restart and each + `try_again_restart`). There are no collateral restarts, so v0.7's "every + factory call" and rank 5f's "recovery rounds" count the same here. +- **What does not count:** a failed `start_child`, a Temporary child's exit, + an exit after `terminate_child`, exits during shutdown. +- **Reservation protocol** (as V5, modelled here): when the loop decides a + restart, it reserves one grant; if none is left the supervisor fails at + once (no backoff is entered). The reservation is stamped into the window + when the restart attempt is admitted, and refunded if the pending restart + is cancelled by `terminate_child` or shutdown. Window expiry releases only + stamped grants, never reservations. +- **Backoff** is opt-in (`backoff: Some(..)`): without it the restart is + attempted as soon as the loop handles the exit, as OTP. With it, the + per-child jitter stream is seeded from the supervisor seed mixed with the + slot's `DynamicChildId`, not only the shared template name, so siblings do + not draw identical delays (agy 9). +- **Exhaustion:** the supervisor fails (`failure()` records "restart + intensity reached" with the child and its last reason, as OTP's + `reached_max_restart_intensity` report), closes admission, stops every + child as in a shutdown, and **exits `Shutdown`** (OTP): a nested dynamic + supervisor under a Transient parent is then not restarted; under a + Permanent parent it is (decision Q5). Stuck cleanups and coordinator + faults still exit abnormally. ### `max_children` -- `None` by default (Elixir `:infinity`). `Some(n)` caps the number of - children the supervisor holds: starting, running, cleaning up, in backoff - or constructing a replacement. A slot is freed only when its child is - removed: its last generation finished cleanup (milestone 7 of the exit - phases) and no restart follows. -- Checked only by `start_child`, atomically with admission. A full - supervisor returns `Err(MaxChildren)` at once; nothing waits or queues. -- Restarts are never re-checked (Elixir) and never need to be: the slot they - use was never freed. So a new start cannot take the place of a child - that is restarting, and a restart cannot fail for want of a slot. -- Spawned frees the slot after cleanup, not at the exit signal (OTP removes - the child when it processes the `'EXIT'`). B: the cap then bounds what is - really held (state, sockets, helpers), not only live processes. - -### Placement and worker budgets - -- Every child runs on the template's `ExecutionGroup`; factories run there - too, as for the static supervisor. The group must belong to the - supervisor's `ActorSystem` (`ForeignGroup` otherwise). -- Async groups: children share the group's workers; `max_children` is the - bound on how many there are. A dynamic supervisor does not change worker - budgets; the supervisor's own bookkeeping runs on the system's control - runtime, outside every group budget (as today). -- `thread_group(n)` (one thread per live actor, `Capacity` beyond `n`) and - pinned groups: a child's slot holds its group lease from admission until - removal, across restarts, so a restart is never refused for capacity that - a newer `start_child` took. A start that finds the group full returns - `Err(System(Capacity))` and takes no slot. Implementation note: this needs - a lease that outlives one actor instance; today a lease is per live scope. -- A closed group (system shutdown) makes `start_child` return - `Err(System(Closed))` and fails restarts, which fails the supervisor (as - static supervisors treat group closure). +- `None` by default. `Some(n)` caps the children held: starting, running, + cleaning up, in backoff, constructing a replacement, or stuck. A slot is + freed only when its child is removed: its last generation finished owned + cleanup (milestone 7 of the exit phases), its retained arguments were + dropped, and no restart follows. +- Checked only when the loop admits a start; a full supervisor answers + `Rejected { MaxChildren, args }` and nothing waits. Restarts are never + re-checked (Elixir) and never need to be: their slot was never freed. +- The slot frees after cleanup, not at the exit signal as OTP removes the + child at `'EXIT'` (B: the cap bounds what is really held). + +### Placement and capacity + +- Every child and every factory runs on the template's `ExecutionGroup`. + The group must belong to the supervisor's `ActorSystem`. +- Async groups: children share the group's workers; nothing is reserved. + The supervisor's control loop runs on the system's control runtime, + outside every group budget, as today. +- Native groups (`thread_group(n)`, one thread per live actor): each slot + takes a **transferable native-capacity reservation** at admission and + keeps it until removal. A generation's scope draws its thread from that + reservation and returns it when the generation's scope has joined; the + replacement draws it again. The reservation is a separate permit owned by + the slot, not the old scope's completion lease, so it never keeps a + finished scope from joining (decision Q4). A start that cannot reserve + answers `Rejected { Placement(Capacity), args }`. +- Pinned groups multiplex actors on a shard and have no per-actor native + capacity; the slot keeps its shard (affinity) across restarts and + reserves nothing. +- A closed group rejects starts (`Placement(Closed)`) and fails restarts, + which fails the supervisor, as static supervisors treat group closure. ### Nesting and the system @@ -339,200 +383,273 @@ pub struct ChildCounts { admission order, stopped in its turn by `ActorSystem::shutdown` (S2); its children stop concurrently in that turn. - Under a static supervisor: `add_dynamic_supervisor` uses the nested - adapter of `add_supervisor` (V8). The parent sees one child; it stops it - in reverse declaration order like any child; a restart by the parent - creates a new, empty dynamic supervisor (OTP: a restarted - `simple_one_for_one` supervisor starts with no children). Old - `DynamicSupervisor` handles keep pointing at the dead instance and return - `Stopped`; `SupervisedDynamicSupervisor::current()` resolves the new one. -- A dynamic supervisor's children are actors in v1. A child may itself start - a static supervisor in its factory only as unmanaged work, which is not - supported; nested supervisor templates are a follow-up. + adapter (V8), with its join barrier over every descendant. A restart by + the parent creates a new, empty dynamic supervisor (OTP). Old handles + return `Stopped`; `SupervisedDynamicSupervisor::current()` resolves the new + one. ## Semantics +### The control loop + +One logical control loop per supervisor handles, in arrival order and one at +a time, as OTP's supervisor process does: + +- a `start_child`: admission (gate, `max_children`, capacity), then the + first generation's factory and `started()`, until readiness commits or the + attempt failed and was cleaned up (or reached its cleanup deadline); +- a child's exit: the restart decision, the reservation, and (after the + optional backoff, which does not hold the loop) the restart attempt until + its readiness or failure; +- a `terminate_child`: signal, kill escalation and removal of that child. + +Not serialised: children run, exit and clean up concurrently; the backoff +timer runs off the loop; `which_children`, `count_children` and `current()` +read a snapshot without waiting for the loop; shutdown closes admission at +once and cancels the operation in progress (row 21 of the alignment table, +B). Factories run on the target group even though the loop awaits them. + +Consequences, all as in OTP: + +- A slow `started()` delays every later start, restart and termination of + that supervisor. Keep `started()` short and do slow initialisation in the + first message (OTP's `handle_continue`). `startup_timeout` bounds the + delay per start. +- With `max_children = 1`, a start that arrives while another is + constructing waits; if the first fails, the second is admitted (Codex 1). +- A `terminate_child` queued behind a start of the same child finds it. + ### `start_child` -1. **Admission** is one atomic step on the supervisor's gate: if the - supervisor is closed (shutdown, failure or exhaustion began), `Stopped`; - else if `max_children` is reached, `MaxChildren`; else the group lease is - reserved (`System(..)` if it cannot be) and a slot is created in - `Starting`, counted from this moment. -2. **Construction** runs the factory on the group and then `started()`, - bounded by `startup_timeout`, as a static child's first generation. - Concurrent `start_child` calls construct concurrently (see - [alignment](#erlang-alignment) row 2). -3. **Commit.** Readiness commits under the gate only while it is open. Then - `start_child` returns `Ok(Started)` and the child is supervised. -4. **First-start failure.** Factory error or panic, panic or stop in - `started()`, or `startup_timeout`: the generation is cleaned up, the slot - removed, `Err(FailedToStart { cause })` returned after that cleanup (S2's - "after rollback" rule, for one child). Never retried, never charged to the - budget, whatever the restart type (OTP returns `{error, Reason}`). -5. **Cancellation.** Dropping the `start_child` future does not cancel the - start (as R4: giving up never cancels). The child, once ready, is - supervised like any other and visible in `which_children`. OTP behaves the - same when the caller of `start_child` dies. +1. **Queued** on the loop. If the supervisor is already closed: + `Rejected { Stopped, args }` at once. +2. **Admission** when the loop reaches it: closed → `Rejected { Stopped, args }` + (also for a request still queued when shutdown begins); cap reached → + `Rejected { MaxChildren, args }`; native capacity → `Rejected { Placement, args }`. + Otherwise the slot is created (`Starting`), counted from now on. +3. **Construction**: arguments are moved (Temporary) or cloned (restartable; + the original is retained by the slot) inside the tracked attempt, on the + target group, outside framework locks. The factory runs, then + `started()`, bounded together by `startup_timeout`. A panicking `clone()` + is a failed attempt like a factory panic. +4. **Commit.** Readiness commits under the gate, only while it is open. The + commit is the success: from then on `start_child` returns + `Ok(Started)`, even if shutdown begins before the caller observes it + (success is historical, like `ActorRef::ready()` and `start()`'s + readiness, `supervision.rs` `start_owned`). The returned `actor` may + already be dead: if the child crashed between the commit and the caller's + wake-up, it is handled by the restart table and `child.current()` + resolves the replacement (Codex 2). +5. **First-start failure.** Factory error or panic, panic or stop in + `started()`, or `startup_timeout`: the generation is stopped and cleaned + up, the slot removed, and `FailedToStart { cause, exit, cleanup_stuck: false }` + returned. Never retried, never charged (OTP). If the cleanup misses its + deadlines (E3 rows 3-4), `FailedToStart { .., cleanup_stuck: true }` is + returned at the deadline; the slot stays counted and its work stays owned + by the system until it really ends, as S2 releases `start()`'s waiter + (Codex 4, agy 7). The supervisor is not failed: see + [Stuck children](#stuck-children). +6. **Shutdown during construction**: the attempt is cancelled (state dropped + without hooks if `started()` had not begun, L2; otherwise the parent exit + signal); `Aborted` is returned after its cleanup or deadline. The error + reflects what ended the attempt: a factory failure recorded before + shutdown stays `FailedToStart` (agy 7). +7. **Cancellation** of the `start_child` future does not cancel the start + (R4); a child that becomes ready is supervised and listed. OTP behaves the + same when the caller dies. ### Races | Race | Outcome | |---|---| -| `start_child` vs shutdown, admission first | the attempt is cancelled at shutdown: if `started()` had not begun, the state is dropped without hooks (L2); otherwise the generation gets the parent exit signal like a running child. `start_child` returns `Stopped`. It never returns `Ok` after shutdown began (model: `NoStartAfterShutdown`) | -| `start_child` vs shutdown, shutdown first | `Stopped` at admission; nothing is constructed | -| `start_child` vs a sibling's crash | the crashed child keeps its slot through cleanup, backoff and the replacement, so at the cap the new start gets `MaxChildren` (model: `MaxChildrenBound`, control `UncountedRestart`) | -| `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup; a start before that sees it counted | +| `start_child` vs shutdown, while queued | `Rejected { Stopped, args }` | +| `start_child` vs shutdown, during construction | `Aborted`; never `Ok` unless readiness committed first (model: `NoStartAfterShutdown`) | +| `start_child` vs shutdown, after the commit | `Ok`; the child is then stopped with the others | +| `start_child` vs a sibling's crash | the crashed child keeps its slot through cleanup, backoff and the replacement, so at the cap the new start gets `MaxChildren` (`MaxChildrenBound`, control `UncountedRestart`) | +| `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup and argument drop | | `start_child` vs budget exhaustion | exhaustion closes the gate; as shutdown | -| `terminate_child` vs a crash of that child | whichever is first under the slot lock wins. Crash first: if a restart is pending, `terminate_child` cancels it (refunding the budget) and removes the child; if the replacement is constructing, it is cancelled. Terminate first: the crash reason is recorded, no restart. Either way the child is removed and never restarted (model: `TerminatedNeverRestarted`) | -| `terminate_child` vs shutdown | after shutdown began, `Stopped`: the supervisor is already stopping every child | -| `terminate_child(id)` with an old generation's id | `NotFound` once that generation is not the slot's current one (a newer generation started); during backoff the last generation's id still names the slot. `DynamicChild::terminate()` targets the slot whatever its generation | +| `terminate_child` vs a crash of that child | serialised by the loop. Exit handled first: if a restart is pending, `terminate_child` cancels it (refunding the reservation) and removes the child; if the restart attempt is running, `terminate_child` waits behind it and then terminates the new generation. Terminate first: the crash is recorded, no restart. Either way the child is removed and never restarted (`TerminatedNeverRestarted`, `NoPublishAfterStop`) | +| `terminate_child` vs shutdown | a call queued or in progress when shutdown begins resolves when the child is removed (`Ok`, or `CleanupStuck`); a call made after shutdown began returns `Stopped` | +| two `terminate_child` calls for one child | the second waits for the same removal and gets the same result; a call after removal gets `NotFound` | -### Restart identity +### Restart identity and termination keys - Every generation has a new `ActorId`, mailbox and state (M4). The slot's - `DynamicChildId` does not change; `DynamicChild::current()` returns the - latest ready generation, `wait_for_generation(n)` waits for a newer one - (V4). Old `ActorRef`s stay bound to their instance and get `NoProc` (v0.8). -- `which_children` shows the current `ActorId`, or `None` while restarting - (OTP's `restarting`). -- Callers that look children up by a domain key (a peer id) keep their own - map from key to `DynamicChild` until rank 12 provides keyed registration; - see [rank 12](#interaction-with-rank-12-scoped-registries). + `DynamicChildId` never changes. `current()` returns the latest ready + generation; `wait_for_generation(n)` waits for a newer one (V4). +- The **termination key** of a slot is its **last published** `ActorId`. + It is kept through cleanup, backoff and failed replacement attempts, and + switches only when a replacement's readiness commits, as OTP keeps the + original pid as the `restarting` entry's key and Elixir keeps + `current_pid` (Codex 3, agy 5). A call with an older id after a + successful replacement gets `NotFound`; only a slot-based call + (`terminate_slot`, `DynamicChild::terminate`) is guaranteed to reach the + child whatever its generation. +- `which_children` reports that key and whether the child is ready, so a + child seen restarting can be terminated by key or by slot. +- Callers that find children by a domain key keep their own map until rank + 12 ([below](#interaction-with-rank-12-scoped-registries)). ### Terminating one child -`terminate_child` is OTP's `shutdown/2` for one child, under the exit-signal -design's E3 row for a supervisor stopping a child: +`terminate_child` is OTP's `shutdown/2` for one child, run by the loop: -1. Mark the slot terminated (no restart from now on), cancel a pending - backoff or replacement. +1. Mark the slot terminated (no restart from now on); cancel a pending + restart and refund its reservation. 2. Send the parent exit signal `Shutdown`: a non-trapping child begins exit after its current handler and skips `stopped()` (E2, decision 2); a trapping child handles the messages queued before the signal, then runs `stopped()` (E3). 3. If the child has not reached its death point `shutdown_timeout` after the - signal, kill it (`Killed`, E1); if it misses a second `shutdown_timeout` - the cleanup is stuck: the supervisor fails (V3) and `terminate_child` - returns `CleanupStuck`. -4. Resolve after the child's cleanup finished and the slot was removed. + signal, kill it (`Killed`, E1). If it misses a second `shutdown_timeout`, + or its owned cleanup misses its deadline, the child is stuck: + `terminate_child` returns `CleanupStuck`. +4. Otherwise resolve `Ok` once the child is removed (`TermOkMeansRemoved`). + +The loop is held for the whole call, as OTP's supervisor is blocked in +`terminate_child`; it is bounded by 2 × `shutdown_timeout` plus the cleanup +deadline. + +### Stuck children + +OTP's `kill` always ends a process, so an OTP supervisor never waits on a +child that ignores `shutdown` longer than the shutdown timeout (except with +`infinity`). Spawned's kill is cooperative (R): a child can stay in a +handler, in `stopped()` or in owned cleanup past every deadline. A child is +**stuck** when, after its stop signal, it misses the kill and the second +interval, or when its owned cleanup misses its deadline (`shutdown_timeout` +from exit begin for a child that ended on its own, E3 row 4: one interval, +no kill). Policy (agy P0): + +| The stuck child | Effect | +|---|---| +| was retiring anyway (Temporary, Transient after a normal exit, removed by `terminate_child`, or a failed first start) while the supervisor is open | **only that child is affected.** Its slot stays counted (it still holds memory, sockets, capacity) and its work stays owned until it really ends, then the slot is removed. The failure is recorded in its `ChildSnapshot::last_error` and the supervisor's report; a waiting `terminate_child` gets `CleanupStuck`, a waiting `start_child` gets `FailedToStart { cleanup_stuck: true }`. The supervisor and its siblings are unaffected, as OTP's supervisor carries on after killing a child | +| would be restarted (Permanent, Transient after an abnormal exit or with the cleanup failure the stall implies) | the supervisor **fails** and exits abnormally: it cannot start the replacement while the old generation lives (V3), and OTP would have restarted it; escalation lets the parent decide (V8) | +| is being stopped by shutdown | the shutdown report is incomplete and records the failure (V9, S3); "done" waits for it | + +Model: `StuckIsolated`, control `StuckFailsAll`. A retiring stuck child is +also never restarted later; its slot simply ends when the work ends. ### Shutdown of all children When the supervisor closes (`stop`, `shutdown(deadline)`, system shutdown, -its parent stopping it, or a failure): - -1. Admission closes: `start_child` returns `Stopped`; pending restarts and - constructions are cancelled (budget refunded); starts in construction are - cancelled as in [Races](#races). -2. **Every live child gets the parent exit signal at once**, not in reverse - order: dynamic children have no declared order (OTP: "not defined") and - sequential stops would make shutdown take up to N × `shutdown_timeout`. - Model: `ConcurrentShutdown`, control `Sequential`. -3. Each child then follows the E3 deadlines from that one moment: kill after - `shutdown_timeout`, stuck after a second one. As every child shares the - template's timeout, this is OTP's single timer. Shutdown therefore takes - at most 2 × `shutdown_timeout` plus cleanup, whatever N is. +its parent, or a failure): + +1. Admission closes: queued starts get `Rejected { Stopped, args }`, the + construction in progress is cancelled (`Aborted`), pending restarts are + cancelled and refunded. +2. **Fan-out:** the supervisor sends every live child the parent exit + signal, one after another without awaiting any child's completion, then + waits. Children have no order (OTP: "not defined"). Model: + `ConcurrentShutdown`, control `Sequential`. +3. **Deadlines:** each child's intervals are measured from its own signal + (E3): kill after `shutdown_timeout`, stuck after a second one. OTP arms + one timer after the fan-out; measuring from each child's own signal + differs only by the fan-out time and never shortens a child's interval + (Codex 9). Shutdown therefore takes at most 2 × `shutdown_timeout` plus + the fan-out time plus owned cleanup up to its deadline, whatever N is. 4. **Done** means every slot was removed: every generation reached - publication and its owned cleanup finished (V9 "done"). Model: - `NoChildOutlivesSupervisor`, `ClosedLeadsToDone`. + publication and its owned cleanup finished, its arguments dropped (V9; + `NoChildOutlivesSupervisor`, `ClosedLeadsToDone`). 5. `shutdown(deadline)` keeps the static meaning: `deadline` is the report - deadline; at it the report is returned with `complete = false` if - children remain, and they stay owned by the system (S3). + deadline; children still running then stay owned by the system (S3). + +The model takes the fan-out as one step: the implementation sends the +signals in a loop with no await between them, and a child acting before its +own signal only ends earlier. Trace validation of the implementation will +check this abstraction (signal events precede any wait event). + +### Retained arguments + +- Restartable slots keep the original `Args`; each attempt receives a clone + made inside the attempt (on the target group, outside framework locks, + within `startup_timeout`). A panicking or hanging clone is a failed + attempt (Codex 10). +- At removal the slot drops its retained arguments as part of its cleanup, + before its native-capacity reservation and its `max_children` slot are + released. A panicking destructor is recorded as a cleanup failure (after + the child's death it changes no exit reason); a hanging one makes the slot + stuck, under the policy above. +- `DynamicChild` handles hold no arguments and no scopes; after removal they + report terminal metadata only. ### In-flight requests and deferred replies of a dying child -Nothing is specific to dynamic supervision; the child dies as any actor -(exit-signals design, phases of an exit, with rank 11): +As for any actor (exit-signals design, phases of an exit, with rank 11): - At the child's death point D every queued request and every open `ReplyTo` token is answered `ActorStopped(final reason)`: `Shutdown` after a clean stop, `Killed` after the kill, the crash reason after a crash. A request or registration after D gets `NoProc`. -- A non-trapping child stopped by its supervisor skips `stopped()`, so - deferred tokens it stored are answered at D, not by `stopped()` code. - Cleanup that must always run belongs in `Drop` or needs `trap_exit` (the - exit-signals migration note). +- A non-trapping child stopped by its supervisor (`terminate_child` or + shutdown) skips `stopped()`, so cleanup that must always run belongs in + `Drop`, in an owner that monitors the child, or needs `trap_exit` (and + even then not after a kill). - Pipe jobs of the child are cancelled at exit begin and joined after - publication (rank 11 step 4), within the cleanup deadline; the slot is - freed only after that join. + publication (rank 11 step 4) within the cleanup deadline; the slot frees + only after that join. - A replacement never receives the old generation's queue, tokens or jobs (V4; rank 11 `ActorId` fencing). ### Links and monitors -- The supervisor is the parent of every child generation (E3), so its - `Shutdown` is a parent exit signal. It is not an actor: it observes - children through internal completion state, not links or a mailbox (V9), - so a child's backlog never delays its restart decision and a child cannot - unlink itself from supervision (OTP guards against such "naughty" children - with monitors; here there is nothing to unlink). -- Children can link to and monitor each other and outside actors as usual; - supervision does not add or remove those edges. A link between two - children propagates a crash under X3 and both are then handled by the - restart table independently (one for one). -- Outside actors that track children should monitor them - (`ctx.monitor(&started.actor.child_handle())`) rather than rely on the - child's `stopped()`: the `Down` arrives exactly once with the final reason, - also after a kill, a skipped `stopped()` or a runtime drop. This is what - replaces ethrex's self-reported removal ([client mapping](dynamic-supervisor-client-mapping.md)). - -### Budget exhaustion takes everything down - -Exhaustion is a supervisor failure: it closes admission and stops every -child exactly as a shutdown, then escalates (V8). Nothing restarts after the -failure (model: `ExhaustionCloses`, `NoGenerationAfterClose`, control -`NoEscalate`). A Temporary child never consumes budget, so a supervisor of -Temporary children (ethrex's case) fails only on a stuck cleanup or a closed -group. +- The supervisor is the parent of every child generation (E3). It is not an + actor: it observes children through internal completion state, not links + or a mailbox (V9), so a child's backlog never delays its exit handling and + a child cannot unlink itself from supervision. (OTP links to children and + uses a monitor only while shutting one down, so that a child that called + `unlink(Sup)` cannot hang the supervisor; Codex 14.) +- Children can link to and monitor each other and outside actors as usual. +- Owners that track children should monitor them; the `Down` arrives exactly + once with the final reason, also after a kill or a skipped `stopped()`. ## Erlang alignment | # | Rule | OTP / Elixir | This design | Reason for any deviation | |---|---|---|---|---| -| 1 | `start_child` is synchronous and returns the child | `{ok, Pid}` after `init` | `Ok(Started { actor, child })` after `started()` | aligned; `child` is an addition (row 9) | -| 2 | Starts are serialised in the supervisor | the supervisor runs each child's `init` and blocks meanwhile | admission is serialised (atomic count and gate); construction runs concurrently, bounded by `startup_timeout` | kept Spawned behaviour, not R/T/B: the supervisor is not a process, and static supervisors already restart children concurrently (v0.7). Observable difference is timing only: a slow `started()` does not delay other starts or restart decisions. Open question 1 | -| 3 | Template plus per-call args | `ExtraArgs` appended (Elixir: fixed `extra_arguments` prepended) | typed `Args` passed to the factory | T: one typed argument instead of a term list | -| 4 | `ignore` | `{ok, undefined}`, nothing stored | not supported | as the static supervisor (S1); a factory cannot return it | -| 5 | Start failure | `{error, Reason}`, not counted, not retried | `FailedToStart { cause }`, not counted, not retried, returned after cleanup | aligned (S2 ordering) | +| 1 | `start_child` is synchronous and returns the child | `{ok, Pid}` after the start function returns | `Ok(Started { actor, child })` after `started()` | aligned; `child` is an addition (row 9) | +| 2 | Operations serialised by the supervisor | one process: starts, `'EXIT'` handling, restarts and `terminate_child` run one at a time | one control loop, same order and blocking | aligned (decision Q1) | +| 3 | Template plus per-call args | `ExtraArgs` appended (Elixir: fixed `extra_arguments` prepended) | typed `Args` passed to the factory | T | +| 4 | `ignore` | `{ok, undefined}`, nothing stored | not supported | as the static supervisor (S1) | +| 5 | Start failure | `{error, Reason}`, not counted, not retried | `FailedToStart`, not counted, not retried, after cleanup or at its deadline | aligned; the deadline release is R (S2) | | 6 | Default restart type | `permanent` | `Permanent` | aligned | | 7 | Temporary children keep no args | `mapsets` | `temporary()` moves args once, no `Clone` | aligned; R for the separate constructor | -| 8 | `terminate_child` | by pid; unknown dead pid `ok` (OTP) or `not_found` (Elixir) | by `ActorId`; `NotFound` for anything not current | Elixir's answer: OTP's `ok` for a dead unknown pid is an artefact of `find_child` | -| 9 | Restart identity | new pid, found through names | new `ActorId`; stable `DynamicChildId` and `DynamicChild::current()` | addition, consistent with `SupervisedChild` (references are instance-bound, an accepted decision); no OTP behaviour changes | +| 8 | `terminate_child` key | pid, kept through failed restarts | last published `ActorId`, kept through failed restarts; `NotFound` (Elixir) for anything else | aligned with Elixir's `not_found`; OTP's `ok` for a dead unknown pid is a `find_child` artefact | +| 9 | Restart identity | new pid, found through names | new `ActorId`; stable `DynamicChildId`, `current()`, `terminate_slot` | addition, consistent with `SupervisedChild` (instance-bound references) | | 10 | `restart_child`, `delete_child` | `{error, simple_one_for_one}` | not provided | aligned (absent at compile time: T) | -| 11 | `which_children` | pid or `restarting`, unordered | `Option`, unordered, with state and generation | aligned plus fields | -| 12 | `count_children` | `specs` 1, `active` excludes restarting | same | aligned with OTP (Elixir's `specs` differs) | -| 13 | `max_children` | Elixir: default infinity, start only, counts restarting | same; frees after cleanup | aligned with Elixir; B for freeing after cleanup | -| 14 | Restart budget | intensity/period, every restart attempt counts, start failures do not | same rule; defaults 5 in 30 s | defaults: kept Spawned behaviour (backoff, V6); open question 3 | -| 15 | Exhaustion | terminate children, supervisor exits `shutdown` | terminate children concurrently, fail, escalate; the nested adapter exits abnormally (as static nested supervisors today) | existing static-supervisor difference, not introduced here: a `Transient` OTP supervisor that exhausts its budget is not restarted by its parent, a Spawned one is. Open question 5 | -| 16 | Shutdown order | parallel, undefined order, one timer, then `kill` | parallel, per-child `shutdown_timeout` from one moment, then `kill`, then stuck | aligned; the stuck stage is R (Rust cannot kill a thread; exit-signals design) | -| 17 | Shutdown signal | `exit(Child, shutdown)`; trapping child runs `terminate` | parent exit signal: trapping child runs `stopped()` after its backlog; non-trapping skips `stopped()` | aligned (E2, E3) | -| 18 | `start_child` during shutdown | call queued, caller exits with the supervisor's reason or `noproc` | `Err(Stopped)` | R: Rust has no caller exit; a typed error is the counterpart | +| 11 | `which_children` | pid or `restarting`, unordered | last published id plus `ready`, unordered | aligned plus fields | +| 12 | `count_children` | `specs` 1, `active` excludes restarting | same | aligned with OTP | +| 13 | `max_children` | Elixir: default infinity, `0` allowed, start only, counts restarting | same; frees after cleanup | aligned; B for freeing after cleanup | +| 14 | Restart budget | every restart attempt counts; Elixir default 3 in 5 s | same rule and defaults | aligned (decision Q3) | +| 15 | Exhaustion | terminate children, exit `shutdown` | terminate children concurrently, exit `Shutdown`, diagnostic in `failure()` | aligned (decision Q5); static nested supervisors follow in a separate PR | +| 16 | Shutdown order | parallel, undefined, one timer after the fan-out, then `kill` | parallel, per-child intervals from each child's signal, `kill`, then stuck | aligned up to the fan-out time; the stuck stage is R | +| 17 | Shutdown signal | `exit(Child, shutdown)`; a trapping child runs `terminate` | parent exit signal; trapping child runs `stopped()` after its backlog; non-trapping skips it | aligned (E2, E3) | +| 18 | `start_child` during shutdown | the queued call exits with the supervisor's reason or `noproc` | `Rejected { Stopped, args }` or `Aborted` | R: no caller exit in Rust; T: the arguments come back | | 19 | Restarted dynamic supervisor | empty | empty | aligned | -| 20 | Backoff | none: restart at once | per-child backoff (V6) | kept Spawned behaviour; budget refunded if cancelled | -| 21 | Shutdown admission | a stopping supervisor's children may start anything | the whole tree closes admission at once | B (exit-signals design) | -| 22 | Cleanup before a slot frees or a replacement starts | the process is gone at `'EXIT'` | after owned cleanup (V3) | R: destructors and owned helpers outlive the last hook | -| 23 | Restarted-child calls in flight | `'EXIT'`/monitor tells callers | `ActorStopped(reason)` at D, `NoProc` after (rank 11, v0.8) | aligned | -| 24 | Heterogeneous specs | Elixir: any spec per call | one typed template | T: `start_child` returns `ActorRef`; open question 2 | +| 20 | Backoff | none | none by default; opt-in per supervisor | aligned by default (decision Q3) | +| 21 | Shutdown admission | the supervisor processes its queue in order; a start queued behind the shutdown is never handled | admission closes at once, the construction in progress is cancelled | B (exit-signals design) | +| 22 | When a slot frees | at `'EXIT'` | after owned cleanup and the argument drop (V3) | R and B: destructors and owned helpers outlive the last hook | +| 23 | A child that ignores `shutdown` | killed; the kill always works | killed cooperatively; if it still does not end it is stuck, isolated unless it would restart | R: Rust cannot kill a thread; policy in [Stuck children](#stuck-children) | +| 24 | Calls in flight to a dying child | `'EXIT'`/monitor tells callers | `ActorStopped(reason)` at D, `NoProc` after | aligned (rank 11, v0.8) | +| 25 | Heterogeneous specs | Elixir: any spec per call | one typed template | T; decision Q2 | +| 26 | Nested supervisor's shutdown wait | `infinity` by default for supervisor children | 2 × child `shutdown_timeout` + 1 s by default | B: a stuck subtree is still reported | ## Non-guarantees -- **No order** among dynamic children, at start or shutdown, and none in - `which_children`. +- **No order** among dynamic children, at start (beyond arrival order on the + loop) or shutdown, and none in `which_children`. - **No queueing** at `max_children`: a full supervisor rejects. Waiting for a slot is rank 17's admission. +- **No parallel construction:** one slow `started()` delays the supervisor's + other operations (OTP's behaviour). Use the first message for slow work. - **No exactly-once start under cancellation:** a dropped `start_child` future may still produce a supervised child. -- **No state or queue carried across a restart** (V4); nothing replays - messages sent to an old generation. +- **No state or queue carried across a restart** (V4). - **No transparent reference:** `Started::actor` is the first generation - only. -- **No preemption:** a child stuck in a handler or `stopped()` is killed - cooperatively; a child whose cleanup never ends fails the supervisor and - keeps the shutdown report incomplete (S3), as today. -- **No fairness between `start_child` callers** beyond the atomic admission - step; under contention for the last slot any one may win. -- **`which_children`/`count_children` are snapshots**; they can be stale by - the time they are read (OTP's "likely to be very accurate"). + only and may already be dead when observed. +- **No preemption:** a stuck child keeps its slot and its resources until + its code returns; Spawned only reports it. +- **`which_children`/`count_children` are snapshots.** - **No bounded memory per child** (rank 18a) or per mailbox (rank 17). ## Semantics contract: what this adds and touches @@ -541,204 +658,281 @@ New guarantees (section 6 of [semantics.md](semantics.md), ids proposed): | Id | Guarantee | Checked by (planned) | |---|---|---| -| V10 | **Dynamic admission.** `start_child` is admitted atomically with the gate and the count; it never returns `Ok` after shutdown began; a full supervisor rejects with `MaxChildren`; a first-start failure returns `FailedToStart` after cleanup, uncharged and unretried. | `DynamicSupervisor!NoStartAfterShutdown`, `!MaxChildrenBound`, `!OkMeansReady`, `!StartAnswered` (liveness); tests | -| V11 | **Cap counts restarting children.** A child holds its slot from admission until removal, through cleanup, backoff and replacement. | `DynamicSupervisor!MaxChildrenBound` | -| V12 | **Terminated means removed.** After `terminate_child` a child is never restarted, whatever its type; a pending restart is cancelled and refunded. | `DynamicSupervisor!TerminatedNeverRestarted`, `!TerminateCompletes` | -| V13 | **Concurrent shutdown.** Every live child gets its stop signal at once; the supervisor is done only after every child finished. | `DynamicSupervisor!ConcurrentShutdown`, `!NoChildOutlivesSupervisor`, `!ClosedLeadsToDone` | +| V10 | **Dynamic admission.** One control loop serialises starts, exit handling and terminations. A start is admitted atomically with the gate and the count; success commits only while admission is open and is then historical; a rejected start returns its arguments; a first-start failure is returned after cleanup or at its deadline, uncharged and unretried. | `DynamicSupervisor!SerialConstruction`, `!NoStartAfterShutdown`, `!OkMeansReady`, `!FirstStartNeverRetried`, `!FailedAfterCleanup`, `!FailureCauseKept`, `!StartAnswered`; tests | +| V11 | **Cap counts restarting children.** A child holds its slot from admission until removal, through cleanup, backoff, replacement and a stuck cleanup. | `!MaxChildrenBound` | +| V12 | **Terminated means removed.** After `terminate_child` a child is never restarted or republished; a pending restart is cancelled and refunded; `Ok` means removed. | `!TerminatedNeverRestarted`, `!NoPublishAfterStop`, `!ReservationsMatch`, `!TermOkMeansRemoved`, `!TerminateAnswered` | +| V13 | **Concurrent shutdown.** Every live child is signalled without awaiting another; the supervisor is done only after every child finished. | `!ConcurrentShutdown`, `!NoChildOutlivesSupervisor`, `!ClosedLeadsToDone` | +| V14 | **Stuck isolation.** A child stuck while retiring fails only itself; the supervisor fails only for a stuck child it would restart, budget exhaustion, or during shutdown. | `!StuckIsolated` | Touched: -- **V1** (restart table): applies per child; the Temporary rule also means no - retained arguments. `DynamicSupervisor!TemporaryNeverRestarted`. -- **V3, V4** (one live generation, publication): per slot. - `DynamicSupervisor!AtMostOneGeneration`, `!PublishedIsLive`. -- **V5** (budget): supervisor-wide, start failures excluded; exhaustion - stops all children concurrently instead of in reverse order. - `DynamicSupervisor!BudgetBounded`, `!ExhaustionCloses`. +- **V1** (restart table) per child, including the cleanup-failure rule: + `!RestartFollowsPolicy` (both directions: no forbidden restart, no + retirement of a restartable child while open), `!TemporaryNeverRestarted`, + `!RecoveryProgress`. +- **V3, V4** per slot: `!AtMostOneGeneration`, `!PublishedIsLive` (also + requiring readiness). +- **V5** (budget): supervisor-wide, start failures excluded, reservations + modelled: `!ChargedRestarts`, `!ReservationsMatch`, `!BudgetBounded`, + `!ExhaustionCloses`. Exhaustion stops all children concurrently and exits + `Shutdown`. - **V8** (escalation): a dynamic supervisor is a nestable subtree. -- **V9** (shutdown): concurrent instead of reverse declaration order for - dynamic children; "done" unchanged. `!NoGenerationAfterClose`. -- **S1, S2** (admission, ordered shutdown): the dynamic supervisor is a - shutdown unit; its own admission closes with the tree. -- **§10** gains the non-guarantees above; **§11** gains rows 2, 14, 15, 20 - of the alignment table. +- **V9** (shutdown): concurrent instead of reverse order for dynamic + children; `!NoGenerationAfterClose`. +- **S1, S2**: the dynamic supervisor is a shutdown unit; its admission + closes with the tree. +- **§10** gains the non-guarantees above; **§11** gains rows 16, 21, 22, 23 + and 26. No guarantee in sections 1-5 changes. ## Interaction with rank 12 (scoped registries) -**Decision: registry-owned lookup is deferred to rank 12.** Rank 12a ships -`start_child` returning a handle and nothing keyed. Reasons: +**Registry-owned lookup is deferred to rank 12.** Rank 12a ships +`start_child` returning a handle and nothing keyed: -- The two are independent: a dynamic supervisor is useful without keys (a - pool of workers, one actor per accepted socket counted by `max_children`), - and a keyed registry is useful without supervision (rank 12's own client - evidence is a peer map whose entries need generation guards). -- What a key means on conflict is domain policy. ethrex keeps the first - connection and rejects a duplicate ("Peer already connected, don't - replace it"); another user may want replace-and-stop-old. That belongs in - the registry's design, not in the supervisor's. -- Folding keys into 12a would make the dynamic supervisor depend on rank 12's - scoping (system-scoped instead of the process-global registry) before it - is designed. +- The two are independent: a dynamic supervisor is useful without keys (one + actor per accepted socket counted by `max_children`), and a keyed registry + is useful without supervision. +- Conflict policy for a key is domain policy (ethrex keeps the first + connection; another user may replace-and-stop), which belongs in the + registry's design. +- Keys in 12a would make it depend on rank 12's scoping before it exists. -Until rank 12, the pattern is the application map plus a monitor: +Until rank 12, the pattern depends on the restart type (Codex 13): + +**Temporary children** (one generation): the owner's map plus a monitor of +that generation, guarded by the monitor reference. ```rust -// In the owner (a peer table actor): let started = peers.start_child(args).await?; let mref = ctx.monitor(&started.actor.child_handle()); self.by_key.insert(key, Entry { child: started.child, monitor: mref }); -// Handler: remove the entry only if it still holds that monitor. +// Handler: remove the entry only if it still holds this monitor. +fn on_down(&mut self, down: Down) { + if let Some(key) = self.by_monitor.remove(&down.monitor_ref) { + if self.by_key.get(&key).is_some_and(|e| e.monitor == down.monitor_ref) { + self.by_key.remove(&key); + } + } +} ``` -The `Down`'s `MonitorRef` (or the `ActorId`, never reused, M4) is the -generation guard that ethrex's `ConnectionId` hand-builds, and the `Down` is -delivered exactly once whatever ends the child (X1), unlike a message sent -from `stopped()`. +**Restartable children:** the entry belongs to the slot, not to a +generation. Keep the `DynamicChild` in the map until the slot is removed +(learned from `terminate_child`'s result, or from `child.snapshot()` being +terminal), and resolve the current generation with `child.current()` on +each use; or let each generation register itself (`ctx.register_name` in +`started()`, generation-owned, S5). A monitor of one generation only says +that generation died, not that the slot is gone. -**How they compose once rank 12 lands:** +**Composition once rank 12 lands:** - A keyed start is `start_child` plus a generation-owned registration made - by the child (in `started()`, as `ctx.register_name` already does, S5) or - by the supervisor on readiness. The registration is released when that + by the child or by the supervisor on readiness, released when that generation's cleanup finishes, so an old cleanup can never remove a replacement (S5, the ethrex #7435 bug). -- For restartable children, each generation re-registers under the same key - with a new generation; lookups between the crash and the replacement's - readiness find nothing (instance-bound references, V4). -- Duplicate policy (reject or replace) is the registry's; `max_children` and - the budget stay the supervisor's. -- The rank 12 model (`an old entry's cleanup never removes its replacement`) - composes with `DynamicSupervisor.tla` through the slot's generations. +- Restartable children re-register per generation; lookups between a crash + and the replacement's readiness find nothing (V4). +- Duplicate policy is the registry's; `max_children` and the budget stay the + supervisor's. ## What the model establishes [`DynamicSupervisor.tla`](../../verification/tla/DynamicSupervisor.tla) -models one dynamic supervisor and N `start_child` calls, each made at most -once at any moment, so starts race crashes, restarts, `terminate_child`, -`max_children`, budget exhaustion and shutdown. Restart types are per slot so -one instance covers all three. Time is abstract: the window is a counter -released at most `MaxExpire` times, which also bounds the generations (no -state constraint); the kill escalation is the nondeterministic reason -(`Shutdown` or `Killed`) of a stopped child; a cleanup can get stuck -(`CleanupStuck`, not fair), which fails the supervisor. - -Invariants: +(revision 2) models one dynamic supervisor, N `start_child` calls made at any +moment, and one `terminate_child` per started child. A control loop (`op`) +serialises admission and first construction, exit handling and restart +attempts, and terminations. Restart types are per slot so one instance +covers all three. Budget reservations, stamping, refunds and window expiry +are explicit. Exits come from children (`Normal`, `Panic`), stop signals +(`Shutdown`, or `Killed` after the kill), start failures (factory error, +`Normal` before readiness), and cleanup may record a failure (the Transient +rule). A child signalled but still running, or with stuck cleanup, can +become stuck, with the policy above. + +Invariants that name an event (a commit, a publication or a generation +after close or terminate; a forbidden restart or retirement; restart +attempts) read history variables that `GhostUpdate` computes from every +step's state change, separately from the actions, so a mutated action cannot +hide its own violation (agy 3). `alive` accumulates generations and only the +one whose cleanup ended is removed (Codex 7). | Invariant | Meaning | |---|---| | `AtMostOneGeneration` | a slot never has two live generations | -| `PublishedIsLive` | `current()` returns only the live, latest generation | +| `PublishedIsLive` | `current()` returns only the live, latest, ready generation | | `NoChildOutlivesSupervisor` | done means every child finished and was removed | -| `MaxChildrenBound` | children held (restarting and cleaning up included) never exceed `max_children` | +| `MaxChildrenBound` | children held never exceed `max_children` | +| `SerialConstruction` | at most one generation is being constructed | | `TemporaryNeverRestarted` | a Temporary child has one generation | -| `TerminatedNeverRestarted` | no generation starts after `terminate_child` | -| `NoStartAfterShutdown` | no `start_child` returns `Ok` after shutdown began | -| `NoGenerationAfterClose` | no generation (start or restart) begins after admission closed | -| `ExhaustionCloses` | budget exhaustion fails and closes the supervisor | -| `BudgetBounded` | grants in the window stay within `max_restarts` | +| `TerminatedNeverRestarted` | no generation begins after `terminate_child` | +| `NoGenerationAfterClose` | no generation begins after admission closed | +| `NoPublishAfterStop` | no generation is published after close or `terminate_child` | +| `NoStartAfterShutdown` | no `start_child` success commits after admission closed | +| `RestartFollowsPolicy` | every restart is allowed by the table, and no restartable exit is retired while open | +| `FirstStartNeverRetried` | a failed first start is never retried | +| `FailedAfterCleanup` | `FailedToStart`/`Aborted` only after cleanup or at the stuck deadline | +| `FailureCauseKept` | the start error reflects what ended the attempt | +| `ChargedRestarts` | every restart attempt is charged exactly once; first starts never | +| `ReservationsMatch` | reservations are exactly the pending restarts | +| `BudgetBounded` | stamped grants plus reservations stay within `max_restarts` | +| `ExhaustionCloses` | exhaustion fails and closes the supervisor | +| `StuckIsolated` | the supervisor fails only by exhaustion, a stuck child it would restart, or while closing | | `ConcurrentShutdown` | once stopping, every running child has its stop signal | | `OkMeansReady` | `Ok` only for a child that became ready | +| `TermOkMeansRemoved` | `terminate_child` returns `Ok` only after removal | -Action property `NeverReopens`. Liveness (fair construction, stop, cleanup -and supervisor steps; crashes and stuck cleanup not fair): -`ClosedLeadsToDone`, `StartAnswered`, `TerminateCompletes`. +Action property `NeverReopens`. Liveness (fair loop, construction, stops, +cleanup and stuck children ending; crashes, start failures and getting stuck +not fair): `ClosedLeadsToDone`, `StartAnswered`, `TerminateAnswered`, +`RecoveryProgress` (a restartable exit is restarted or superseded by +terminate, shutdown, exhaustion or a stuck cleanup). TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (every PR; safety, action property, liveness) | 3 slots (Permanent, Transient, Temporary), `max_children` 2, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Panic` | 1,601,979 (5,010,572 generated) | 33 | 2m16s | -| `MCDynamicSupervisorDeep` (weekly; safety, action property) | the same 3 slots and cap with budget 2 and two window releases (up to four restarts); one reason per class, since the decisions depend only on normal versus abnormal. Four slots did not fit: 222M distinct states after 30 min without finishing | 5,617,958 (18,853,746 generated) | 42 | 41 s | +RESULT_ROWS -Negative controls (`negative_checks.py`; each checks `TypeOK` and only the -named property, on `MCDynamicSupervisor`): +Negative controls (`negative_checks.py`, run one at a time; each checks +`TypeOK` and only the named property, on `MCDynamicSupervisor`): -| `Fault` | Mutation | Fails | Result | +| `Fault` | Mutation | Fails | Origin | |---|---|---|---| -| `IgnoreMax` | admission ignores the cap | `MaxChildrenBound` | counterexample | -| `UncountedRestart` | a crashed child frees its slot until it restarts (counts only starting and running children) | `MaxChildrenBound` | counterexample | -| `RestartTemporary` | Temporary children follow the Permanent rule | `TemporaryNeverRestarted` | counterexample | -| `RestartTerminated` | `terminate_child` does not cancel a restart | `TerminatedNeverRestarted` | counterexample | -| `AcceptDuringShutdown` | `start_child` admitted and committed after close | `NoStartAfterShutdown` | counterexample | -| `RestartAfterClose` | a pending restart proceeds after close | `NoGenerationAfterClose` | counterexample | -| `NoEscalate` | budget exhaustion drops only that child | `ExhaustionCloses` | counterexample | -| `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | counterexample | -| `Sequential` | children stopped one at a time (static order) | `ConcurrentShutdown` | counterexample | -| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` (liveness) | counterexample | -| `StuckBackoff` | shutdown does not cancel a pending restart | `ClosedLeadsToDone` (liveness) | counterexample | - -All eleven produce their counterexample in under 5 s each. - -Not modelled: handler bodies, mailboxes and requests (the child's own death -is `Lifecycle.tla`'s; replies are rank 11's models); real time (backoff -delays, the window's clock); group capacity and leases; nesting (a failed -dynamic supervisor is one abnormal child of `NestedSupervision.tla`'s -parent). Trace validation of real runs against this model, as -`TraceSupervision.tla` does for OneForOne, is part of the implementation -(verification rule). Apalache inductive invariants are follow-up work: the -model needs type annotations and an `IndInv` with helper lemmas like -`SupervisionInd.tla`. +| `IgnoreMax` | admission ignores the cap | `MaxChildrenBound` | rev. 1 | +| `UncountedRestart` | a crashed child frees its slot until it restarts | `MaxChildrenBound` | rev. 1 | +| `RestartTemporary` | Temporary children follow the Permanent rule | `TemporaryNeverRestarted` | rev. 1 | +| `RestartTerminated` | `terminate_child` does not cancel a restart | `TerminatedNeverRestarted` | rev. 1 | +| `AcceptDuringShutdown` | a start admitted and committed after close | `NoStartAfterShutdown` | rev. 1 | +| `RestartAfterClose` | a pending restart proceeds after close | `NoGenerationAfterClose` | rev. 1 | +| `NoEscalate` | exhaustion drops only that child | `ExhaustionCloses` | rev. 1 | +| `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | rev. 1 | +| `Sequential` | children stopped one at a time | `ConcurrentShutdown` | rev. 1 | +| `ConcurrentStart` | constructions not serialised | `SerialConstruction` | Codex 1 | +| `StuckFailsAll` | any stuck child fails the supervisor | `StuckIsolated` | agy P0 | +| `PublishAfterClose` | a constructing replacement publishes after close | `NoPublishAfterStop` | agy 3 | +| `OverlapGeneration` | the replacement starts before the old cleanup ended | `AtMostOneGeneration` | Codex 7 | +| `PublishBeforeReady` | a replacement is published at construction | `PublishedIsLive` | Codex 7 | +| `PrematureOk` | `Ok` committed at admission | `OkMeansReady` | Codex 7 | +| `UnchargedRetry` | a failed replacement's retry is not charged | `ChargedRestarts` | Codex 7 | +| `ChargeFirstStart` | a first start is charged | `ChargedRestarts` | agy 2 | +| `TransientNormalRestart` | Transient restarts after a normal exit | `RestartFollowsPolicy` | agy 2, Codex 8 | +| `PermanentRetires` | Permanent is not restarted | `RestartFollowsPolicy` | Codex 8 | +| `RetryFirstStart` | a failed first start is retried | `FirstStartNeverRetried` | agy 2 | +| `EarlyFailedReply` | `FailedToStart` returned before cleanup | `FailedAfterCleanup` | agy 2 | +| `LateFailedOverwrite` | a pre-shutdown failure reported as `Aborted` | `FailureCauseKept` | agy 7 | +| `NoRefund` | a cancelled restart keeps its reservation | `ReservationsMatch` | agy 4, Codex 6 | +| `ExpireReserved` | window expiry releases a reservation | `ReservationsMatch` | Codex 6 | +| `TermReplyEarly` | `terminate_child` returns before removal | `TermOkMeansRemoved` | agy 7 | +| `NoAbort` (liveness) | a start racing shutdown is not cancelled | `StartAnswered` | rev. 1 | +| `StuckBackoff` (liveness) | shutdown does not cancel a pending restart | `ClosedLeadsToDone` | rev. 1 | + +CONTROLS_NOTE + +Not modelled: handler bodies, mailboxes and requests (`Lifecycle.tla`; +rank 11's models); real time (backoff delays, the window clock, deadlines +as durations); native-capacity reservations (a counting resource like the +cap, checked by tests); nesting (a failed dynamic supervisor is one child of +`NestedSupervision.tla`'s parent); the fan-out as separate steps (see +[Shutdown](#shutdown-of-all-children)). Trace validation of real runs is part +of the implementation (verification rule). Apalache inductive invariants +are a follow-up. ## Alternatives considered -1. **Elixir-style heterogeneous children as the primary API** - (`start_child(ChildSpec)` on an untyped supervisor). Rejected for v1: - every call site would carry its own spec and restart type, which the one - client does not need, and a typed template gives `ActorRef` directly. - It can be added later as a separate untyped handle without changing this - contract (the model already allows per-slot restart types). Open - question 2. -2. **Serialise starts like OTP.** Rejected (open question 1): one slow - handshake in `started()` would block every other start and every restart - decision, and static supervisors already restart concurrently. The OTP - idiom for this is `handle_continue`, which Spawned users get by keeping - `started()` short and doing the handshake in a handler. -3. **Make `max_children` wait instead of reject.** Rejected: Elixir rejects, - and waiting is rank 17's admission with its own deadlines and credits. -4. **Free the slot at the exit signal, as OTP removes the child at - `'EXIT'`.** Rejected: owned cleanup (helpers, destructors) still holds - resources, and the cap is meant to bound them (B). -5. **Stop children in reverse start order, as the static supervisor.** - Rejected: OTP does not order them, and N sequential stops multiply the - shutdown time (control `Sequential`). -6. **Terminate by slot only (no `ActorId` variant).** Rejected: OTP and - Elixir terminate by pid, and a stale id must not hit a newer generation - silently; the slot variant (`DynamicChild::terminate`) is offered too. -7. **Return only the `ActorRef` from `start_child`.** Rejected: restartable - children need a way to find the next generation without a registry - (rank 12); the `DynamicChild` handle is that way, as `SupervisedChild` is - for static children. -8. **Put keyed registration in 12a.** Rejected; see - [rank 12](#interaction-with-rank-12-scoped-registries). -9. **Add `terminate_child` to static supervisors in the same change.** The - ROADMAP research note asks for it (today an individually stopped - Permanent child restarts). It is the same mechanism (mark removed, parent - exit, kill, remove) but changes the static API and `Supervision.tla`; - proposed as a follow-up, open question 6. - -## Open questions - -1. **Concurrent construction** (alignment row 2). Keep concurrent starts - (proposed), or serialise factory plus `started()` per supervisor as OTP - does? -2. **Heterogeneous children.** Is the typed template enough for v1, or - should an untyped `start_child(ChildSpec)` ship with it? -3. **Default budget and cap.** Keep the static defaults (5 per 30 s, - `max_children = None`), or use Elixir's 3 per 5 s? Should the dynamic - supervisor default to a finite `max_children` (B), given that every - other Spawned bound is opt-in today? -4. **Group leases across restarts.** Is holding a thread-group lease from - admission to removal acceptable, or should `max_children` simply be - required to fit the group's capacity? -5. **Exit reason of an exhausted supervisor.** OTP exits `shutdown` (so a - Transient parent does not restart it); Spawned's nested adapter exits - with a panic reason. Align both static and dynamic supervisors with OTP - (a separate change), or keep it? -6. **`terminate_child` on static supervisors.** Add it in the same release - (ROADMAP research note), or later? -7. **Rejected arguments.** `MaxChildren`, `Stopped` and `System` errors drop - the arguments (ethrex's `TcpStream` closes without an RLPx Disconnect, as - today). Should these errors hand the arguments back (`Err((error, args))`), - or should a `try_reserve()` permit precede `start_child`? +1. **Concurrent construction with atomic admission** (revision 1). + Rejected (Q1, Codex 1): it changes outcomes, not only timing (with a cap + of one, a start arriving during a failing start is rejected instead of + admitted), and OTP's own idiom for slow initialisation is to leave it out + of `init`. +2. **Heterogeneous children as the primary API.** Rejected for v1 (Q2); can + be added as a separate untyped handle without changing this contract. +3. **`max_children` that waits.** Rejected: Elixir rejects; waiting is rank + 17's admission. +4. **Free the slot at the exit signal.** Rejected: owned cleanup still holds + resources (B). +5. **Stop children in reverse start order.** Rejected: OTP does not order + them, and sequential stops multiply shutdown time. +6. **Fail the supervisor on any stuck child** (revision 1, as static V3). + Rejected (agy P0): a retiring child needs no replacement, so V3 does not + force it, and OTP's supervisor carries on after killing such a child. +7. **Terminate by slot only.** Rejected: OTP and Elixir terminate by pid; + both forms are offered. +8. **Keyed registration in 12a.** Rejected; see rank 12 above. +9. **Hold the old scope's capacity lease across a restart** (one reading of + revision 1's Q4). Rejected (Codex): it would keep the finished scope from + joining; the reservation is a separate permit owned by the slot. +10. **Return the arguments from every error.** Not possible after the + factory consumed them; only pre-admission rejections return them. +11. **A `try_reserve()` permit before `start_child`.** Not needed for the + ethrex migration once rejections return the arguments; it adds permit + lifetime and cancellation semantics. Later if a user needs it. + +## Follow-ups + +- **Static nested supervisors exit `Shutdown` on budget exhaustion** (Q5): + separate PR, changing `nested.rs`'s failure path; `NestedSupervision.tla` + and semantics V8 accordingly. The nested adapter's default + `shutdown_timeout` collision (agy 6) applies to static nested supervisors + too and belongs in that PR. +- **`terminate_child` on static supervisors** (Q6): separate design (OTP + keeps the spec, `restart_child`, `delete_child`; `Supervision.tla` and + `Cohorts.tla` changes). +- **Apalache inductive invariants** for `DynamicSupervisor.tla`. +- **Implementation acceptance:** trace validation, the fan-out abstraction + check, evidence that dynamic tables, retained arguments and capacity + reservations add nothing to the ordinary actor and message paths (Codex + closing note). + +## Review resolutions + +Round 1: Codex (1 P0, 12 P1, 1 P2) and agy (1 P0, 7 P1, 1 P2), both "approve +after the listed changes", run without the author's answers. Each finding +was checked against the code and sources before acting on it. + +### Codex + +| # | Finding | Resolution | +|---|---|---| +| 1 (P0) | Concurrent construction changes outcomes (cap of one, a failing start), not only timing; not Rust-forced. Also: OTP's supervisor runs the start function, not `init` | **Accepted** (decision Q1). [The control loop](#the-control-loop) serialises starts, exit handling and terminations; model `SerialConstruction`, control `ConcurrentStart`. Sources row corrected | +| 2 | "Never `Ok` after shutdown" conflates commit with observation; `start()` keeps readiness historical | **Accepted.** Success commits only while open and is then historical, possibly with a dead first-generation reference ([`start_child`](#start_child) step 4). The model separates the commit (`Ready`) from anything the caller does later; `NoStartAfterShutdown` is about the commit | +| 3 | Termination identity must survive failed replacement attempts | **Accepted.** The termination key is the last published `ActorId`, kept until a replacement's readiness commits, as OTP and Elixir ([Restart identity](#restart-identity-and-termination-keys)); duplicates and stale ids specified in [Races](#races) | +| 4 | A failed start whose cleanup never joins leaves `start_child` unanswered | **Accepted.** `FailedToStart { cleanup_stuck: true }` at the deadline, slot still counted, work still owned (S2's rule). Model: `StuckDone` is fair only for liveness; `StartAnswered` holds because the stuck step answers | +| 5 | Exhaustion must exit `Shutdown`, not abnormally | **Accepted** (decision Q5): exits `Shutdown` with the diagnostic in `failure()`; static nested supervisors in a follow-up PR | +| 6 | The model omits reservation, stamping and refund | **Accepted.** Explicit `reserved`, `stamped`, `windowed`; reservation at the decision with immediate exhaustion; refund on cancel; expiry only of stamped grants. `ReservationsMatch`, `ChargedRestarts`, `BudgetBounded` (no longer implied by `TypeOK`); controls `NoRefund`, `ExpireReserved`, `UnchargedRetry`, `ChargeFirstStart`, and the per-PR `MCDynamicSupervisorZero` (budget 0) | +| 7 | Invariants too weak: `alive` overwritten, `BudgetBounded` implied by `TypeOK`, `PublishedIsLive` ignores readiness | **Accepted.** `alive` accumulates and loses only the cleaned generation; restart attempts are counted by a history variable; publication requires readiness. Controls `OverlapGeneration`, `UnchargedRetry`, `PublishBeforeReady`, `PrematureOk`. The point that controls prove sensitivity only for their mutation stands; every named invariant now has at least one control | +| 8 | Restart table and progress unverified; configs miss normal pre-readiness replacement failures and the cleanup-failure rule | **Accepted.** `RestartFollowsPolicy` in both directions, `RecoveryProgress`; `StartFails` includes `Normal`, `CleanupFails = {FALSE, TRUE}`, a zero-budget configuration. Controls `TransientNormalRestart`, `PermanentRetires`. "Fence asynchronous events by attempt identity": the model's per-slot generation plays that role; the implementation keeps the existing generation fencing | +| 9 | Concurrent shutdown needs a fan-out and deadline contract; stuck detection before D | **Accepted in the doc, kept abstract in the model.** Fan-out without awaits, each child's intervals from its own signal, timing including the fan-out ([Shutdown](#shutdown-of-all-children)); stuck detection before D (signalled but still running) and after D (cleanup) are both `CleanupStuck` in the model. The one-step fan-out is a documented abstraction, to be checked by trace validation | +| 10 | Retained arguments: clone can panic or block, drop can panic or hang | **Accepted.** [Retained arguments](#retained-arguments); `Sync` dropped | +| 11 | ethrex: the raw `tokio::spawn` writer (`server.rs:1542`) and framework listeners/timers are not covered; "no helpers" is wrong | **Accepted, confirmed in the code.** The client mapping now migrates the writer into an owned helper with a pending-frame policy and states that until then `max_children` bounds slots, not sockets | +| 12 | Monitoring fixes peer-table removal but not `teardown()`'s mempool and announcement cleanup when `stopped()` is skipped | **Accepted, confirmed** (`teardown()` clears in-flight transaction hashes). The mapping assigns each resource an owner that releases it on the `Down`, guarded by the connection's identity | +| 13 | The map-plus-monitor example only works for Temporary children | **Accepted.** Labelled Temporary-only, with the restartable pattern ([rank 12](#interaction-with-rank-12-scoped-registries)) | +| 14 (P2) | Elixir allows `max_children = 0`; pinned groups have no per-actor capacity; OTP's monitors are only for shutdown; termination by slot | **Accepted.** `Some(0)` allowed; pinned groups keep affinity only; links text corrected; `terminate_slot` added | + +### agy + +| # | Finding | Resolution | +|---|---|---| +| 1 (P0) | One stuck child (even retiring) fails the whole supervisor; a root ethrex supervisor would be bricked. The ethrex text also misstated self-exit timing (one interval, no kill) | **Accepted.** [Stuck children](#stuck-children): a retiring stuck child fails only itself; the supervisor fails only for a stuck child it would restart (V3) or during shutdown. `StuckIsolated`, control `StuckFailsAll`. The mapping's timing statement is corrected and recommends nesting the connection supervisors under the p2p supervisor anyway | +| 2 | No `RestartFollowsPolicy`; first-start rules unchecked | **Accepted.** `RestartFollowsPolicy`, `FirstStartNeverRetried`, `FailedAfterCleanup`, `ChargedRestarts` (covers "uncharged"), with controls. agy's `FailedAfterCleanup` required `alive = {}`; it is `removed` or `stuck`, because of Codex 4 | +| 3 | A constructing replacement can publish after `TerminateChild` or `Close` undetected; `lateOk` only covered first starts and was set by the guarded action | **Accepted in part.** History variables are computed outside the actions (`GhostUpdate`) for commits, publications and generations; `NoPublishAfterStop`, control `PublishAfterClose`. The `TerminateChild` half no longer arises: with the serialised loop, `terminate_child` cannot run while a replacement is being constructed (it waits behind it), which `SerialConstruction` and the loop's ownership enforce | +| 4 | Budget charged after backoff in the model; refund unchecked | **Accepted**, as Codex 6 | +| 5 | `terminate_child` target resolution: contradictions during replacement construction and `Starting`; no way to terminate from `which_children` | **Accepted.** Last-published key; serialisation answers the `Starting` case (a terminate queued behind a start finds the child); `terminate_slot`; `DynamicChildInfo` reports the key and readiness | +| 6 | The nested adapter's 5 s wait collides with the inner 2 × `shutdown_timeout`; redundant constructor arguments | **Accepted.** `DynamicSupervisorSpec::new(name, config, spec)`; default wait 2 × child `shutdown_timeout` + 1 s. The same collision in static nested supervisors goes to the follow-up PR | +| 7 | Stuck first-start cleanup must release `start_child`; a pre-shutdown failure must not be reported as shutdown; in-flight and duplicate terminates | **Accepted.** As Codex 4; `cause` recorded when the attempt ends (`FailureCauseKept`, control `LateFailedOverwrite`); terminate resolution modelled (`TermOkMeansRemoved`, `TerminateAnswered`, control `TermReplyEarly`) | +| 8 | ethrex `Handler` lacks the monitor check on the entry; awaiting `start_child` stalls accepts; `terminate_child` also skips `stopped()` | **Accepted.** The guard is in both sketches; the handshake moves out of `started()` (required now that starts are serialised); the `stopped()` note covers `terminate_child` | +| 9 (P2) | Jitter seed per slot; drop `Sync`; unconditional `Clone`, slot eviction; structured `FailedToStart` | **Accepted**, all four | + +### Open questions for round 2 + +1. **Holding the loop during `terminate_child`** (OTP blocks the supervisor + up to 2 × `shutdown_timeout`): faithful, but a slow-to-stop child delays + every start meanwhile. Keep, or let the loop continue while the + termination completes? +2. **Stuck children that would restart** fail the supervisor (V3). An + alternative is to retire such a slot instead (no restart, no failure). + Failing matches the static supervisor and escalates; retiring would + silently weaken Permanent. +3. **The one-step fan-out abstraction** in the model: acceptable with trace + validation, or should the model interleave signals with child steps? From a821497f89cd314accaf4eda676b98079ae9f861 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:11:41 -0300 Subject: [PATCH 13/52] docs: client mapping revision 2 (handshake out of started, owned writer, teardown owners, monitor guard) --- .../dynamic-supervisor-client-mapping.md | 157 ++++++++++-------- 1 file changed, 85 insertions(+), 72 deletions(-) diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index cfac87c1..c5d43073 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -150,44 +150,56 @@ discovery. In supervision terms every connection is **Temporary**. ## Mapping +Revision 2 of the design changes three things for ethrex: the supervisor +serialises starts like OTP (so the RLPx handshake must leave `started()`), +rejections return the arguments, and a connection whose cleanup gets stuck +fails only itself. + | ethrex today | With the dynamic supervisor | |---|---| | `.start()`, unmanaged, on the caller's runtime | `start_child(args)` on a `DynamicSupervisor`, placed on the `p2p` group | -| no owner; shutdown does not stop connections | the supervisor is a shutdown unit of the existing `ActorSystem`; admitted after the peer table, so in reverse admission order connections stop before the table, all at once, each killed after its `shutdown_timeout` | -| `Semaphore(150)` plus `_admission_permit` in actor state | `max_children = Some(150)` on an inbound supervisor; `Err(MaxChildren)` instead of `try_acquire_owned()` failing. The slot is freed after the actor's cleanup, as the permit is freed when the state drops | -| outbound uncapped | a second supervisor for dials without `max_children` (or one supervisor if a shared cap is wanted; that would change behaviour) | -| `ConnectionId` (process-global counter) | the connection's `ActorId` (never reused, M4) or the table's `MonitorRef` | -| `stopped()` sends `remove_peer(node_id, connection_id)` | the peer table monitors the connection when it registers it; `Handler` removes the entry whose monitor matches. `Down` arrives exactly once whatever ends the actor (X1), including kills and skipped `stopped()` | -| `handle_remove_peer` guard | the monitor match in `Handler` | +| handshake inside `started()` (up to `HANDSHAKE_TIMEOUT`, 10 s) | **required change:** `started()` sends itself a `Handshake` message and returns; the handshake runs as the first handler turn, still under `HANDSHAKE_TIMEOUT`, and a failure ends the actor (a Temporary exit, not a start failure). Otherwise one silent peer would hold the supervisor's control loop, and with it every accept and dial, for the whole `startup_timeout` | +| no owner; shutdown does not stop connections | the supervisors are owned by the system, nested under the p2p supervisor (recommended) or as roots admitted after the peer table; connections stop before the table, all at once, each killed after its `shutdown_timeout` | +| `Semaphore(150)` plus `_admission_permit` in actor state | `max_children = Some(150)` on an inbound supervisor; `Rejected { MaxChildren, args }` returns the `ConnArgs`, so the accept loop can still send an RLPx Disconnect or just drop the stream, as today | +| outbound uncapped | a second supervisor for dials without `max_children` | +| `ConnectionId` (process-global counter) | the table's `MonitorRef` (or the connection's `ActorId`, never reused, M4) | +| `stopped()` sends `remove_peer(node_id, connection_id)` | the peer table monitors the connection when it registers it; `Handler` removes the entry only if it still holds that monitor. `Down` arrives exactly once whatever ends the actor (X1), including kills and a skipped `stopped()` | | timed-out registration undone by an unconditional `remove_peer` | the table's monitor fires for that registration too; a monitor installed after the connection died gets `Down { NoProc }` at once (E4) | | connections never restarted | `DynamicChildSpec::temporary(..)`: no restarts, no budget use, args (the `TcpStream`, the `Node`) moved without `Clone` | -| first-wins on duplicates (`new_connected_peer` returns `false`) | unchanged: domain policy in the table (rank 12 may later own it) | -| `teardown()` mempool cleanup, `tx_broadcaster.remove_peer` | unchanged code, but see the note on `stopped()` below | +| first-wins on duplicates | unchanged: domain policy in the table (rank 12 may later own it) | +| `teardown()` and `tx_broadcaster.remove_peer` in `stopped()` | moved to owners that act on the `Down` (below): `stopped()` is skipped for a non-trapping connection on `terminate_child`, on shutdown and after a kill | +| raw `tokio::spawn` outbound writer holding the socket sink ([server.rs:1530-1557](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L1530-L1557)) | **required change:** an owned helper (`ctx.spawn_helper`), cancelled at exit and joined within the cleanup deadline, with an explicit policy for frames still queued (drop them; the peer is being disconnected). Until then the writer can outlive the actor and keep the socket open after the supervisor freed the slot, so `max_children` would bound slots, not sockets | ### After (sketch) ```rust -// Wiring (initializers.rs), after the peer table so connections stop first. -let inbound = system - .dynamic_supervisor( - DynamicSupervisorConfig { max_children: Some(MAX_INBOUND_CONNECTIONS), ..Default::default() }, - DynamicChildSpec::temporary("rlpx-inbound", &p2p_group, |_ctx, args: ConnArgs| async move { - Ok::<_, Infallible>(PeerConnectionServer::new(args)) - }) - .shutdown_timeout(Duration::from_secs(2)), - )? - .start() - .await?; -let outbound = system - .dynamic_supervisor(DynamicSupervisorConfig::default(), /* same template */)? - .start() - .await?; +// Wiring (initializers.rs): nested under the p2p supervisor, so a failure +// of the connection supervisor is restarted instead of disabling accepts. +let inbound = p2p_tree.add_dynamic_supervisor(DynamicSupervisorSpec::new( + "rlpx-inbound", + DynamicSupervisorConfig { max_children: Some(MAX_INBOUND_CONNECTIONS), ..Default::default() }, + DynamicChildSpec::temporary("rlpx-inbound", &p2p_group, |_ctx, args: ConnArgs| async move { + Ok::<_, Infallible>(PeerConnectionServer::new(args)) + }) + .shutdown_timeout(Duration::from_secs(2)), +))?; +// `outbound` likewise, without max_children. // Accept loop (network.rs): the semaphore and the permit plumbing go away. -match context.inbound.start_child(ConnArgs::receiver(context.clone(), peer_addr, stream)).await { +// start_child resolves as soon as started() returns, which is now immediate. +match inbound.current()?.start_child(ConnArgs::receiver(context.clone(), peer_addr, stream)).await { Ok(_) => {} - Err(StartChildError::MaxChildren) => debug!(peer = %peer_addr, "Inbound connection cap reached"), - Err(e) => debug!(peer = %peer_addr, "Inbound connection not started: {e}"), + Err(StartChildError::Rejected { reason: RejectReason::MaxChildren, args }) => { + debug!(peer = %peer_addr, "Inbound connection cap reached"); + drop(args); // closes the stream, as today + } + Err(e) => debug!(peer = %peer_addr, "Inbound connection not started: {}", e.without_args()), +} + +// Connection actor (server.rs): the handshake leaves started(). +#[started] +async fn started(&mut self, ctx: &Context) { + let _ = ctx.send(Handshake); // first turn, OTP's handle_continue } // Peer table (peer_table.rs): registration installs the monitor... @@ -199,31 +211,33 @@ async fn handle_new_connected_peer(&mut self, msg: NewConnectedPeer, ctx: &Conte self.peers.insert(id, PeerData::new(/* … */).with_monitor(monitor)); true } -// ...and removal is the Down, matched by monitor, not reported by the peer. +// ...and removal is the Down, guarded by the entry's own monitor (agy 8): impl Handler for PeerTableServer { async fn handle(&mut self, down: Down, _ctx: &Context) { - if let Some(id) = self.by_monitor.remove(&down.monitor_ref) { + let Some(id) = self.by_monitor.remove(&down.monitor_ref) else { return }; + if self.peers.get(&id).is_some_and(|p| p.monitor == down.monitor_ref) { self.peers.swap_remove(&id); self.sessions.remove(&id); + self.on_connection_gone(id, down.monitor_ref); // see "Skipped teardown" } } } ``` -`start_child` resolves when the connection's `started()` returns. ethrex's -`started()` runs the whole RLPx handshake today, under its own -`HANDSHAKE_TIMEOUT`, and calls `ctx.stop()` when it fails -([server.rs:537-620](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L537-L620)). -Under the dynamic supervisor that is a first-start failure: `start_child` -returns `FailedToStart`, the slot is freed, nothing is restarted or charged, -which is the behaviour ethrex wants. Two consequences: the accept loop would -wait for the handshake, and `startup_timeout` (default 5 s) must exceed -`HANDSHAKE_TIMEOUT` (10 s) or the supervisor cuts handshakes short. Either the -accept loop spawns the `start_child` future (it does not need the result), -or the handshake moves from `started()` into a first self-sent message, the -Spawned counterpart of OTP's `handle_continue`. The second is better: -`startup_timeout` then bounds only construction, and a slow peer never holds -a start; a failed handshake then becomes an ordinary Temporary exit. +### Skipped teardown: who releases what + +`stopped()` does not run for a non-trapping connection that is terminated, +shut down or killed (v0.8, decision 2). `trap_exit` would bring it back for +`terminate_child` and shutdown, but not for a kill, so each resource the +hook releases needs an owner that also hears about the death (Codex 12): + +| Released in `stopped()` today | Owner after the migration | +|---|---| +| peer-table entry and discv5 session (`remove_peer`) | the peer table's `Down` handler, guarded by its monitor | +| tx-broadcaster index (`tx_broadcaster.remove_peer`) | forwarded by the peer table from the same `Down`, carrying the node id and the monitor reference so a stale forward cannot drop a newer connection's index; or the broadcaster monitors connections itself | +| in-flight transaction reservations and re-requests to alternates (`teardown()`, [server.rs:449-511](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L449-L511)) | a guard value in the actor state whose `Drop` clears the hashes in the mempool (runs on every exit, kills included, before the slot frees); the re-request to alternates, which is async, becomes a message to the peer table sent from that `Drop`, guarded by the connection identity | +| announcement and availability bookkeeping | same guard | +| disconnect metrics | the `Down` handler (it has the final reason) | ### What disappears @@ -233,47 +247,46 @@ About 45-60 of the ~135 hand-written lines (counted on `main`): |---|---|---| | `ConnectionId` type and `next()` ([server.rs:176-188](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L176-L188)) | 13 | removed | | `connection_id` in `Established`, `PeerData` and the handshake ([server.rs:432-437](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L432-L437), [handshake.rs:177](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/handshake.rs#L177), [M peer_table.rs:324-326](https://github.com/lambdaclass/ethrex/blob/fd5c57ddada097952660f6b749c739119e4de852/crates/networking/p2p/peer_table.rs#L324-L326)) | ~8 | removed (replaced by the monitor) | -| `remove_peer` protocol method and `handle_remove_peer` with its guard ([V peer_table.rs:693-711](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/peer_table.rs#L693-L711)) | ~22 | replaced by a ~10-line `Handler` | +| `remove_peer` protocol method and `handle_remove_peer` with its guard ([V peer_table.rs:693-711](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/peer_table.rs#L693-L711)) | ~22 | replaced by a ~12-line `Handler` | | `remove_peer` send in `stopped()` | ~6 | removed | | inbound semaphore, `try_acquire_owned`, `_admission_permit` field and parameter | ~15 | replaced by `max_children` | | "connections are not admitted" comment and unmanaged `.start()` | | connections owned by the system | -What stays: first-wins registration (domain policy), `teardown()`'s mempool -cleanup, the tx-broadcaster index, `target_peers`. #7435's integration test -(171 lines) stays as a regression test, now exercising the monitor match. +Added: the `Handshake` first message (moving existing code), the writer as an +owned helper, the teardown guard and the broadcaster forward. Net lines are +roughly even; what changes is that every path out of a connection is +covered. #7435's integration test (171 lines) stays as a regression test, +now exercising the monitor guard. ### Bugs and gaps it closes - **Stale entries when `stopped()` does not run** (panic in `started()` - today; kills and supervisor shutdown under v0.8): the `Down` still comes. -- **The #7435 class** (an old connection's cleanup removing a new entry): - impossible by construction, because removal is keyed by the monitor of the - registered connection, not by node id. -- **Unowned connections at shutdown**: they are stopped, within the system - deadline, before the peer table, instead of dying with the process. -- **Placement**: connections run on the `p2p` group instead of whichever - runtime started them. + today; kills, `terminate_child` and shutdown under v0.8): the `Down` still + comes. +- **The #7435 class**: removal is keyed by the registered connection's + monitor and re-checked against the entry, not by node id. +- **Unowned connections at shutdown**: stopped within the system deadline, + before the peer table, instead of dying with the process. +- **Unowned writer tasks**: joined before the slot frees. +- **Placement**: connections run on the `p2p` group. ### Things to watch in a migration -- **`stopped()` is skipped on supervisor shutdown under v0.8** for a - non-trapping actor. `teardown()` and `tx_broadcaster.remove_peer` live in - `stopped()`; at node shutdown that is harmless (everything is going away), - but on a `kill` during normal operation (a connection stuck past its - `shutdown_timeout`) the broadcaster index would leak. Fix: the broadcaster - also monitors connections, or the peer table forwards removal from its - `Down` handler. Same reasoning as the exit-signals migration note. -- **Caps change meaning slightly**: the semaphore counted live actor states; - `max_children` counts slots until owned cleanup ends (helpers included). - For these actors (no helpers) that is the same. -- **Rejected arguments are dropped**: on `MaxChildren` the `TcpStream` inside - `ConnArgs` is dropped, closing the socket without an RLPx Disconnect, - exactly as today. Returning the arguments with the error is open question - 7 of the design. -- **Budget**: Temporary children never consume it, so the supervisor fails - only on a stuck cleanup or a closed group. A connection whose cleanup - hangs past twice its `shutdown_timeout` would fail the whole connection - supervisor; set `shutdown_timeout` with that in mind. +- **Serialised starts.** Anything slow left in `started()` holds every + accept and dial. Keep `started()` to state setup and the self-send. +- **Stuck cleanup.** A connection's own exit gets one `shutdown_timeout` + for its owned cleanup (E3 row 4, no kill); a terminated or shut-down one + gets the signal, the kill after `shutdown_timeout` and a second interval. + A connection that misses them is stuck: its slot stays counted until its + code returns, it is reported, and **nothing else is affected**, since + connections are Temporary (design: stuck children). Only budget + exhaustion (impossible for Temporary children) or a closed group fails + the supervisor; nesting it under the p2p supervisor still restarts it in + that case instead of leaving accepts disabled (agy P0). +- **Caps count slots until owned cleanup ends**, writer helper included, + so a slow socket close keeps its slot a little longer than the permit did. +- **Rejected arguments come back**, so the accept loop decides what to do + with the stream. ## ethlambda From e7617e553c46341b1bbbd8f88a23d1dfb6d57444 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:12:18 -0300 Subject: [PATCH 14/52] docs: revision 2 controls list, semantics, roadmap and changelog --- CHANGELOG.md | 8 ++++++-- ROADMAP.md | 2 +- docs/design/semantics.md | 4 ++-- verification/tla/negative_checks.py | 20 ++++++++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 947258ed..624434ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,8 +52,12 @@ - Design (not implementation) for rank 12a, the dynamic supervisor (`DynamicSupervisor` / `simple_one_for_one`, PR #245): `docs/design/dynamic-supervisor.md`, the ethrex/ethlambda client mapping - and the `DynamicSupervisor` TLA+ model (per-PR and weekly configurations) - with 11 named negative controls. + and the `DynamicSupervisor` TLA+ model (per-PR, zero-budget and weekly + configurations). Revision 2 applies the maintainer's decisions (OTP-style + serialised starts, Elixir's 3-in-5 s budget with opt-in backoff, + `Shutdown` on exhaustion, arguments returned on rejection) and both + reviews (stuck-child isolation, reservation protocol, history-based + invariants), with 27 named negative controls. ## 0.7.3 — 2026-11-10 diff --git a/ROADMAP.md b/ROADMAP.md index 35071e66..93b7cef9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -555,7 +555,7 @@ the first start (semantics §11; found reviewing #233). Behaviour change: extend | 11a | Design inputs to rank 11: promise pipelining and transactional turns | Decide within the rank 11 design, not as a separate API: (1) whether a pipelined send ("on reply to R, send M to the target R names") is expressible without an extra handler turn, with M failing like R rather than being dropped (E/Cap'n Proto/Goblins pipelining; the main latency tool if rank 44 ever ships); (2) whether deferred replies and completions are sent at the end of a turn or immediately, and an opt-in Goblins-style transactional turn that holds a handler's sends until it returns and discards them on panic (state rollback only for snapshot-able state; external effects are never rolled back); (3) whether per-turn snapshots can serve ranks 21 and 30 too. The rank 11 TLA+ model must cover a reply sent by a turn that then panics. See [request semantics references](docs/research/request-semantics-references.md); STM's "effects only on commit" is the same idea restricted to one actor, while its `retry` is selective receive and stays out ([STM note](docs/research/concurrency-models.md#5-software-transactional-memory)). | | 11b | Managed compute pool (CPU offload with pipe-to-self) | **Client evidence:** ethrex's four hand-built rayon pools, 50 `spawn_blocking` calls and the L2 import deadlock at 0% CPU (535647e45); synchronous cell serving that occupied every runtime worker (d958dcba4); ethlambda's hand-owned aggregation worker and semaphore-bounded, `catch_unwind`-guarded gossip checks ([evidence](docs/research/concurrency-models.md#who-uses-it)). One bounded CPU pool per `ActorSystem` or group, never on Tokio workers; `ctx.compute(job, map)` (name illustrative) delivers the result to the submitting actor as a message; per-actor and per-pool admission caps that reject at once or follow rank 17; cancellation tied to owner stop, restart and scope close (rank 15a) with generation fencing; a panic becomes an error message, exactly once; metrics for queue depth, wait, run time and rejections. Document that jobs must not wait on their own pool and gang jobs need a dedicated pool. Rayon behind a feature first; a fixed native pool as the alternative. Built on rank 11's completion contract; shares admission design with rank 34. Prior art: Akka dispatchers, Orleans `Task.Run`, Ray tasks, BEAM dirty schedulers ([concurrency models](docs/research/concurrency-models.md#1-cpu-parallel-compute-pools-integrated-with-actors)). | | 12 | Scoped, lifecycle-owned registries | **Client evidence:** ethrex's per-peer connection map and its `ConnectionId` guard (#7435). Keyed registration that owns the actor's lifetime, with generations so an old cleanup cannot remove its replacement. Also scope the registry to an `ActorSystem` instead of the process-global static, so parallel tests and multi-node simulations do not share names. Prerequisite for groups/pub-sub. Orleans keeps the same one-live-instance-per-key invariant ([virtual actors](docs/research/virtual-actors.md#what-it-means-for-spawned)). | -| 12a | Dynamic supervisor (`DynamicSupervisor` / `simple_one_for_one`) | **Design in review** (PR #245, not implemented): [design contract](docs/design/dynamic-supervisor.md), [client mapping](docs/design/dynamic-supervisor-client-mapping.md), TLA+ model `DynamicSupervisor.tla` (starts racing crashes, restarts, `terminate_child`, `max_children`, budget exhaustion and shutdown) with 11 negative controls. Registry-owned lookup stays in rank 12. **Client evidence:** ethrex starts one connection actor per peer at runtime and cleans it up by hand (see [per-key actor maps](#client-search-results)); today's `Supervisor` only owns the fixed child list declared at build time. Proposed contract: children are started at runtime from one `ChildSpec` template with per-call arguments, sharing its restart mode (Elixir-style heterogeneous specs only if a user needs them). The restart budget is supervisor-wide, as in OTP: exhausting it fails the supervisor and escalates to its parent, so a flapping child cannot restart forever. `max_children` counts restarting children, so a new start cannot take a slot mid-restart, and rejects with a typed error rather than queueing. `start_child` after shutdown begins is rejected, never dropped or leaked. Explicitly stopping a child does not restart it. Shutdown stops all live children concurrently under one supervisor deadline (OTP does not order dynamic children; sequential stops would multiply per-child timeouts). Usable inside a nested tree. Independent of rank 12 (`start_child` returns a handle); when combined, a restarted child re-registers under a new generation so an old cleanup cannot remove it. Planned after v0.7, not part of it; extend the TLA+ model (start/stop racing restart and shutdown, cap including restarting children) before release, as for OneForAll/RestForOne. The other OTP variants stay out: `Task.Supervisor` folds into ranks 11/15, `PartitionSupervisor` is sharding, and significant children/`auto_shutdown` wait for a user. **Research ([supervision models](docs/research/supervision-models.md)):** counting restarting children against `max_children` matches Elixir, which keeps a restarting child in its map (corrected by the design; this said it went beyond Elixir); add `terminate_child` (stop without restart; today an individually stopped Permanent child is restarted) and consider it on static supervisors too. Significant children/`auto_shutdown` would be a trigger for draining (rank 13). Document why Spawned's default budget (5 restarts in 30 s) is looser than OTP's (1 in 5 s): per-child backoff already spreads restarts. Non-goals: an Akka-style Resume directive (keep state after a failure) and buffering messages during backoff, because references are instance-bound. | +| 12a | Dynamic supervisor (`DynamicSupervisor` / `simple_one_for_one`) | **Design in review** (PR #245, not implemented): [design contract](docs/design/dynamic-supervisor.md), [client mapping](docs/design/dynamic-supervisor-client-mapping.md), TLA+ model `DynamicSupervisor.tla` (starts racing crashes, restarts, `terminate_child`, `max_children`, budget exhaustion and shutdown) with 27 negative controls; revision 2 settles the open questions as OTP/Elixir (serialised starts, 3 restarts in 5 s with opt-in backoff, `Shutdown` on exhaustion) and isolates children stuck while retiring. Follow-ups: static nested supervisors exit `Shutdown` on exhaustion, static `terminate_child`. Registry-owned lookup stays in rank 12. **Client evidence:** ethrex starts one connection actor per peer at runtime and cleans it up by hand (see [per-key actor maps](#client-search-results)); today's `Supervisor` only owns the fixed child list declared at build time. Proposed contract: children are started at runtime from one `ChildSpec` template with per-call arguments, sharing its restart mode (Elixir-style heterogeneous specs only if a user needs them). The restart budget is supervisor-wide, as in OTP: exhausting it fails the supervisor and escalates to its parent, so a flapping child cannot restart forever. `max_children` counts restarting children, so a new start cannot take a slot mid-restart, and rejects with a typed error rather than queueing. `start_child` after shutdown begins is rejected, never dropped or leaked. Explicitly stopping a child does not restart it. Shutdown stops all live children concurrently under one supervisor deadline (OTP does not order dynamic children; sequential stops would multiply per-child timeouts). Usable inside a nested tree. Independent of rank 12 (`start_child` returns a handle); when combined, a restarted child re-registers under a new generation so an old cleanup cannot remove it. Planned after v0.7, not part of it; extend the TLA+ model (start/stop racing restart and shutdown, cap including restarting children) before release, as for OneForAll/RestForOne. The other OTP variants stay out: `Task.Supervisor` folds into ranks 11/15, `PartitionSupervisor` is sharding, and significant children/`auto_shutdown` wait for a user. **Research ([supervision models](docs/research/supervision-models.md)):** counting restarting children against `max_children` matches Elixir, which keeps a restarting child in its map (corrected by the design; this said it went beyond Elixir); add `terminate_child` (stop without restart; today an individually stopped Permanent child is restarted) and consider it on static supervisors too. Significant children/`auto_shutdown` would be a trigger for draining (rank 13). The dynamic supervisor uses Elixir's defaults (3 restarts in 5 s, no backoff unless configured). Non-goals: an Akka-style Resume directive (keep state after a failure) and buffering messages during backoff, because references are instance-bound. | | 13 | Coordinated shutdown and bounded draining | Ordered stop exists; add explicit draining as a policy: close admission, handle queued work by policy, join helpers and report unfinished work. Standalone scopes currently use reverse admission order; explore declared dependencies with cycle/unknown-edge rejection and model-checked deadline propagation. Keep current stop/discard as the default. Design with rank 15a: draining is closing the actor's scope under its deadline. Keep "stuck cleanup fails the tree" instead of OTP's `brutal_kill`; add only reporting of unfinished work ([supervision models](docs/research/supervision-models.md)). | | 14 | Separate fair control path | Part of supervision/shutdown design; defer a generic public priority API. Non-trapping actors already receive abnormal exits through cancellation; queued `ExitEnvelope` only matters for trapping actors. Specify boundedness and per-sender ordering for startup, monitor and trapped-exit events. Priority cannot preempt a running handler or starve user work. | | 15 | Actor-owned helper completion scopes | **Client evidence:** ethlambda hand-owns its aggregation worker; add cancel-previous, generation fencing and join-with-deadline. Managed helpers require ActorSystem; explore standalone ownership without detached fallback or changing actor-join guarantees implicitly. Gate new helper registration, cancel, then explicitly join; helpers depending on their stopped owner can deadlock. | diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 9075cf8e..3803265d 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -554,8 +554,8 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table - Rank 12a, dynamic supervisor (design in review, [dynamic-supervisor.md](dynamic-supervisor.md)): adds V10-V13 (dynamic admission, a cap that counts restarting children, terminated means - removed, concurrent shutdown of dynamic children); touches V1, V3-V5, V8, - V9, S1, S2. + removed, concurrent shutdown of dynamic children) and V14 (a child stuck + while retiring fails only itself); touches V1, V3-V5, V8, V9, S1, S2. - Rank 13, bounded draining: an opt-in stop that handles queued work by policy instead of discarding it (D4 remains the default). - Rank 14, fair control path: lifecycle signals not stuck behind a backlog (X7). diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 70046a1d..4c8563b1 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -182,6 +182,10 @@ def named_temporal(module, fault, prop, tlc, constants=None): # Rank 12a design model (docs/design/dynamic-supervisor.md): each mutation # against the invariant it must break, on the per-PR configuration. +# Rank 12a design model (docs/design/dynamic-supervisor.md, revision 2): +# each mutation against the invariant it must break, on the per-PR +# configuration. Run one at a time (parallel TLC runs race on the standard +# modules TLC extracts to /tmp). DYNAMIC_CONTROLS = [ ("MCDynamicSupervisor", "IgnoreMax", "MaxChildrenBound"), ("MCDynamicSupervisor", "UncountedRestart", "MaxChildrenBound"), @@ -192,6 +196,22 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "NoEscalate", "ExhaustionCloses"), ("MCDynamicSupervisor", "EarlyDone", "NoChildOutlivesSupervisor"), ("MCDynamicSupervisor", "Sequential", "ConcurrentShutdown"), + ("MCDynamicSupervisor", "ConcurrentStart", "SerialConstruction"), + ("MCDynamicSupervisor", "StuckFailsAll", "StuckIsolated"), + ("MCDynamicSupervisor", "PublishAfterClose", "NoPublishAfterStop"), + ("MCDynamicSupervisor", "OverlapGeneration", "AtMostOneGeneration"), + ("MCDynamicSupervisor", "PublishBeforeReady", "PublishedIsLive"), + ("MCDynamicSupervisor", "PrematureOk", "OkMeansReady"), + ("MCDynamicSupervisor", "UnchargedRetry", "ChargedRestarts"), + ("MCDynamicSupervisor", "ChargeFirstStart", "ChargedRestarts"), + ("MCDynamicSupervisor", "TransientNormalRestart", "RestartFollowsPolicy"), + ("MCDynamicSupervisor", "PermanentRetires", "RestartFollowsPolicy"), + ("MCDynamicSupervisor", "RetryFirstStart", "FirstStartNeverRetried"), + ("MCDynamicSupervisor", "EarlyFailedReply", "FailedAfterCleanup"), + ("MCDynamicSupervisor", "LateFailedOverwrite", "FailureCauseKept"), + ("MCDynamicSupervisor", "NoRefund", "ReservationsMatch"), + ("MCDynamicSupervisor", "ExpireReserved", "ReservationsMatch"), + ("MCDynamicSupervisor", "TermReplyEarly", "TermOkMeansRemoved"), ] DYNAMIC_TEMPORAL_CONTROLS = [ ("NoAbort", "StartAnswered"), From 61d426c68abdd91437acdbda118ce7b40290feef Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:43:08 -0300 Subject: [PATCH 15/52] verify: two slots with every pair of restart types (three slots exceed 100M states) --- verification/tla/DynamicSupervisor.tla | 31 ++++++++++++-------- verification/tla/MCDynamicSupervisor.cfg | 14 ++++----- verification/tla/MCDynamicSupervisor.tla | 6 +--- verification/tla/MCDynamicSupervisorDeep.cfg | 12 ++++---- verification/tla/MCDynamicSupervisorZero.cfg | 10 +++---- 5 files changed, 37 insertions(+), 36 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 6a610594..ace5ff47 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -13,8 +13,11 @@ (* up concurrently. Shutdown (Close) and failure close admission at once *) (* and cancel the operation in progress (B: tree-wide admission closure). *) (* *) -(* Restart types are per slot so one instance covers all three (a *) -(* simple_one_for_one template has one type; mixing them loses nothing). *) +(* Restart types are chosen per slot in Init (`mode`), so one run covers *) +(* every combination (a simple_one_for_one template has one type; mixing *) +(* them is the general case). Two slots suffice: every property relates *) +(* at most a pair of children (one holds the cap or restarts, the other *) +(* races it); three slots exceed 100M states with revision 2's detail. *) (* *) (* Abstractions: *) (* - `open` is the admission gate and the supervisor's cancel together. *) @@ -42,7 +45,7 @@ EXTENDS Naturals, FiniteSets CONSTANTS N, \* number of start_child calls (slots 1..N) - Mode, \* Mode[s] \in {"Permanent", "Transient", "Temporary"} + ModeSet, \* restart types a slot may have; chosen per slot in Init MaxChildren, \* max_children MaxRestarts, \* restart intensity (grants per window) MaxExpire, \* how often the rolling window may release a grant @@ -71,7 +74,7 @@ Faults == {"None", ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat ASSUME MaxRestarts \in Nat /\ MaxExpire \in Nat -ASSUME \A s \in Slots : Mode[s] \in Modes +ASSUME ModeSet \subseteq Modes /\ ModeSet # {} ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons ASSUME CleanupFails \subseteq BOOLEAN /\ FALSE \in CleanupFails ASSUME Fault \in Faults @@ -107,6 +110,7 @@ VARIABLES reserved, \* slots holding a reservation (pending restarts) stamped, \* grants ever stamped (charged restart attempts) op, \* the control loop's current operation [k, s] + mode, \* restart type of each slot, fixed after Init sl, \* per-slot record, see SlotInit \* History, written only by GhostUpdate. lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, @@ -114,7 +118,7 @@ VARIABLES sup == <> ghost == <> -vars == <> +vars == <> SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, reason |-> "none", cfail |-> FALSE, readyGen |-> FALSE, @@ -139,6 +143,7 @@ Init == /\ open = TRUE /\ failed = FALSE /\ exhausted = FALSE /\ sphase = "running" /\ windowed = 0 /\ expired = 0 /\ reserved = {} /\ stamped = 0 /\ op = Idle /\ sl = [s \in Slots |-> SlotInit] + /\ mode \in [Slots -> ModeSet] /\ lateCommit = [s \in Slots |-> FALSE] /\ latePub = [s \in Slots |-> FALSE] /\ lateGenClose = [s \in Slots |-> FALSE] /\ lateGenTerm = [s \in Slots |-> FALSE] /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] @@ -164,17 +169,17 @@ DesignAllowsRestart(m, r, cf) == \* the table, or a failed replacement (never ready) of a restartable child. Allowed(s) == /\ sl[s].everReady - /\ \/ DesignAllowsRestart(Mode[s], sl[s].reason, sl[s].cfail) - \/ ~sl[s].readyGen /\ Mode[s] # "Temporary" + /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, sl[s].cfail) + \/ ~sl[s].readyGen /\ mode[s] # "Temporary" \* What the supervisor decides (with the policy faults). WouldRestart(s, cf) == /\ sl[s].everReady \/ Fault = "RetryFirstStart" - /\ \/ Mode[s] = "Permanent" /\ Fault # "PermanentRetires" - \/ Mode[s] = "Transient" + /\ \/ mode[s] = "Permanent" /\ Fault # "PermanentRetires" + \/ mode[s] = "Transient" /\ (sl[s].reason \in Abnormal \/ cf \/ Fault = "TransientNormalRestart") - \/ Mode[s] # "Temporary" /\ ~sl[s].readyGen - \/ Mode[s] = "Temporary" /\ Fault = "RestartTemporary" + \/ mode[s] # "Temporary" /\ ~sl[s].readyGen + \/ mode[s] = "Temporary" /\ Fault = "RestartTemporary" CloseGate == open' = FALSE Fail == failed' = TRUE /\ CloseGate @@ -450,7 +455,7 @@ Act == \/ Close \/ WindowExpire \/ SupSignal \/ SupSignalOne \/ SupDone \/ \E s \in Slots : SlotStep(s) -Next == Act /\ GhostUpdate +Next == Act /\ GhostUpdate /\ UNCHANGED mode \* Progress: the loop, construction, stops, cleanup and stuck children \* eventually move (crashes, start failures and getting stuck are not fair). @@ -492,7 +497,7 @@ MaxChildrenBound == Cardinality({s \in Slots : sl[s].cs \in Active}) <= MaxChild SerialConstruction == Cardinality({s \in Slots : sl[s].cs \in {"starting", "restarting"}}) <= 1 -TemporaryNeverRestarted == \A s \in Slots : Mode[s] = "Temporary" => sl[s].gen <= 1 +TemporaryNeverRestarted == \A s \in Slots : mode[s] = "Temporary" => sl[s].gen <= 1 \* No generation begins after terminate_child, or after admission closed. TerminatedNeverRestarted == \A s \in Slots : ~lateGenTerm[s] diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index 747c1aa0..2153e6aa 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -1,17 +1,17 @@ -\* Fast instance (CI, every PR): three start_child calls (Permanent, -\* Transient, Temporary), each terminable, against max_children 2; budget 1 -\* with one window release; exits Normal/Panic, start failures factory and -\* Normal-before-readiness, cleanup failures. Safety, NeverReopens and +\* Per PR: two start_child calls, every pair of restart types, each child +\* terminable, max_children 1 (one holds it while the other races), budget +\* 1 with one window release; exits Normal/Panic, start failures factory +\* and Normal-before-readiness, cleanup failures. Safety, NeverReopens and \* liveness. Every DynamicSupervisor negative control runs here. SPECIFICATION Spec CONSTANTS - N = 3 - Mode <- MCMode3 - MaxChildren = 2 + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} Fault = "None" + MaxChildren = 1 MaxRestarts = 1 MaxExpire = 1 INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla index e5c82bef..49d4c9bb 100644 --- a/verification/tla/MCDynamicSupervisor.tla +++ b/verification/tla/MCDynamicSupervisor.tla @@ -1,9 +1,5 @@ ------------------------ MODULE MCDynamicSupervisor ------------------------ (* Model-checking instances for DynamicSupervisor (rank 12a design). *) +(* Restart types come from ModeSet in the .cfg files. *) EXTENDS DynamicSupervisor - -\* One start_child slot per restart type (fast, every PR). -MCMode3 == [s \in 1..3 |-> CASE s = 1 -> "Permanent" - [] s = 2 -> "Transient" - [] OTHER -> "Temporary"] ============================================================================= diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 17bd39e8..cd54e0c0 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,15 +1,15 @@ -\* Deep instance (weekly, --deep): as the fast instance with budget 2 and -\* two window releases (up to four restarts). Safety and NeverReopens. -\* Four slots do not fit: revision 1 passed 222M distinct states in 30 min. +\* Weekly (--deep): as the per-PR instance with budget 2 and two window +\* releases (up to four restarts) and max_children 2 (both children held +\* together while restarting). Safety and NeverReopens. SPECIFICATION Spec CONSTANTS - N = 3 - Mode <- MCMode3 - MaxChildren = 2 + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} Fault = "None" + MaxChildren = 2 MaxRestarts = 2 MaxExpire = 2 INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg index 3bf9141a..487a6e1f 100644 --- a/verification/tla/MCDynamicSupervisorZero.cfg +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -1,14 +1,14 @@ -\* Zero budget (CI, every PR): max_restarts 0, so the first restartable -\* exit fails the supervisor at once, without entering backoff. +\* Per PR, zero budget: max_restarts 0, so the first restartable exit fails +\* the supervisor at once, without entering backoff. SPECIFICATION Spec CONSTANTS - N = 3 - Mode <- MCMode3 - MaxChildren = 2 + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} Fault = "None" + MaxChildren = 1 MaxRestarts = 0 MaxExpire = 0 INVARIANT TypeOK From 91e5163ef7bddd2d66b31a5eece97ee2cfe3824f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:43:31 -0300 Subject: [PATCH 16/52] docs: model descriptions for revision 2 --- docs/TESTING.md | 20 +++++----- docs/design/dynamic-supervisor.md | 7 +++- verification/tla/README.md | 61 ++++++++++++------------------- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index bb9905e2..e8ba9772 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -925,15 +925,17 @@ found one model gap (a running child may exit with `Shutdown` while the supervisor is open, via `ChildHandle::shutdown`); no implementation deviation. `verification/tla/DynamicSupervisor.tla` models the rank 12a design (not -implemented): `start_child` racing crashes, restarts, `terminate_child`, -`max_children`, budget exhaustion and shutdown. TLC checks twelve safety -invariants (no child outlives the supervisor, the cap counts restarting -children, Temporary and terminated children are never restarted, no start -succeeds after shutdown begins, exhaustion closes the supervisor, shutdown -signals every child at once), `NeverReopens` and three liveness properties; -`MCDynamicSupervisor` runs on every PR and `MCDynamicSupervisorDeep` weekly. -Eleven named negative controls (nine safety, two liveness) each produce a -counterexample. See the [design](design/dynamic-supervisor.md#what-the-model-establishes). +implemented, revision 2): `start_child` racing crashes, restarts, +`terminate_child`, `max_children`, the budget reservation protocol, stuck +children and shutdown, with OTP's serialised control loop. TLC checks 23 +safety invariants (among them: no child outlives the supervisor, the cap +counts restarting children, restarts follow the table in both directions, +every restart attempt is charged once, no commit, publication or generation +after close or terminate, a child stuck while retiring fails only itself), +`NeverReopens` and four liveness properties. `MCDynamicSupervisor` and +`MCDynamicSupervisorZero` run on every PR, `MCDynamicSupervisorDeep` weekly. +Twenty-seven named negative controls each produce a counterexample. See the +[design](design/dynamic-supervisor.md#what-the-model-establishes). Apalache checks that the safety invariants of `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` are inductive (`Init => IndInv`, diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index c0d286e4..4acb8167 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -743,8 +743,11 @@ that generation died, not that the slot is gone. (revision 2) models one dynamic supervisor, N `start_child` calls made at any moment, and one `terminate_child` per started child. A control loop (`op`) serialises admission and first construction, exit handling and restart -attempts, and terminations. Restart types are per slot so one instance -covers all three. Budget reservations, stamping, refunds and window expiry +attempts, and terminations. Each slot's restart type is chosen in `Init`, +so one run covers every combination. The instances use two slots: every +property relates at most a pair of children (one holds the cap or restarts +while the other races it), and with revision 2's detail three slots passed +6.7 GB of TLC state in 38 minutes without finishing. Budget reservations, stamping, refunds and window expiry are explicit. Exits come from children (`Normal`, `Panic`), stop signals (`Shutdown`, or `Killed` after the kill), start failures (factory error, `Normal` before readiness), and cleanup may record a failure (the Transient diff --git a/verification/tla/README.md b/verification/tla/README.md index 2d7652d1..8e8eeb22 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): one dynamic supervisor and N `start_child` calls racing crashes, restarts with fresh generations, `terminate_child`, `max_children` (counting restarting children), the supervisor-wide budget and its exhaustion, and concurrent shutdown with the kill escalation and stuck cleanup. Eleven named negative controls (nine safety, two liveness). Per PR: `MCDynamicSupervisor` (3 slots, one per restart type); weekly: `MCDynamicSupervisorDeep` (the same 3 slots with budget 2 and two window releases). | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Zero,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): `start_child` calls racing crashes, restarts, `terminate_child`, `max_children` (counting restarting children), the budget reservation protocol and its exhaustion, stuck children and concurrent shutdown, with OTP's serialised control loop. Twenty-seven named negative controls. Per PR: `MCDynamicSupervisor`, `MCDynamicSupervisorZero`; weekly: `MCDynamicSupervisorDeep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -583,45 +583,32 @@ All 22 named controls find their counterexample, each in a few seconds. ## Dynamic supervisor (rank 12a design) -`DynamicSupervisor.tla` backs the [dynamic supervisor design](../../docs/design/dynamic-supervisor.md), -written before any implementation. Slots 1..N are `start_child` calls, each -made at most once at any moment, so starts race everything else. Restart -types are per slot (one instance covers all three). The rolling window is a -counter released at most `MaxExpire` times, which also bounds the number of -generations, so no state constraint is needed and the liveness properties -are checked on the full graph. - -Invariants: `AtMostOneGeneration`, `PublishedIsLive`, -`NoChildOutlivesSupervisor`, `MaxChildrenBound`, `TemporaryNeverRestarted`, -`TerminatedNeverRestarted`, `NoStartAfterShutdown`, `NoGenerationAfterClose`, -`ExhaustionCloses`, `BudgetBounded`, `ConcurrentShutdown`, `OkMeansReady`. -Action property `NeverReopens`; liveness `ClosedLeadsToDone`, -`StartAnswered`, `TerminateCompletes` (crashes and stuck cleanup are not -fair). - -| Configuration | Mutation | Breaks | -|---|---|---| -| `MCDynamicSupervisor` | `IgnoreMax` | `MaxChildrenBound` | -| `MCDynamicSupervisor` | `UncountedRestart` (a crashed child frees its slot until it restarts) | `MaxChildrenBound` | -| `MCDynamicSupervisor` | `RestartTemporary` | `TemporaryNeverRestarted` | -| `MCDynamicSupervisor` | `RestartTerminated` | `TerminatedNeverRestarted` | -| `MCDynamicSupervisor` | `AcceptDuringShutdown` | `NoStartAfterShutdown` | -| `MCDynamicSupervisor` | `RestartAfterClose` | `NoGenerationAfterClose` | -| `MCDynamicSupervisor` | `NoEscalate` (exhaustion drops only that child) | `ExhaustionCloses` | -| `MCDynamicSupervisor` | `EarlyDone` | `NoChildOutlivesSupervisor` | -| `MCDynamicSupervisor` | `Sequential` (static reverse-order stops) | `ConcurrentShutdown` | -| `MCDynamicSupervisor` (liveness) | `NoAbort` (a start racing shutdown is not cancelled) | `StartAnswered` | -| `MCDynamicSupervisor` (liveness) | `StuckBackoff` (shutdown does not cancel a pending restart) | `ClosedLeadsToDone` | +`DynamicSupervisor.tla` backs the [dynamic supervisor design](../../docs/design/dynamic-supervisor.md) +(revision 2), written before any implementation. Slots are `start_child` +calls made at any moment, each started child can get one `terminate_child`, +and one control loop serialises starts, exit handling and terminations as +OTP's supervisor process does. Each slot's restart type is chosen in `Init`. +Budget reservations, stamping, refunds and window expiry are explicit; +children can get stuck (before or after their death point) under the +stuck-isolation policy. Invariants about events (late commits, publications +and generations, forbidden restarts or retirements, restart attempts) read +history variables computed by `GhostUpdate` from each step's state change, +separately from the actions. + +Configurations: `MCDynamicSupervisor` (per PR; two slots, every pair of +restart types, `max_children` 1, budget 1; safety, `NeverReopens`, +liveness), `MCDynamicSupervisorZero` (per PR; budget 0), +`MCDynamicSupervisorDeep` (weekly; budget 2, two window releases, +`max_children` 2; safety). Twenty-seven named negative controls (25 safety, +2 liveness), listed with their origin in the design's +[model section](../../docs/design/dynamic-supervisor.md#what-the-model-establishes), +run one at a time from `negative_checks.py`. Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (fast: 3 slots, `max_children` 2, budget 1, one window release; safety, `NeverReopens`, liveness) | 1,601,979 | 5,010,572 | 33 | 2m16s | -| `MCDynamicSupervisorDeep` (budget 2, two releases; safety, `NeverReopens`) | 5,617,958 | 18,853,746 | 42 | 41 s | - -All eleven controls find their counterexample in under 5 s each. Not yet: -Apalache inductive invariants (needs type annotations and an `IndInv`, as -`SupervisionInd.tla`) and trace validation, which come with the -implementation. +README_ROWS +Not yet: Apalache inductive invariants and trace validation, which come +with the implementation. From 32c9348405783936914cb68c7f7517c901d2cff5 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:44:04 -0300 Subject: [PATCH 17/52] verify: keep mode unchanged inside GhostUpdate (fairness actions) --- verification/tla/DynamicSupervisor.tla | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index ace5ff47..fb6dd3fe 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -440,6 +440,7 @@ GhostUpdate == (sl[s].cs = "decide" /\ sl'[s].cs = "removed" /\ open' /\ ~sl[s].term /\ Allowed(s))] /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) + /\ UNCHANGED mode SlotStep(s) == \/ Request(s) \/ LoopAdmit(s) \/ RejectQueued(s) @@ -455,7 +456,7 @@ Act == \/ Close \/ WindowExpire \/ SupSignal \/ SupSignalOne \/ SupDone \/ \E s \in Slots : SlotStep(s) -Next == Act /\ GhostUpdate /\ UNCHANGED mode +Next == Act /\ GhostUpdate \* Progress: the loop, construction, stops, cleanup and stuck children \* eventually move (crashes, start failures and getting stuck are not fair). From dada3e17ba7c8b0a7396fd75a090fdf91f775ea2 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:00:29 -0300 Subject: [PATCH 18/52] verify: per-PR liveness in MCDynamicSupervisorLive --- verification/tla/MCDynamicSupervisor.cfg | 8 +--- verification/tla/MCDynamicSupervisorLive.cfg | 43 ++++++++++++++++++++ verification/tla/MCDynamicSupervisorLive.tla | 3 ++ verification/tla/negative_checks.py | 2 +- 4 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorLive.cfg create mode 100644 verification/tla/MCDynamicSupervisorLive.tla diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index 2153e6aa..827a2abf 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -1,8 +1,8 @@ \* Per PR: two start_child calls, every pair of restart types, each child \* terminable, max_children 1 (one holds it while the other races), budget \* 1 with one window release; exits Normal/Panic, start failures factory -\* and Normal-before-readiness, cleanup failures. Safety, NeverReopens and -\* liveness. Every DynamicSupervisor negative control runs here. +\* and Normal-before-readiness, cleanup failures. Safety and NeverReopens; +\* every safety negative control runs here. Liveness: MCDynamicSupervisorLive. SPECIFICATION Spec CONSTANTS N = 2 @@ -38,7 +38,3 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermOkMeansRemoved PROPERTY NeverReopens -PROPERTY ClosedLeadsToDone -PROPERTY StartAnswered -PROPERTY TerminateAnswered -PROPERTY RecoveryProgress diff --git a/verification/tla/MCDynamicSupervisorLive.cfg b/verification/tla/MCDynamicSupervisorLive.cfg new file mode 100644 index 00000000..dda48fc2 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorLive.cfg @@ -0,0 +1,43 @@ +\* Per PR, liveness: as MCDynamicSupervisor without recorded cleanup +\* failures and with factory failures only, to keep the liveness graph +\* small. ClosedLeadsToDone, StartAnswered, TerminateAnswered, +\* RecoveryProgress; the liveness negative controls run here. +SPECIFICATION Spec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + SelfExits = {"Normal", "Panic"} + StartFails = {"none"} + CleanupFails = {FALSE} + Fault = "None" + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermOkMeansRemoved +PROPERTY NeverReopens +PROPERTY ClosedLeadsToDone +PROPERTY StartAnswered +PROPERTY TerminateAnswered +PROPERTY RecoveryProgress diff --git a/verification/tla/MCDynamicSupervisorLive.tla b/verification/tla/MCDynamicSupervisorLive.tla new file mode 100644 index 00000000..b83af0fb --- /dev/null +++ b/verification/tla/MCDynamicSupervisorLive.tla @@ -0,0 +1,3 @@ +----------------------- MODULE MCDynamicSupervisorLive ----------------------- +EXTENDS MCDynamicSupervisor +============================================================================= diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 4c8563b1..59ac3a7b 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -307,7 +307,7 @@ def fail_instead_of_retry(events): for module, fault, invariant in DYNAMIC_CONTROLS: named_fault(module, fault, invariant, args.tlc) for fault, prop in DYNAMIC_TEMPORAL_CONTROLS: - named_temporal("MCDynamicSupervisor", fault, prop, args.tlc) + named_temporal("MCDynamicSupervisorLive", fault, prop, args.tlc) if __name__ == "__main__": From fbd121ecf84d3812ca9523380098c494219c09f8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:07:03 -0300 Subject: [PATCH 19/52] docs: revision 2 TLC results --- docs/TESTING.md | 4 ++-- docs/design/dynamic-supervisor.md | 8 ++++++-- verification/tla/README.md | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index e8ba9772..06504f70 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -932,8 +932,8 @@ safety invariants (among them: no child outlives the supervisor, the cap counts restarting children, restarts follow the table in both directions, every restart attempt is charged once, no commit, publication or generation after close or terminate, a child stuck while retiring fails only itself), -`NeverReopens` and four liveness properties. `MCDynamicSupervisor` and -`MCDynamicSupervisorZero` run on every PR, `MCDynamicSupervisorDeep` weekly. +`NeverReopens` and four liveness properties. `MCDynamicSupervisor`, +`MCDynamicSupervisorLive` and `MCDynamicSupervisorZero` run on every PR, `MCDynamicSupervisorDeep` weekly. Twenty-seven named negative controls each produce a counterexample. See the [design](design/dynamic-supervisor.md#what-the-model-establishes). diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 4acb8167..58ab2ee4 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -796,10 +796,14 @@ TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -RESULT_ROWS +| `MCDynamicSupervisor` (per PR; safety, `NeverReopens`) | 2 slots, every pair of restart types, `max_children` 1, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Normal`, cleanup failures | 3,459,556 | 33 | 39 s | +| `MCDynamicSupervisorLive` (per PR; liveness) | as above without cleanup failures, factory start failures only | 1,628,738 | 33 | 5m21s | +| `MCDynamicSupervisorZero` (per PR; safety, liveness) | as `MCDynamicSupervisor` with budget 0 | 589,159 | 22 | 1m23s | +| `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two window releases, `max_children` 2 | 31,620,430 | 42 | 5m06s | Negative controls (`negative_checks.py`, run one at a time; each checks -`TypeOK` and only the named property, on `MCDynamicSupervisor`): +`TypeOK` and only the named property, safety controls on +`MCDynamicSupervisor`, liveness controls on `MCDynamicSupervisorLive`): | `Fault` | Mutation | Fails | Origin | |---|---|---|---| diff --git a/verification/tla/README.md b/verification/tla/README.md index 8e8eeb22..ed0d04db 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Zero,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): `start_child` calls racing crashes, restarts, `terminate_child`, `max_children` (counting restarting children), the budget reservation protocol and its exhaustion, stuck children and concurrent shutdown, with OTP's serialised control loop. Twenty-seven named negative controls. Per PR: `MCDynamicSupervisor`, `MCDynamicSupervisorZero`; weekly: `MCDynamicSupervisorDeep`. | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Live,Zero,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): `start_child` calls racing crashes, restarts, `terminate_child`, `max_children` (counting restarting children), the budget reservation protocol and its exhaustion, stuck children and concurrent shutdown, with OTP's serialised control loop. Twenty-seven named negative controls. Per PR: `MCDynamicSupervisor`, `MCDynamicSupervisorLive`, `MCDynamicSupervisorZero`; weekly: `MCDynamicSupervisorDeep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -596,8 +596,9 @@ history variables computed by `GhostUpdate` from each step's state change, separately from the actions. Configurations: `MCDynamicSupervisor` (per PR; two slots, every pair of -restart types, `max_children` 1, budget 1; safety, `NeverReopens`, -liveness), `MCDynamicSupervisorZero` (per PR; budget 0), +restart types, `max_children` 1, budget 1; safety, `NeverReopens`), +`MCDynamicSupervisorLive` (per PR; liveness on a slightly smaller +instance), `MCDynamicSupervisorZero` (per PR; budget 0), `MCDynamicSupervisorDeep` (weekly; budget 2, two window releases, `max_children` 2; safety). Twenty-seven named negative controls (25 safety, 2 liveness), listed with their origin in the design's @@ -608,7 +609,10 @@ Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -README_ROWS +| `MCDynamicSupervisor` (safety, `NeverReopens`) | 3,459,556 | 7,826,992 | 33 | 39 s | +| `MCDynamicSupervisorLive` (liveness) | 1,628,738 | 3,579,100 | 33 | 5m21s | +| `MCDynamicSupervisorZero` (safety, liveness) | 589,159 | 1,172,771 | 22 | 1m23s | +| `MCDynamicSupervisorDeep` (safety) | 31,620,430 | 95,086,511 | 42 | 5m06s | Not yet: Apalache inductive invariants and trace validation, which come with the implementation. From e23883c02d5e26f062b0c957dc6cce287016830a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:14:46 -0300 Subject: [PATCH 20/52] verify: ConcurrentStart control on max_children 2 (named_fault constant overrides) --- verification/tla/negative_checks.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 59ac3a7b..d5022345 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -96,10 +96,13 @@ def model_fault(module, fault, tlc, temporal=False, invariant="Safety", drop=()) print(f"COUNTEREXAMPLE model: {module}/{fault}" + ("" if temporal else f" ({invariant})")) -def named_fault(module, fault, invariant, tlc): +def named_fault(module, fault, invariant, tlc, constants=None): """Run MODULE (safety only) with Fault = FAULT, checking TypeOK and the single named INVARIANT, and require that invariant to be violated. This - shows the mutation breaks the property it targets, not just Safety.""" + shows the mutation breaks the property it targets, not just Safety. + CONSTANTS overrides constant values (an instance where the fault can + show).""" + constants = constants or {} with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): @@ -112,6 +115,9 @@ def named_fault(module, fault, invariant, tlc): line = "SPECIFICATION Spec" if line.startswith("INVARIANT"): line = f"INVARIANT TypeOK\nINVARIANT {invariant}" + name = line.strip().split(" ")[0] + if name in constants: + line = f" {name} = {constants[name]}" lines.append(line.replace('Fault = "None"', f'Fault = "{fault}"')) (work / f"{module}.cfg").write_text("\n".join(lines) + "\n") result = subprocess.run(shlex.split(tlc) + ["-workers", "1", "-deadlock", "-cleanup", module], @@ -196,7 +202,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "NoEscalate", "ExhaustionCloses"), ("MCDynamicSupervisor", "EarlyDone", "NoChildOutlivesSupervisor"), ("MCDynamicSupervisor", "Sequential", "ConcurrentShutdown"), - ("MCDynamicSupervisor", "ConcurrentStart", "SerialConstruction"), + # Two constructions at once need room for two children. + ("MCDynamicSupervisor", "ConcurrentStart", "SerialConstruction", {"MaxChildren": "2"}), ("MCDynamicSupervisor", "StuckFailsAll", "StuckIsolated"), ("MCDynamicSupervisor", "PublishAfterClose", "NoPublishAfterStop"), ("MCDynamicSupervisor", "OverlapGeneration", "AtMostOneGeneration"), @@ -304,8 +311,8 @@ def fail_instead_of_retry(events): named_fault(module, fault, invariant, args.tlc) for fault, prop in LIFECYCLE_TEMPORAL_CONTROLS: named_temporal("MCLifecycleLive", fault, prop, args.tlc, LIFECYCLE_SMALL) - for module, fault, invariant in DYNAMIC_CONTROLS: - named_fault(module, fault, invariant, args.tlc) + for module, fault, invariant, *constants in DYNAMIC_CONTROLS: + named_fault(module, fault, invariant, args.tlc, *constants) for fault, prop in DYNAMIC_TEMPORAL_CONTROLS: named_temporal("MCDynamicSupervisorLive", fault, prop, args.tlc) From a21eb02b94b1d4db4da32791e974c7ee12876751 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:15:01 -0300 Subject: [PATCH 21/52] docs: negative-control results for revision 2 --- docs/design/dynamic-supervisor.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 58ab2ee4..70c18838 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -816,7 +816,7 @@ Negative controls (`negative_checks.py`, run one at a time; each checks | `NoEscalate` | exhaustion drops only that child | `ExhaustionCloses` | rev. 1 | | `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | rev. 1 | | `Sequential` | children stopped one at a time | `ConcurrentShutdown` | rev. 1 | -| `ConcurrentStart` | constructions not serialised | `SerialConstruction` | Codex 1 | +| `ConcurrentStart` | constructions not serialised (checked with `max_children` 2: with a cap of one the cap alone serialises) | `SerialConstruction` | Codex 1 | | `StuckFailsAll` | any stuck child fails the supervisor | `StuckIsolated` | agy P0 | | `PublishAfterClose` | a constructing replacement publishes after close | `NoPublishAfterStop` | agy 3 | | `OverlapGeneration` | the replacement starts before the old cleanup ended | `AtMostOneGeneration` | Codex 7 | @@ -835,7 +835,7 @@ Negative controls (`negative_checks.py`, run one at a time; each checks | `NoAbort` (liveness) | a start racing shutdown is not cancelled | `StartAnswered` | rev. 1 | | `StuckBackoff` (liveness) | shutdown does not cancel a pending restart | `ClosedLeadsToDone` | rev. 1 | -CONTROLS_NOTE +All 27 produce their counterexample on the named property when run one at a time, each in 1-5 s. (Run in parallel, TLC instances race on the standard modules TLC extracts to `/tmp`; `negative_checks.py` runs them serially.) Not modelled: handler bodies, mailboxes and requests (`Lifecycle.tla`; rank 11's models); real time (backoff delays, the window clock, deadlines From 210ac5a28a406d66f9c05d2f3645aedecd2a587a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:39:00 -0300 Subject: [PATCH 22/52] verify: DynamicSupervisor revision 3 (FIFO queue, terminate lookup, caller fairness, backoff switch, fan-out refinement, ignore, 40 controls) --- verification/tla/DynamicSupervisor.tla | 714 ++++++++++-------- verification/tla/MCDynamicSupervisor.cfg | 25 +- verification/tla/MCDynamicSupervisor.tla | 14 +- .../tla/MCDynamicSupervisorBackoff.cfg | 44 ++ .../tla/MCDynamicSupervisorBackoff.tla | 3 + .../tla/MCDynamicSupervisorCallers.cfg | 24 + .../tla/MCDynamicSupervisorCallers.tla | 3 + verification/tla/MCDynamicSupervisorDeep.cfg | 20 +- verification/tla/MCDynamicSupervisorDeep.tla | 4 +- verification/tla/MCDynamicSupervisorLive.cfg | 42 +- verification/tla/MCDynamicSupervisorLive.tla | 4 +- verification/tla/MCDynamicSupervisorThree.cfg | 45 ++ verification/tla/MCDynamicSupervisorThree.tla | 3 + verification/tla/MCDynamicSupervisorZero.cfg | 23 +- verification/tla/MCDynamicSupervisorZero.tla | 4 +- verification/tla/negative_checks.py | 42 +- 16 files changed, 626 insertions(+), 388 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorBackoff.cfg create mode 100644 verification/tla/MCDynamicSupervisorBackoff.tla create mode 100644 verification/tla/MCDynamicSupervisorCallers.cfg create mode 100644 verification/tla/MCDynamicSupervisorCallers.tla create mode 100644 verification/tla/MCDynamicSupervisorThree.cfg create mode 100644 verification/tla/MCDynamicSupervisorThree.tla diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index fb6dd3fe..f0d58d80 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -2,226 +2,262 @@ (***************************************************************************) (* Rank 12a design model (not implemented): a dynamic supervisor, OTP's *) (* simple_one_for_one / Elixir's DynamicSupervisor. Design contract: *) -(* docs/design/dynamic-supervisor.md (revision 2, after review). *) +(* docs/design/dynamic-supervisor.md (revision 3). *) (* *) -(* Slots 1..N are potential start_child calls; each is made at most once, *) -(* at any time, and each started child can get one terminate_child call. *) -(* As in OTP, one control loop (`op`) serialises the supervisor's *) -(* operations: admitting a start and constructing its first generation, *) -(* a restart decision and the restart attempt, and a terminate_child, *) -(* each runs to completion before the next. Children run, exit and clean *) -(* up concurrently. Shutdown (Close) and failure close admission at once *) -(* and cancel the operation in progress (B: tree-wide admission closure). *) +(* Slots 1..N are start_child calls made at any moment. TermCalls are *) +(* terminate_child calls, each aimed at one slot either by the actor id *) +(* the caller saw (the slot's last published generation at call time) or *) +(* by the slot (terminate_slot); several calls may target one slot. *) (* *) -(* Restart types are chosen per slot in Init (`mode`), so one run covers *) -(* every combination (a simple_one_for_one template has one type; mixing *) -(* them is the general case). Two slots suffice: every property relates *) -(* at most a pair of children (one holds the cap or restarts, the other *) -(* races it); three slots exceed 100M states with revision 2's detail. *) +(* As in OTP, the supervisor is one control loop. Requests and child *) +(* exits arrive in one FIFO queue `q` (start requests, terminate calls, *) +(* exit notices, backoff retries). The loop takes the head, runs it to *) +(* completion (`op` holds a start, restart or terminate in progress), *) +(* then the next. The exit of the child an operation is waiting for is *) +(* consumed by that operation directly, as OTP's start_child and *) +(* shutdown/2 consume it. Children run, exit and clean up concurrently. *) +(* Shutdown and failure close admission at once (B, tree-wide closure). *) (* *) -(* Abstractions: *) -(* - `open` is the admission gate and the supervisor's cancel together. *) -(* - Time is abstract. The rolling window is `windowed` (stamped grants *) -(* still in the window), released by WindowExpire at most MaxExpire *) -(* times, which also bounds the generations. Backoff is the "backoff" *) -(* state (zero-length when backoff is disabled, the default). *) -(* - The shutdown_timeout -> kill escalation is the nondeterministic *) -(* reason of StopObserved (Shutdown, or Killed after the kill). A *) -(* child that misses its deadlines, before its death point (signalled *) -(* but still running) or after it (owned cleanup), is CleanupStuck. *) -(* - The shutdown fan-out is one step (the implementation sends every *) -(* signal without awaiting between sends; each child's intervals run *) -(* from its own signal, design E3). *) +(* Backoff (opt-in) is a constant: without it the restart attempt runs in *) +(* the same loop turn as the exit decision (OTP restart/2); a failed *) +(* attempt re-queues its exit at the tail (OTP try_again_restart). *) (* *) -(* Invariants that name an event (a late commit, a late generation, a *) -(* policy violation) read history variables that GhostUpdate computes *) -(* from every step's state change, independently of the actions, so a *) -(* mutated action cannot hide its own violation. *) +(* Two fairness sets: CallerFairness has no progress for user cleanup *) +(* (CleanupDone, StuckDone, StopObserved, Ready); only deadlines fire. It *) +(* checks that callers are answered even if cleanup never ends. *) +(* Fairness adds cleanup progress, for shutdown completion and recovery. *) (* *) -(* Fault selects a negative control; negative_checks.py requires each to *) -(* violate its named property (see DYNAMIC_CONTROLS there). *) +(* Instances are bounded: the per-PR ones use two slots; targeted *) +(* three-slot instances check admission order, the cap and the shared *) +(* budget with more children. These are bounded checks, not a cutoff. *) +(* *) +(* Event invariants read history variables that GhostUpdate computes *) +(* from each step's state change, separately from the actions. *) +(* Fault selects a negative control (DYNAMIC_CONTROLS, negative_checks). *) (***************************************************************************) -EXTENDS Naturals, FiniteSets +EXTENDS Naturals, Sequences, FiniteSets CONSTANTS - N, \* number of start_child calls (slots 1..N) + N, \* start_child calls (slots 1..N) ModeSet, \* restart types a slot may have; chosen per slot in Init - MaxChildren, \* max_children - MaxRestarts, \* restart intensity (grants per window) - MaxExpire, \* how often the rolling window may release a grant + MaxChildren, MaxRestarts, MaxExpire, SelfExits, \* reasons a ready child may exit with on its own - StartFails, \* outcomes of a failed construction or readiness + StartFails, \* non-deadline outcomes of a failed construction CleanupFails, \* {FALSE} or {FALSE, TRUE}: cleanup may record a failure + TermCalls, \* terminate_child calls + CallSlot, \* CallSlot[c]: slot the call is about + CallKind, \* CallKind[c] \in {"id", "slot"} + BackoffOn, \* backoff configured (opt-in) + FanoutSteps, \* shutdown fan-out as separate steps (refinement) Fault Slots == 1..N Modes == {"Permanent", "Transient", "Temporary"} -\* "none": factory error or panic (no actor reason); "timeout": startup -\* deadline; otherwise the actor's exit reason (v0.8 reasons, design E5). Reasons == {"none", "timeout", "Normal", "Shutdown", "Panic", "Kill", "Killed"} Abnormal == {"none", "timeout", "Panic", "Kill", "Killed"} Faults == {"None", - \* revision 1 controls "IgnoreMax", "UncountedRestart", "RestartTemporary", "RestartTerminated", "AcceptDuringShutdown", "RestartAfterClose", "NoEscalate", "EarlyDone", - "Sequential", "NoAbort", "StuckBackoff", - \* revision 2 controls (review findings) - "ConcurrentStart", "StuckFailsAll", "PublishAfterClose", "OverlapGeneration", - "PublishBeforeReady", "PrematureOk", "UnchargedRetry", "ChargeFirstStart", - "TransientNormalRestart", "PermanentRetires", "RetryFirstStart", - "EarlyFailedReply", "NoRefund", "ExpireReserved", "TermReplyEarly", - "LateFailedOverwrite"} - -ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat -ASSUME MaxRestarts \in Nat /\ MaxExpire \in Nat + "Sequential", "NoAbort", "StuckBackoff", "ConcurrentStart", "StuckFailsAll", + "PublishAfterClose", "OverlapGeneration", "PublishBeforeReady", "PrematureOk", + "UnchargedRetry", "ChargeFirstStart", "TransientNormalRestart", + "PermanentRetires", "RetryFirstStart", "EarlyFailedReply", "NoRefund", + "ExpireReserved", "TermReplyEarly", "LateFailedOverwrite", + \* revision 3 + "LaterOp", "MisclassifyStuck", "EarlyRemove", "OverReserve", "LostRestart", + "TermLost", "NoDeadline", "SkipSignal", "SlotLookup", "QueuedTermOk", + "PublishAfterExit", "StuckResultLost"} + +ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat /\ MaxRestarts \in Nat /\ MaxExpire \in Nat ASSUME ModeSet \subseteq Modes /\ ModeSet # {} ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons ASSUME CleanupFails \subseteq BOOLEAN /\ FALSE \in CleanupFails -ASSUME Fault \in Faults - -\* Slot states. "idle": start_child not called. "queued": the call waits -\* for the control loop. "starting": the first generation is constructed. -\* "restarting": a replacement is constructed. "cleanup": the generation is -\* exiting or its owned cleanup has not finished. "decide": cleanup done, -\* the loop has not handled the exit yet. "backoff": a restart is pending, -\* holding a budget reservation. "stuck": cleanup missed its deadline; the -\* slot stays counted until it really ends. "removed"/"rejected": terminal. +ASSUME \A c \in TermCalls : CallSlot[c] \in Slots /\ CallKind[c] \in {"id", "slot"} +ASSUME BackoffOn \in BOOLEAN /\ FanoutSteps \in BOOLEAN /\ Fault \in Faults + States == {"idle", "queued", "starting", "running", "cleanup", "decide", - "backoff", "restarting", "stuck", "removed", "rejected"} + "backoff", "restarting", "stuck", "removed", "rejected", "ignored"} Active == {"starting", "running", "cleanup", "decide", "backoff", "restarting", "stuck"} -Terminal == {"idle", "removed", "rejected"} -\* start_child: "stopped" and "max" are pre-admission rejections (the -\* arguments are returned); "aborted" is shutdown during construction; -\* "failed" is FailedToStart. -Results == {"none", "pending", "ok", "failed", "aborted", "max", "stopped"} -\* terminate_child: "queued" waits for the loop, "pending" for removal. -TermResults == {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped"} - -Idle == [k |-> "idle", s |-> 0] +Terminal == {"idle", "removed", "rejected", "ignored"} +Idle == [k |-> "idle", s |-> 0, c |-> 0] VARIABLES - open, \* admission open: neither shut down nor failed - failed, \* the supervisor failed - exhausted, \* a restart was refused for lack of budget - sphase, \* "running", "signal" (Sequential only), "waiting", "done" - windowed, \* stamped grants still in the window - expired, \* window releases used (bounds the model) - reserved, \* slots holding a reservation (pending restarts) - stamped, \* grants ever stamped (charged restart attempts) - op, \* the control loop's current operation [k, s] - mode, \* restart type of each slot, fixed after Init - sl, \* per-slot record, see SlotInit - \* History, written only by GhostUpdate. + open, failed, exhausted, sphase, + windowed, expired, reserved, stamped, + q, \* FIFO of [k, s, c, t]: "start", "exit", "retry", "term" + ticket, \* next arrival ticket + served, \* ticket of the last operation the loop took from the head + op, \* operation in progress [k, s, c] + mode, sl, tc, + \* history (GhostUpdate only) lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, - restarts + restarts, outOfOrder -sup == <> -ghost == <> -vars == <> +sup == <> +ghost == <> +vars == <> -SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, +SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, lastPub |-> 0, reason |-> "none", cfail |-> FALSE, readyGen |-> FALSE, everReady |-> FALSE, result |-> "none", cause |-> "none", - term |-> FALSE, termRes |-> "none", stopSig |-> FALSE, - stuck |-> "none"] + term |-> FALSE, stopSig |-> FALSE, stuck |-> "none", + retryQ |-> FALSE] +\* terminate call: st \in none, queued, pending, ok, stuck, notfound, stopped; +\* g = the actor (generation) the caller named. +CallInit == [st |-> "none", g |-> 0] TypeOK == /\ open \in BOOLEAN /\ failed \in BOOLEAN /\ exhausted \in BOOLEAN /\ sphase \in {"running", "signal", "waiting", "done"} - /\ windowed \in Nat /\ expired \in 0..MaxExpire - /\ reserved \subseteq Slots /\ stamped \in Nat - /\ op.k \in {"idle", "start", "restart", "term"} /\ op.s \in 0..N - /\ \A s \in Slots : - /\ sl[s].cs \in States /\ sl[s].gen \in Nat /\ sl[s].alive \subseteq Nat - /\ sl[s].pub \in Nat /\ sl[s].reason \in Reasons - /\ sl[s].result \in Results /\ sl[s].termRes \in TermResults - /\ sl[s].cause \in {"none", "failed", "aborted"} - /\ sl[s].stuck \in {"none", "retire", "restart", "closing"} + /\ windowed \in Nat /\ expired \in 0..MaxExpire /\ reserved \subseteq Slots + /\ stamped \in Nat /\ ticket \in Nat /\ served \in Nat + /\ op.k \in {"idle", "start", "restart", "term"} + /\ \A s \in Slots : sl[s].cs \in States /\ sl[s].alive \subseteq Nat + /\ \A c \in TermCalls : + tc[c].st \in {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped"} Init == /\ open = TRUE /\ failed = FALSE /\ exhausted = FALSE /\ sphase = "running" - /\ windowed = 0 /\ expired = 0 /\ reserved = {} /\ stamped = 0 /\ op = Idle - /\ sl = [s \in Slots |-> SlotInit] + /\ windowed = 0 /\ expired = 0 /\ reserved = {} /\ stamped = 0 + /\ q = <<>> /\ ticket = 1 /\ served = 0 /\ op = Idle /\ mode \in [Slots -> ModeSet] + /\ sl = [s \in Slots |-> SlotInit] /\ tc = [c \in TermCalls |-> CallInit] /\ lateCommit = [s \in Slots |-> FALSE] /\ latePub = [s \in Slots |-> FALSE] /\ lateGenClose = [s \in Slots |-> FALSE] /\ lateGenTerm = [s \in Slots |-> FALSE] /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] - /\ restarts = 0 + /\ restarts = 0 /\ outOfOrder = FALSE + +----------------------------------------------------------------------------- +(* Helpers *) -\* Children held: counted against max_children. Counted == IF Fault = "UncountedRestart" THEN {s \in Slots : sl[s].cs \in {"starting", "running"}} ELSE {s \in Slots : sl[s].cs \in Active} +Budget == windowed + Cardinality(reserved) -LoopFree(s) == op.k = "idle" \/ op.s = s \/ Fault = "ConcurrentStart" -Release(s) == op' = IF op.s = s THEN Idle ELSE op -Take(k, s) == op' = IF Fault = "ConcurrentStart" /\ k # "term" THEN op ELSE [k |-> k, s |-> s] -Budget == windowed + Cardinality(reserved) +Enq(e) == q' = Append(q, [e EXCEPT !.t = ticket]) /\ ticket' = ticket + 1 +Rm(i) == [j \in 1..Len(q) - 1 |-> IF j < i THEN q[j] ELSE q[j + 1]] +IsHead(k, s, c) == Len(q) > 0 /\ q[1].k = k /\ q[1].s = s /\ q[1].c = c +\* Index of the exit notice of slot s, if queued. +ExitIdx(s) == {i \in 1..Len(q) : q[i].k = "exit" /\ q[i].s = s} +LoopIdle == op.k = "idle" \/ Fault = "ConcurrentStart" -\* The design's restart table, stated independently of the actions. DesignAllowsRestart(m, r, cf) == CASE m = "Temporary" -> FALSE [] m = "Permanent" -> TRUE [] m = "Transient" -> r \in {"Panic", "Kill", "Killed"} \/ cf -\* A restart of slot s is allowed now: a ready-then-exited generation under -\* the table, or a failed replacement (never ready) of a restartable child. Allowed(s) == /\ sl[s].everReady /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, sl[s].cfail) \/ ~sl[s].readyGen /\ mode[s] # "Temporary" -\* What the supervisor decides (with the policy faults). -WouldRestart(s, cf) == +WouldRestart(s) == /\ sl[s].everReady \/ Fault = "RetryFirstStart" /\ \/ mode[s] = "Permanent" /\ Fault # "PermanentRetires" \/ mode[s] = "Transient" - /\ (sl[s].reason \in Abnormal \/ cf \/ Fault = "TransientNormalRestart") + /\ (sl[s].reason \in Abnormal \/ sl[s].cfail \/ Fault = "TransientNormalRestart") \/ mode[s] # "Temporary" /\ ~sl[s].readyGen \/ mode[s] = "Temporary" /\ Fault = "RestartTemporary" -CloseGate == open' = FALSE -Fail == failed' = TRUE /\ CloseGate +Fail == failed' = TRUE /\ open' = FALSE + +\* The pending terminate call that holds the loop for slot s, if any. +HeldCall(s) == IF op.k = "term" /\ op.s = s THEN {op.c} ELSE {} ----------------------------------------------------------------------------- -(* Callers and environment *) +(* Callers *) -\* start_child(args): queued for the loop, or rejected at once if closed. +\* start_child: queued, or rejected at once (arguments returned) if closed. Request(s) == /\ sl[s].cs = "idle" - /\ sl' = IF ~open - THEN [sl EXCEPT ![s].cs = "rejected", ![s].result = "stopped"] - ELSE [sl EXCEPT ![s].cs = "queued", ![s].result = "pending"] - /\ UNCHANGED sup - -\* The loop admits a queued start: cap check, then the first generation. + /\ IF open + THEN /\ sl' = [sl EXCEPT ![s].cs = "queued", ![s].result = "pending"] + /\ Enq([k |-> "start", s |-> s, c |-> 0, t |-> 0]) + ELSE /\ sl' = [sl EXCEPT ![s].cs = "rejected", ![s].result = "stopped"] + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* The loop admits the start at the head (LaterOp: any queued start). LoopAdmit(s) == - /\ sl[s].cs = "queued" /\ LoopFree(s) + /\ sl[s].cs = "queued" /\ LoopIdle /\ open \/ Fault = "AcceptDuringShutdown" + /\ \E i \in 1..Len(q) : + /\ q[i].k = "start" /\ q[i].s = s + /\ i = 1 \/ Fault = "LaterOp" + /\ q' = Rm(i) /\ served' = q[i].t /\ IF Cardinality(Counted) >= MaxChildren /\ Fault # "IgnoreMax" THEN /\ sl' = [sl EXCEPT ![s].cs = "rejected", ![s].result = "max"] /\ UNCHANGED <> ELSE /\ sl' = [sl EXCEPT ![s].cs = "starting", ![s].gen = 1, ![s].alive = {1}, - ![s].result = IF Fault = "PrematureOk" THEN "ok" ELSE "pending"] - /\ Take("start", s) + ![s].result = IF Fault = "PrematureOk" THEN "ok" ELSE "pending"] + /\ op' = IF Fault = "ConcurrentStart" THEN op ELSE [k |-> "start", s |-> s, c |-> 0] /\ IF Fault = "ChargeFirstStart" THEN stamped' = stamped + 1 /\ windowed' = windowed + 1 ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> -\* Shutdown rejects queued starts before admission (arguments returned). +\* Shutdown answers queued starts and terminate calls (Stopped, args back). RejectQueued(s) == - /\ sl[s].cs = "queued" /\ ~open + /\ sl[s].cs = "queued" /\ ~open /\ Fault # "AcceptDuringShutdown" + /\ \E i \in 1..Len(q) : q[i].k = "start" /\ q[i].s = s /\ q' = Rm(i) /\ sl' = [sl EXCEPT ![s].cs = "rejected", ![s].result = "stopped"] - /\ UNCHANGED sup + /\ UNCHANGED <> + +\* terminate_child (by the id the caller saw) or terminate_slot. +TermCall(c) == + /\ tc[c].st = "none" /\ sl[CallSlot[c]].everReady + /\ IF open + THEN /\ tc' = [tc EXCEPT ![c] = [st |-> "queued", g |-> sl[CallSlot[c]].lastPub]] + /\ Enq([k |-> "term", s |-> CallSlot[c], c |-> c, t |-> 0]) + ELSE /\ tc' = [tc EXCEPT ![c].st = "stopped"] /\ UNCHANGED <> + /\ UNCHANGED <> + +RejectQueuedTerm(c) == + /\ tc[c].st = "queued" /\ ~open + /\ \E i \in 1..Len(q) : q[i].k = "term" /\ q[i].c = c /\ q' = Rm(i) + /\ tc' = [tc EXCEPT ![c].st = IF Fault = "QueuedTermOk" THEN "pending" ELSE "stopped"] + /\ UNCHANGED <> + +\* The loop looks the target up when it handles the call (Elixir). +LoopTerm(c) == + /\ tc[c].st = "queued" /\ op.k = "idle" /\ open /\ IsHead("term", CallSlot[c], c) + /\ q' = Tail(q) /\ served' = q[1].t + /\ LET s == CallSlot[c] + found == /\ sl[s].cs \in Active \ {"stuck"} + /\ CallKind[c] = "slot" \/ sl[s].lastPub = tc[c].g \/ Fault = "SlotLookup" + IN IF sl[s].cs = "stuck" /\ (CallKind[c] = "slot" \/ sl[s].lastPub = tc[c].g) + THEN /\ tc' = [tc EXCEPT ![c].st = "stuck"] /\ UNCHANGED <> + ELSE IF ~found + THEN /\ tc' = [tc EXCEPT ![c].st = "notfound"] /\ UNCHANGED <> + ELSE /\ tc' = [tc EXCEPT ![c].st = "pending"] + /\ sl' = IF Fault = "TermLost" THEN sl + ELSE [sl EXCEPT ![s].term = TRUE, ![s].stopSig = TRUE] + /\ op' = [k |-> "term", s |-> s, c |-> c] + /\ UNCHANGED <> + +\* The terminated child was removed: Ok, and the loop moves on. +TermResolve(c) == + /\ tc[c].st = "pending" + /\ sl[CallSlot[c]].cs = "removed" \/ Fault = "TermReplyEarly" + /\ tc' = [tc EXCEPT ![c].st = "ok"] + /\ op' = IF op.k = "term" /\ op.c = c THEN Idle ELSE op + /\ UNCHANGED <> -\* Supervisor::stop / shutdown, system shutdown, or the parent stopping it. Close == - /\ open /\ CloseGate - /\ UNCHANGED <> + /\ open /\ open' = FALSE + /\ UNCHANGED <> -\* The rolling window forgets its oldest stamped grant; reservations stay. WindowExpire == /\ expired < MaxExpire /\ IF windowed > 0 @@ -230,345 +266,409 @@ WindowExpire == /\ \E r \in reserved : reserved' = reserved \ {r} /\ UNCHANGED windowed /\ expired' = expired + 1 - /\ UNCHANGED <> - -\* terminate_child(id) by a caller holding the child's last published id. -TerminateRequest(s) == - /\ sl[s].termRes = "none" /\ sl[s].everReady - /\ sl' = [sl EXCEPT ![s].termRes = IF open THEN "queued" ELSE "stopped"] - /\ UNCHANGED sup - -\* The loop handles it: mark terminated, signal, and hold the loop until -\* the child is removed (OTP terminate_child blocks the supervisor). -LoopTerm(s) == - /\ sl[s].termRes = "queued" /\ op.k = "idle" /\ open - /\ IF sl[s].cs = "removed" - THEN sl' = [sl EXCEPT ![s].termRes = "notfound"] /\ UNCHANGED op - ELSE IF sl[s].cs = "stuck" - THEN sl' = [sl EXCEPT ![s].termRes = "stuck"] /\ UNCHANGED op - ELSE /\ sl' = [sl EXCEPT ![s].term = TRUE, ![s].stopSig = TRUE, - ![s].termRes = "pending"] - /\ op' = [k |-> "term", s |-> s] - /\ UNCHANGED <> - -\* A terminate queued when shutdown began resolves with the removal. -TermQueuedClosed(s) == - /\ sl[s].termRes = "queued" /\ ~open - /\ sl' = [sl EXCEPT ![s].termRes = "pending"] - /\ UNCHANGED sup - -TermResolve(s) == - /\ sl[s].termRes = "pending" - /\ sl[s].cs \in {"removed", "stuck"} \/ Fault = "TermReplyEarly" - /\ sl' = [sl EXCEPT ![s].termRes = IF sl[s].cs = "stuck" THEN "stuck" ELSE "ok"] - /\ IF op = [k |-> "term", s |-> s] THEN op' = Idle ELSE UNCHANGED op - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- -(* One child slot *) +(* Children *) + +Unchanged1 == UNCHANGED <> -\* started() returned while admission is open: readiness commits; for a -\* first start the caller's Ok is committed (historical from now on). +\* Readiness commits while open; a first start's Ok is committed now. Ready(s) == /\ sl[s].cs \in {"starting", "restarting"} /\ \/ open \/ Fault = "AcceptDuringShutdown" /\ sl[s].cs = "starting" \/ Fault = "PublishAfterClose" /\ sl[s].cs = "restarting" /\ sl' = [sl EXCEPT ![s].cs = "running", ![s].pub = sl[s].gen, + ![s].lastPub = sl[s].gen, ![s].readyGen = TRUE, ![s].everReady = TRUE, ![s].result = IF sl[s].cs = "starting" THEN "ok" ELSE sl[s].result] - /\ Release(s) - /\ UNCHANGED <> + /\ op' = IF op.s = s /\ op.k # "term" THEN Idle ELSE op + /\ UNCHANGED <> -\* Factory error or panic, panic or stop in started(), startup timeout. -StartFail(s, r) == - /\ sl[s].cs \in {"starting", "restarting"} /\ r \in StartFails - /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0, +\* The factory returned ignore: no actor, nothing stored (OTP {ok, undefined}). +Ignore(s) == + /\ sl[s].cs \in {"starting", "restarting"} + /\ sl' = [sl EXCEPT ![s].cs = "ignored", ![s].alive = {}, + ![s].result = IF sl[s].cs = "starting" THEN "ignored" ELSE sl[s].result] + /\ op' = IF op.s = s /\ op.k # "term" THEN Idle ELSE op + /\ UNCHANGED <> + +FailStart(s, r) == + /\ sl' = [sl EXCEPT ![s].cs = IF Fault = "EarlyRemove" /\ sl[s].cs = "starting" + THEN "removed" ELSE "cleanup", + ![s].reason = r, ![s].pub = 0, ![s].cause = IF sl[s].cs = "starting" THEN "failed" ELSE sl[s].cause, - ![s].result = IF Fault = "EarlyFailedReply" /\ sl[s].cs = "starting" + ![s].result = IF Fault \in {"EarlyFailedReply", "EarlyRemove"} + /\ sl[s].cs = "starting" THEN "failed" ELSE sl[s].result] - /\ UNCHANGED sup -\* Admission closed during construction: the attempt is cancelled. +\* Factory error or panic, panic or stop in started(). +StartFail(s, r) == + /\ sl[s].cs \in {"starting", "restarting"} /\ r \in StartFails + /\ FailStart(s, r) /\ Unchanged1 + +\* startup_timeout fired (a deadline: fair in both fairness sets). +StartTimeout(s) == + /\ sl[s].cs \in {"starting", "restarting"} + /\ FailStart(s, "timeout") /\ Unchanged1 + AbortStart(s) == /\ sl[s].cs \in {"starting", "restarting"} /\ ~open /\ Fault # "NoAbort" /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = "Shutdown", ![s].pub = 0, ![s].cause = IF sl[s].cs = "starting" THEN "aborted" ELSE sl[s].cause] - /\ UNCHANGED sup + /\ Unchanged1 SelfExit(s, r) == /\ sl[s].cs = "running" /\ r \in SelfExits - /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0] - /\ UNCHANGED sup + /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, + ![s].pub = IF Fault = "PublishAfterExit" THEN sl[s].pub ELSE 0] + /\ Unchanged1 -\* The parent exit signal (terminate_child or shutdown) reached the child: -\* it ends Shutdown, or Killed after missing shutdown_timeout (design E3). StopObserved(s, r) == /\ sl[s].cs = "running" /\ sl[s].stopSig /\ r \in {"Shutdown", "Killed"} /\ sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = r, ![s].pub = 0] - /\ UNCHANGED sup + /\ Unchanged1 -\* Publication and owned cleanup finished; a destructor or helper failure -\* may be recorded (a Transient child then restarts). +\* Owned cleanup finished. The exit is queued for the loop (OTP 'EXIT'), +\* unless the operation in progress waits for this very child; a failed +\* restart attempt gives the loop back and re-queues (try_again_restart). CleanupDone(s, f) == /\ sl[s].cs = "cleanup" /\ f \in CleanupFails /\ sl' = [sl EXCEPT ![s].cs = "decide", ![s].cfail = f, ![s].alive = sl[s].alive \ {sl[s].gen}] - /\ UNCHANGED sup - -\* The child missed its deadlines: still running after its stop signal and -\* the kill, or its cleanup is stuck. The slot stays counted and alive. -\* It fails the supervisor only when the child would have been restarted -\* (V3: no replacement while the old generation lives) or while closing; -\* a child that was retiring anyway fails only itself. + /\ Enq([k |-> "exit", s |-> s, c |-> 0, t |-> 0]) + /\ op' = IF op.k = "restart" /\ op.s = s THEN Idle ELSE op + /\ UNCHANGED <> + +\* A deadline was missed: signalled but still running after the kill and +\* the second interval, or owned cleanup past its deadline. The slot stays +\* counted. Results waiting on it are committed now and stay (historical). +\* Kind: "restart" only for a child the table would restart, judged from +\* the exit reason alone (a stall is not a recorded cleanup failure); +\* "closing" during shutdown; otherwise "retire" (fails only the child). CleanupStuck(s) == + /\ Fault # "NoDeadline" /\ sl[s].stuck = "none" /\ \/ sl[s].cs = "cleanup" \/ sl[s].cs = "running" /\ sl[s].stopSig - /\ LET kind == IF ~open THEN "closing" - ELSE IF ~sl[s].term /\ sl[s].cs = "cleanup" /\ WouldRestart(s, TRUE) + /\ LET restartable == + /\ sl[s].everReady /\ ~sl[s].term /\ sl[s].cs = "cleanup" + /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, FALSE) + \/ ~sl[s].readyGen /\ mode[s] # "Temporary" + kind == IF ~open THEN "closing" + ELSE IF restartable \/ (Fault = "MisclassifyStuck" /\ ~sl[s].term) THEN "restart" ELSE "retire" IN /\ sl' = [sl EXCEPT ![s].cs = "stuck", ![s].stuck = kind, ![s].pub = 0, ![s].result = IF sl[s].result = "pending" THEN sl[s].cause ELSE sl[s].result] /\ IF kind # "retire" \/ Fault = "StuckFailsAll" THEN Fail ELSE UNCHANGED <> - /\ IF op.s = s /\ op.k # "term" THEN op' = Idle ELSE UNCHANGED op - /\ UNCHANGED <> + /\ tc' = [c \in TermCalls |-> + IF c \in HeldCall(s) /\ Fault # "StuckResultLost" + THEN [tc[c] EXCEPT !.st = "stuck"] ELSE tc[c]] + /\ op' = IF op.s = s /\ (op.k # "term" \/ Fault # "StuckResultLost") THEN Idle ELSE op + /\ UNCHANGED <> -\* A stuck child finally ends (assumed for liveness only). StuckDone(s) == /\ sl[s].cs = "stuck" /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].alive = {}] - /\ UNCHANGED sup + /\ Unchanged1 -\* The loop handles the exit: first-start failure, removal, or a restart -\* that reserves budget first (failing at once if none is left). +\* The loop handles an exit: at the head of the queue, or directly when the +\* operation in progress waits for this child (start or terminate). Decide(s) == /\ \/ sl[s].cs = "decide" \/ Fault = "OverlapGeneration" /\ sl[s].cs = "cleanup" - /\ LoopFree(s) - /\ IF ~sl[s].everReady /\ Fault # "RetryFirstStart" - THEN /\ sl' = [sl EXCEPT ![s].cs = "removed", + /\ \/ \E i \in ExitIdx(s) : + /\ \/ i = 1 /\ LoopIdle + \/ op.s = s /\ op.k \in {"start", "term"} + /\ q' = Rm(i) + /\ served' = IF i = 1 /\ LoopIdle THEN q[i].t ELSE served + \/ Fault = "OverlapGeneration" /\ sl[s].cs = "cleanup" /\ LoopIdle + /\ UNCHANGED <> + /\ LET ownStart == op.s = s /\ op.k = "start" + IN IF ~sl[s].everReady /\ Fault # "RetryFirstStart" + THEN \* first-start failure: answered after cleanup; never retried + /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].result = IF Fault = "LateFailedOverwrite" /\ ~open THEN "aborted" ELSE sl[s].cause] - /\ UNCHANGED <> + /\ op' = IF ownStart THEN Idle ELSE op + /\ UNCHANGED <> ELSE IF ~open \/ (sl[s].term /\ Fault # "RestartTerminated") THEN /\ sl' = [sl EXCEPT ![s].cs = "removed"] - /\ UNCHANGED <> - ELSE IF WouldRestart(s, sl[s].cfail) - THEN IF Budget < MaxRestarts - THEN /\ sl' = [sl EXCEPT ![s].cs = "backoff"] - /\ reserved' = reserved \cup {s} - /\ UNCHANGED <> + /\ op' = IF ownStart THEN Idle ELSE op \* a terminate keeps the loop until TermResolve + /\ UNCHANGED <> + ELSE IF WouldRestart(s) + THEN IF Budget < MaxRestarts \/ Fault = "OverReserve" + THEN IF BackoffOn \/ Fault = "LostRestart" + THEN \* reserve, wait off the loop + /\ sl' = [sl EXCEPT ![s].cs = "backoff"] + /\ reserved' = reserved \cup {s} + /\ op' = IF ownStart THEN Idle ELSE op + /\ UNCHANGED <> + ELSE \* reserve and stamp now; the attempt runs in this turn (OTP) + /\ LET g == sl[s].gen + 1 IN + sl' = [sl EXCEPT ![s].cs = "restarting", ![s].gen = g, + ![s].alive = sl[s].alive \cup {g}, + ![s].pub = IF Fault = "PublishBeforeReady" THEN g ELSE 0, + ![s].reason = "none", ![s].cfail = FALSE, ![s].readyGen = FALSE] + /\ IF Fault = "UnchargedRetry" /\ ~sl[s].readyGen + THEN UNCHANGED <> + ELSE stamped' = stamped + 1 /\ windowed' = windowed + 1 + /\ op' = [k |-> "restart", s |-> s, c |-> 0] + /\ UNCHANGED <> ELSE /\ exhausted' = TRUE /\ sl' = [sl EXCEPT ![s].cs = "removed"] /\ IF Fault = "NoEscalate" THEN UNCHANGED <> ELSE Fail - /\ UNCHANGED reserved + /\ op' = IF ownStart THEN Idle ELSE op + /\ UNCHANGED <> ELSE /\ sl' = [sl EXCEPT ![s].cs = "removed"] - /\ UNCHANGED <> - /\ Release(s) - /\ UNCHANGED <> - -\* Backoff elapsed: stamp the reservation and construct the replacement. + /\ op' = IF ownStart THEN Idle ELSE op + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* The backoff timer fires: the retry joins the queue. +BackoffFire(s) == + /\ sl[s].cs = "backoff" /\ ~sl[s].retryQ /\ open /\ Fault # "LostRestart" + /\ sl' = [sl EXCEPT ![s].retryQ = TRUE] + /\ Enq([k |-> "retry", s |-> s, c |-> 0, t |-> 0]) + /\ UNCHANGED <> + +\* The loop takes the retry: stamp the reservation, construct. BackoffDone(s) == - /\ sl[s].cs = "backoff" /\ LoopFree(s) + /\ sl[s].cs = "backoff" /\ sl[s].retryQ /\ LoopIdle + /\ IsHead("retry", s, 0) /\ open \/ Fault = "RestartAfterClose" /\ ~sl[s].term \/ Fault = "RestartTerminated" - /\ LET g == sl[s].gen + 1 - charge == ~(Fault = "UnchargedRetry" /\ ~sl[s].readyGen) - IN /\ sl' = [sl EXCEPT ![s].cs = "restarting", ![s].gen = g, - ![s].alive = sl[s].alive \cup {g}, - ![s].pub = IF Fault = "PublishBeforeReady" THEN g ELSE 0, - ![s].reason = "none", ![s].cfail = FALSE, - ![s].readyGen = FALSE] - /\ reserved' = reserved \ {s} - /\ IF charge THEN stamped' = stamped + 1 /\ windowed' = windowed + 1 - ELSE UNCHANGED <> - /\ Take("restart", s) - /\ UNCHANGED <> - -\* Shutdown or terminate_child cancels a pending restart and refunds it. + /\ q' = Tail(q) /\ served' = q[1].t + /\ LET g == sl[s].gen + 1 IN + sl' = [sl EXCEPT ![s].cs = "restarting", ![s].gen = g, + ![s].alive = sl[s].alive \cup {g}, ![s].retryQ = FALSE, + ![s].pub = IF Fault = "PublishBeforeReady" THEN g ELSE 0, + ![s].reason = "none", ![s].cfail = FALSE, ![s].readyGen = FALSE] + /\ reserved' = reserved \ {s} + /\ IF Fault = "UnchargedRetry" /\ ~sl[s].readyGen + THEN UNCHANGED <> + ELSE stamped' = stamped + 1 /\ windowed' = windowed + 1 + /\ op' = IF Fault = "ConcurrentStart" THEN op ELSE [k |-> "restart", s |-> s, c |-> 0] + /\ UNCHANGED <> + +\* Shutdown or terminate cancels a pending restart and refunds it. CancelBackoff(s) == /\ sl[s].cs = "backoff" /\ \/ ~open /\ Fault # "StuckBackoff" \/ sl[s].term /\ Fault # "RestartTerminated" - /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].retryQ = FALSE] + /\ q' = SelectSeq(q, LAMBDA e : ~(e.k = "retry" /\ e.s = s)) /\ reserved' = IF Fault = "NoRefund" THEN reserved ELSE reserved \ {s} - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- -(* Supervisor shutdown: every child is signalled without awaiting another *) +(* Shutdown *) +\* Atomic fan-out: every admitted child signalled in one step. SupSignal == - /\ sphase = "running" /\ ~open /\ Fault # "Sequential" - /\ sl' = [s \in Slots |-> [sl[s] EXCEPT !.stopSig = TRUE]] + /\ ~FanoutSteps /\ sphase = "running" /\ ~open + /\ sl' = [s \in Slots |-> + IF Fault = "SkipSignal" /\ s = 1 THEN sl[s] + ELSE [sl[s] EXCEPT !.stopSig = TRUE]] /\ sphase' = "waiting" - /\ UNCHANGED <> - -\* Control: one child at a time, the next only after the previous ended. -SupSignalOne == - /\ sphase \in {"running", "signal"} /\ ~open /\ Fault = "Sequential" - /\ \A s \in Slots : sl[s].stopSig => sl[s].cs \in Terminal \cup {"stuck"} - /\ LET todo == {s \in Slots : ~sl[s].stopSig /\ sl[s].cs \notin Terminal} + /\ UNCHANGED <> + +\* Refinement: one signal per step, interleaved with the children. +\* Sequential waits for each signalled child to finish first. +SupSignalStep == + /\ FanoutSteps /\ sphase \in {"running", "signal"} /\ ~open + /\ Fault = "Sequential" => \A s \in Slots : sl[s].stopSig => sl[s].cs \notin Active + /\ LET todo == {s \in Slots : ~sl[s].stopSig /\ sl[s].cs \in Active} IN IF todo = {} THEN sphase' = "waiting" /\ UNCHANGED sl ELSE \E s \in todo : sl' = [sl EXCEPT ![s].stopSig = TRUE] /\ sphase' = "signal" - /\ UNCHANGED <> + /\ UNCHANGED <> SupDone == /\ sphase = "waiting" /\ (\A s \in Slots : sl[s].cs \in Terminal) \/ Fault = "EarlyDone" /\ sphase' = "done" - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- Grew(s) == sl'[s].gen > sl[s].gen +\* A restart was decided (reservation, attempt or exhaustion) for slot s. +Decided(s) == + /\ sl[s].cs = "decide" + /\ \/ sl'[s].cs \in {"backoff", "restarting"} + \/ exhausted' /\ ~exhausted /\ sl'[s].cs = "removed" -\* History, computed from each step's effect, not from the action taken. GhostUpdate == /\ lateCommit' = [s \in Slots |-> lateCommit[s] \/ (sl'[s].result = "ok" /\ sl[s].result # "ok" /\ ~open)] /\ latePub' = [s \in Slots |-> latePub[s] \/ - (sl'[s].pub # 0 /\ sl[s].pub = 0 /\ (~open \/ sl[s].term))] + (sl'[s].pub # sl[s].pub /\ sl'[s].pub # 0 /\ (~open \/ sl[s].term))] /\ lateGenClose' = [s \in Slots |-> lateGenClose[s] \/ (Grew(s) /\ ~open)] /\ lateGenTerm' = [s \in Slots |-> lateGenTerm[s] \/ (Grew(s) /\ sl[s].term)] /\ badRestart' = [s \in Slots |-> badRestart[s] \/ - (Grew(s) /\ sl[s].gen > 0 /\ ~Allowed(s))] + ((Grew(s) /\ sl[s].gen > 0 \/ Decided(s)) /\ ~Allowed(s))] /\ badRetire' = [s \in Slots |-> badRetire[s] \/ (sl[s].cs = "decide" /\ sl'[s].cs = "removed" /\ open' /\ ~sl[s].term /\ Allowed(s))] /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) + /\ outOfOrder' = (outOfOrder \/ (served' # served /\ served' < served)) /\ UNCHANGED mode SlotStep(s) == \/ Request(s) \/ LoopAdmit(s) \/ RejectQueued(s) - \/ TerminateRequest(s) \/ LoopTerm(s) \/ TermQueuedClosed(s) \/ TermResolve(s) - \/ Ready(s) \/ \E r \in StartFails : StartFail(s, r) \/ AbortStart(s) + \/ Ready(s) \/ Ignore(s) \/ \E r \in StartFails : StartFail(s, r) + \/ StartTimeout(s) \/ AbortStart(s) \/ \E r \in SelfExits : SelfExit(s, r) \/ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r) \/ \E f \in CleanupFails : CleanupDone(s, f) \/ CleanupStuck(s) \/ StuckDone(s) - \/ Decide(s) \/ BackoffDone(s) \/ CancelBackoff(s) + \/ Decide(s) \/ BackoffFire(s) \/ BackoffDone(s) \/ CancelBackoff(s) + +CallStep(c) == TermCall(c) \/ RejectQueuedTerm(c) \/ LoopTerm(c) \/ TermResolve(c) Act == - \/ Close \/ WindowExpire \/ SupSignal \/ SupSignalOne \/ SupDone + \/ Close \/ WindowExpire \/ SupSignal \/ SupSignalStep \/ SupDone \/ \E s \in Slots : SlotStep(s) + \/ \E c \in TermCalls : CallStep(c) Next == Act /\ GhostUpdate -\* Progress: the loop, construction, stops, cleanup and stuck children -\* eventually move (crashes, start failures and getting stuck are not fair). +F(A) == WF_vars(A /\ GhostUpdate) + +\* Loop work and deadlines only: no user cleanup is assumed to finish. +CallerFairness == + /\ \A s \in Slots : + /\ F(LoopAdmit(s)) /\ F(RejectQueued(s)) /\ F(StartTimeout(s)) + /\ F(AbortStart(s)) /\ F(CleanupStuck(s)) /\ F(Decide(s)) + /\ F(BackoffFire(s)) /\ F(BackoffDone(s)) /\ F(CancelBackoff(s)) + /\ \A c \in TermCalls : F(RejectQueuedTerm(c)) /\ F(LoopTerm(c)) /\ F(TermResolve(c)) + /\ F(SupSignal) /\ F(SupSignalStep) /\ F(SupDone) + +\* Plus progress of children and cleanup. Fairness == + /\ CallerFairness /\ \A s \in Slots : - /\ WF_vars(LoopAdmit(s) /\ GhostUpdate) /\ WF_vars(RejectQueued(s) /\ GhostUpdate) - /\ WF_vars(LoopTerm(s) /\ GhostUpdate) /\ WF_vars(TermQueuedClosed(s) /\ GhostUpdate) - /\ WF_vars(TermResolve(s) /\ GhostUpdate) - /\ WF_vars(Ready(s) /\ GhostUpdate) /\ WF_vars(AbortStart(s) /\ GhostUpdate) - /\ WF_vars((\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) /\ GhostUpdate) - /\ WF_vars((\E f \in CleanupFails : CleanupDone(s, f)) /\ GhostUpdate) - /\ WF_vars(StuckDone(s) /\ GhostUpdate) - /\ WF_vars(Decide(s) /\ GhostUpdate) /\ WF_vars(BackoffDone(s) /\ GhostUpdate) - /\ WF_vars(CancelBackoff(s) /\ GhostUpdate) - /\ WF_vars(SupSignal /\ GhostUpdate) /\ WF_vars(SupSignalOne /\ GhostUpdate) - /\ WF_vars(SupDone /\ GhostUpdate) - -Spec == Init /\ [][Next]_vars /\ Fairness + /\ F(Ready(s)) + /\ F(\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) + /\ F(\E f \in CleanupFails : CleanupDone(s, f)) + /\ F(StuckDone(s)) + +Spec == Init /\ [][Next]_vars /\ Fairness +CallerSpec == Init /\ [][Next]_vars /\ CallerFairness ----------------------------------------------------------------------------- (* Safety *) -\* A slot never has two live generations (V3); alive accumulates and only -\* the generation whose cleanup finished is removed. AtMostOneGeneration == \A s \in Slots : Cardinality(sl[s].alive) <= 1 -\* current() returns only the live, latest, ready generation (V4). PublishedIsLive == \A s \in Slots : sl[s].pub # 0 => - sl[s].pub = sl[s].gen /\ sl[s].pub \in sl[s].alive /\ sl[s].readyGen + /\ sl[s].cs = "running" /\ sl[s].pub = sl[s].gen + /\ sl[s].pub \in sl[s].alive /\ sl[s].readyGen NoChildOutlivesSupervisor == sphase = "done" => \A s \in Slots : sl[s].alive = {} /\ sl[s].cs \in Terminal -\* Children held, restarting and stuck ones included, never exceed the cap. MaxChildrenBound == Cardinality({s \in Slots : sl[s].cs \in Active}) <= MaxChildren -\* The control loop constructs one generation at a time (OTP). SerialConstruction == Cardinality({s \in Slots : sl[s].cs \in {"starting", "restarting"}}) <= 1 -TemporaryNeverRestarted == \A s \in Slots : mode[s] = "Temporary" => sl[s].gen <= 1 +\* The loop takes queued operations in arrival order. +FifoOrder == ~outOfOrder -\* No generation begins after terminate_child, or after admission closed. +TemporaryNeverRestarted == \A s \in Slots : mode[s] = "Temporary" => sl[s].gen <= 1 TerminatedNeverRestarted == \A s \in Slots : ~lateGenTerm[s] NoGenerationAfterClose == \A s \in Slots : ~lateGenClose[s] - -\* No generation is published after terminate_child or close. NoPublishAfterStop == \A s \in Slots : ~latePub[s] - -\* A start_child success is committed only while admission is open. NoStartAfterShutdown == \A s \in Slots : ~lateCommit[s] -\* Restarts follow the table; a restartable exit is never retired while open. +\* Restart decisions (reservation, attempt, exhaustion) only for exits the +\* table allows; no restartable exit is retired while open. RestartFollowsPolicy == \A s \in Slots : ~badRestart[s] /\ ~badRetire[s] -\* A failed first start is never retried. FirstStartNeverRetried == \A s \in Slots : sl[s].gen > 1 => sl[s].everReady -\* FailedToStart / aborted is returned only after cleanup, or at the stuck -\* deadline (the slot then stays counted). +\* FailedToStart/Aborted only once cleanup really finished, or with the +\* cleanup deadline failure recorded. FailedAfterCleanup == \A s \in Slots : sl[s].result \in {"failed", "aborted"} => - sl[s].cs \in {"removed", "stuck"} + \/ sl[s].cs = "removed" /\ sl[s].alive = {} + \/ sl[s].stuck # "none" -\* The error names what happened to the attempt, not the gate afterwards. FailureCauseKept == \A s \in Slots : sl[s].result \in {"failed", "aborted"} => sl[s].result = sl[s].cause -\* Every restart attempt is charged exactly once; first starts never. ChargedRestarts == restarts = stamped - -\* Reservations are exactly the pending restarts. ReservationsMatch == reserved = {s \in Slots : sl[s].cs = "backoff"} - BudgetBounded == Budget <= MaxRestarts - ExhaustionCloses == exhausted => failed /\ ~open -\* A child stuck while retiring fails only itself. +\* The supervisor fails only by exhaustion, a stuck child the table would +\* restart, or while closing; and a "restart" classification is checked +\* against the table, not trusted. StuckIsolated == - failed => exhausted \/ \E s \in Slots : sl[s].stuck \in {"restart", "closing"} + /\ failed => exhausted \/ \E s \in Slots : sl[s].stuck \in {"restart", "closing"} + /\ \A s \in Slots : sl[s].stuck = "restart" => + /\ ~sl[s].term /\ sl[s].everReady + /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, FALSE) + \/ ~sl[s].readyGen /\ mode[s] # "Temporary" ConcurrentShutdown == - sphase # "running" => \A s \in Slots : sl[s].cs = "running" => sl[s].stopSig + sphase \in {"waiting", "done"} => \A s \in Slots : sl[s].cs = "running" => sl[s].stopSig OkMeansReady == \A s \in Slots : sl[s].result = "ok" => sl[s].everReady -\* terminate_child returns Ok only once the child was removed. -TermOkMeansRemoved == - \A s \in Slots : sl[s].termRes = "ok" => sl[s].cs = "removed" /\ sl[s].alive = {} +\* terminate results: Ok only after removal; Stuck stays Stuck; a call +\* still queued at shutdown is Stopped, never Ok; an id call never reaches a +\* generation published after the call was made. +TermResultsSound == + \A c \in TermCalls : + LET s == CallSlot[c] IN + /\ tc[c].st = "ok" => + sl[s].cs = "removed" /\ sl[s].alive = {} /\ sl[s].stuck = "none" + /\ tc[c].st = "pending" => op = [k |-> "term", s |-> s, c |-> c] + /\ tc[c].st = "pending" /\ CallKind[c] = "id" => sl[s].lastPub = tc[c].g Safety == /\ TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ NoChildOutlivesSupervisor - /\ MaxChildrenBound /\ SerialConstruction /\ TemporaryNeverRestarted + /\ MaxChildrenBound /\ SerialConstruction /\ FifoOrder /\ TemporaryNeverRestarted /\ TerminatedNeverRestarted /\ NoGenerationAfterClose /\ NoPublishAfterStop /\ NoStartAfterShutdown /\ RestartFollowsPolicy /\ FirstStartNeverRetried /\ FailedAfterCleanup /\ FailureCauseKept /\ ChargedRestarts /\ ReservationsMatch /\ BudgetBounded /\ ExhaustionCloses /\ StuckIsolated /\ ConcurrentShutdown - /\ OkMeansReady /\ TermOkMeansRemoved + /\ OkMeansReady /\ TermResultsSound NeverReopens == [][open' => open]_vars +\* A committed terminate result never changes. +TermResultStable == + [][\A c \in TermCalls : + tc[c].st \in {"ok", "stuck", "notfound", "stopped"} => tc'[c].st = tc[c].st]_vars -(* Liveness (assumes stuck children eventually end) *) -ClosedLeadsToDone == (~open) ~> (sphase = "done") +(* Liveness *) +\* Under CallerSpec (cleanup may never end): StartAnswered == \A s \in Slots : (sl[s].result = "pending") ~> (sl[s].result # "pending") -TerminateAnswered == \A s \in Slots : - (sl[s].termRes \in {"queued", "pending"}) ~> (sl[s].termRes \notin {"queued", "pending"}) -\* A restartable exit is restarted, or superseded by terminate, shutdown, -\* exhaustion or a stuck cleanup. +TerminateAnswered == \A c \in TermCalls : + (tc[c].st \in {"queued", "pending"}) ~> (tc[c].st \notin {"queued", "pending"}) +\* Every admitted child is signalled even if siblings never finish. +FanoutCompletes == (~open) ~> (sphase \in {"waiting", "done"}) +\* Under Spec (cleanup ends): +ClosedLeadsToDone == (~open) ~> (sphase = "done") RecoveryProgress == \A s \in Slots : (sl[s].cs \in {"decide", "backoff", "restarting"} /\ sl[s].everReady) - ~> (sl[s].cs \in {"running", "removed", "stuck"}) + ~> (sl[s].cs \in {"running", "removed", "stuck", "ignored"}) ============================================================================= diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index 827a2abf..802b0eae 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -1,25 +1,31 @@ -\* Per PR: two start_child calls, every pair of restart types, each child -\* terminable, max_children 1 (one holds it while the other races), budget -\* 1 with one window release; exits Normal/Panic, start failures factory -\* and Normal-before-readiness, cleanup failures. Safety and NeverReopens; -\* every safety negative control runs here. Liveness: MCDynamicSupervisorLive. +\* Per PR, safety: two start_child calls (every pair of restart types), +\* two terminate calls on slot 1 (by id and by slot), max_children 1, +\* budget 1 with one window release, no backoff (the default), exits +\* Normal/Panic, start failures factory/Normal/timeout, cleanup failures. +\* The safety negative controls run here unless noted. SPECIFICATION Spec CONSTANTS N = 2 ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = FALSE + FanoutSteps = FALSE Fault = "None" - MaxChildren = 1 - MaxRestarts = 1 - MaxExpire = 1 INVARIANT TypeOK INVARIANT AtMostOneGeneration INVARIANT PublishedIsLive INVARIANT NoChildOutlivesSupervisor INVARIANT MaxChildrenBound INVARIANT SerialConstruction +INVARIANT FifoOrder INVARIANT TemporaryNeverRestarted INVARIANT TerminatedNeverRestarted INVARIANT NoGenerationAfterClose @@ -36,5 +42,6 @@ INVARIANT ExhaustionCloses INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady -INVARIANT TermOkMeansRemoved +INVARIANT TermResultsSound PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla index 49d4c9bb..ca1407d0 100644 --- a/verification/tla/MCDynamicSupervisor.tla +++ b/verification/tla/MCDynamicSupervisor.tla @@ -1,5 +1,17 @@ ------------------------ MODULE MCDynamicSupervisor ------------------------ (* Model-checking instances for DynamicSupervisor (rank 12a design). *) -(* Restart types come from ModeSet in the .cfg files. *) EXTENDS DynamicSupervisor + +\* Two terminate calls on slot 1: one by the id the caller saw, one by slot. +MCCalls2 == {1, 2} +MCCallSlot2 == [c \in MCCalls2 |-> 1] +MCCallKind2 == [c \in MCCalls2 |-> IF c = 1 THEN "id" ELSE "slot"] +\* One terminate call by id. +MCCalls1 == {1} +MCCallSlot1 == [c \in MCCalls1 |-> 1] +MCCallKind1 == [c \in MCCalls1 |-> "id"] +\* No terminate calls. +MCCalls0 == {} +MCCallSlot0 == [c \in MCCalls0 |-> 1] +MCCallKind0 == [c \in MCCalls0 |-> "id"] ============================================================================= diff --git a/verification/tla/MCDynamicSupervisorBackoff.cfg b/verification/tla/MCDynamicSupervisorBackoff.cfg new file mode 100644 index 00000000..45688b4f --- /dev/null +++ b/verification/tla/MCDynamicSupervisorBackoff.cfg @@ -0,0 +1,44 @@ +\* Per PR, safety with backoff configured: reservation, timer, retry +\* through the queue, refund on cancel. Reservation controls run here. +SPECIFICATION Spec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = TRUE + FanoutSteps = FALSE + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT FifoOrder +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermResultsSound +PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorBackoff.tla b/verification/tla/MCDynamicSupervisorBackoff.tla new file mode 100644 index 00000000..f3c2683d --- /dev/null +++ b/verification/tla/MCDynamicSupervisorBackoff.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorBackoff ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/MCDynamicSupervisorCallers.cfg b/verification/tla/MCDynamicSupervisorCallers.cfg new file mode 100644 index 00000000..bf1846a1 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorCallers.cfg @@ -0,0 +1,24 @@ +\* Per PR, liveness under CallerSpec (no user cleanup ever assumed to +\* finish, only deadlines fire): every start_child and terminate_child is +\* answered, and the fan-out, as separate steps (refinement of the atomic +\* SupSignal), signals every admitted child even if siblings never finish. +SPECIFICATION CallerSpec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 2 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none"} + CleanupFails = {FALSE} + TermCalls <- MCCalls1 + CallSlot <- MCCallSlot1 + CallKind <- MCCallKind1 + BackoffOn = FALSE + FanoutSteps = TRUE + Fault = "None" +INVARIANT TypeOK +PROPERTY StartAnswered +PROPERTY TerminateAnswered +PROPERTY FanoutCompletes diff --git a/verification/tla/MCDynamicSupervisorCallers.tla b/verification/tla/MCDynamicSupervisorCallers.tla new file mode 100644 index 00000000..731f643a --- /dev/null +++ b/verification/tla/MCDynamicSupervisorCallers.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorCallers ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index cd54e0c0..7f08e524 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,23 +1,28 @@ -\* Weekly (--deep): as the per-PR instance with budget 2 and two window -\* releases (up to four restarts) and max_children 2 (both children held -\* together while restarting). Safety and NeverReopens. +\* Weekly (--deep): budget 2, two window releases, max_children 2, +\* backoff on, two terminate calls. Safety. SPECIFICATION Spec CONSTANTS N = 2 ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 2 + MaxRestarts = 2 + MaxExpire = 2 SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = TRUE + FanoutSteps = FALSE Fault = "None" - MaxChildren = 2 - MaxRestarts = 2 - MaxExpire = 2 INVARIANT TypeOK INVARIANT AtMostOneGeneration INVARIANT PublishedIsLive INVARIANT NoChildOutlivesSupervisor INVARIANT MaxChildrenBound INVARIANT SerialConstruction +INVARIANT FifoOrder INVARIANT TemporaryNeverRestarted INVARIANT TerminatedNeverRestarted INVARIANT NoGenerationAfterClose @@ -34,5 +39,6 @@ INVARIANT ExhaustionCloses INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady -INVARIANT TermOkMeansRemoved +INVARIANT TermResultsSound PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorDeep.tla b/verification/tla/MCDynamicSupervisorDeep.tla index 05e74666..2b821daf 100644 --- a/verification/tla/MCDynamicSupervisorDeep.tla +++ b/verification/tla/MCDynamicSupervisorDeep.tla @@ -1,3 +1,3 @@ ----------------------- MODULE MCDynamicSupervisorDeep ---------------------- +---- MODULE MCDynamicSupervisorDeep ---- EXTENDS MCDynamicSupervisor -============================================================================= +==== diff --git a/verification/tla/MCDynamicSupervisorLive.cfg b/verification/tla/MCDynamicSupervisorLive.cfg index dda48fc2..81589d40 100644 --- a/verification/tla/MCDynamicSupervisorLive.cfg +++ b/verification/tla/MCDynamicSupervisorLive.cfg @@ -1,43 +1,21 @@ -\* Per PR, liveness: as MCDynamicSupervisor without recorded cleanup -\* failures and with factory failures only, to keep the liveness graph -\* small. ClosedLeadsToDone, StartAnswered, TerminateAnswered, -\* RecoveryProgress; the liveness negative controls run here. +\* Per PR, liveness under Spec (cleanup eventually ends): shutdown +\* completes, recovery progresses. Backoff on, one terminate call. SPECIFICATION Spec CONSTANTS N = 2 ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 SelfExits = {"Normal", "Panic"} StartFails = {"none"} CleanupFails = {FALSE} + TermCalls <- MCCalls1 + CallSlot <- MCCallSlot1 + CallKind <- MCCallKind1 + BackoffOn = TRUE + FanoutSteps = FALSE Fault = "None" - MaxChildren = 1 - MaxRestarts = 1 - MaxExpire = 1 INVARIANT TypeOK -INVARIANT AtMostOneGeneration -INVARIANT PublishedIsLive -INVARIANT NoChildOutlivesSupervisor -INVARIANT MaxChildrenBound -INVARIANT SerialConstruction -INVARIANT TemporaryNeverRestarted -INVARIANT TerminatedNeverRestarted -INVARIANT NoGenerationAfterClose -INVARIANT NoPublishAfterStop -INVARIANT NoStartAfterShutdown -INVARIANT RestartFollowsPolicy -INVARIANT FirstStartNeverRetried -INVARIANT FailedAfterCleanup -INVARIANT FailureCauseKept -INVARIANT ChargedRestarts -INVARIANT ReservationsMatch -INVARIANT BudgetBounded -INVARIANT ExhaustionCloses -INVARIANT StuckIsolated -INVARIANT ConcurrentShutdown -INVARIANT OkMeansReady -INVARIANT TermOkMeansRemoved -PROPERTY NeverReopens PROPERTY ClosedLeadsToDone -PROPERTY StartAnswered -PROPERTY TerminateAnswered PROPERTY RecoveryProgress diff --git a/verification/tla/MCDynamicSupervisorLive.tla b/verification/tla/MCDynamicSupervisorLive.tla index b83af0fb..ad366e00 100644 --- a/verification/tla/MCDynamicSupervisorLive.tla +++ b/verification/tla/MCDynamicSupervisorLive.tla @@ -1,3 +1,3 @@ ------------------------ MODULE MCDynamicSupervisorLive ----------------------- +---- MODULE MCDynamicSupervisorLive ---- EXTENDS MCDynamicSupervisor -============================================================================= +==== diff --git a/verification/tla/MCDynamicSupervisorThree.cfg b/verification/tla/MCDynamicSupervisorThree.cfg new file mode 100644 index 00000000..199e204c --- /dev/null +++ b/verification/tla/MCDynamicSupervisorThree.cfg @@ -0,0 +1,45 @@ +\* Targeted, three start_child calls (Permanent, crashing): FIFO +\* admission, max_children 2 with a restarting child holding its slot, +\* and two restartable children sharing a budget of 1. Safety. +SPECIFICATION Spec +CONSTANTS + N = 3 + ModeSet = {"Permanent"} + MaxChildren = 2 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Panic"} + StartFails = {} + CleanupFails = {FALSE} + TermCalls <- MCCalls0 + CallSlot <- MCCallSlot0 + CallKind <- MCCallKind0 + BackoffOn = FALSE + FanoutSteps = FALSE + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT FifoOrder +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermResultsSound +PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorThree.tla b/verification/tla/MCDynamicSupervisorThree.tla new file mode 100644 index 00000000..a4e39f7f --- /dev/null +++ b/verification/tla/MCDynamicSupervisorThree.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorThree ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg index 487a6e1f..6c3a691c 100644 --- a/verification/tla/MCDynamicSupervisorZero.cfg +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -1,22 +1,28 @@ -\* Per PR, zero budget: max_restarts 0, so the first restartable exit fails -\* the supervisor at once, without entering backoff. +\* Per PR, safety with max_restarts 0: the first restartable exit fails +\* the supervisor at once. SPECIFICATION Spec CONSTANTS N = 2 ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 0 + MaxExpire = 0 SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = FALSE + FanoutSteps = FALSE Fault = "None" - MaxChildren = 1 - MaxRestarts = 0 - MaxExpire = 0 INVARIANT TypeOK INVARIANT AtMostOneGeneration INVARIANT PublishedIsLive INVARIANT NoChildOutlivesSupervisor INVARIANT MaxChildrenBound INVARIANT SerialConstruction +INVARIANT FifoOrder INVARIANT TemporaryNeverRestarted INVARIANT TerminatedNeverRestarted INVARIANT NoGenerationAfterClose @@ -33,9 +39,6 @@ INVARIANT ExhaustionCloses INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady -INVARIANT TermOkMeansRemoved +INVARIANT TermResultsSound PROPERTY NeverReopens -PROPERTY ClosedLeadsToDone -PROPERTY StartAnswered -PROPERTY TerminateAnswered -PROPERTY RecoveryProgress +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorZero.tla b/verification/tla/MCDynamicSupervisorZero.tla index ee809910..a603fe12 100644 --- a/verification/tla/MCDynamicSupervisorZero.tla +++ b/verification/tla/MCDynamicSupervisorZero.tla @@ -1,3 +1,3 @@ ------------------------ MODULE MCDynamicSupervisorZero ----------------------- +---- MODULE MCDynamicSupervisorZero ---- EXTENDS MCDynamicSupervisor -============================================================================= +==== diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index d5022345..8efde8dd 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -186,43 +186,53 @@ def named_temporal(module, fault, prop, tlc, constants=None): ] LIFECYCLE_SMALL = {"Reqs": "{}", "MaxSignals": "0", "DropEnabled": "FALSE"} -# Rank 12a design model (docs/design/dynamic-supervisor.md): each mutation -# against the invariant it must break, on the per-PR configuration. -# Rank 12a design model (docs/design/dynamic-supervisor.md, revision 2): -# each mutation against the invariant it must break, on the per-PR -# configuration. Run one at a time (parallel TLC runs race on the standard -# modules TLC extracts to /tmp). +# Rank 12a design model (docs/design/dynamic-supervisor.md, revision 3): +# each mutation against the property it must break. Every invariant and +# liveness property of DynamicSupervisor.tla has at least one. Run one at a +# time (parallel TLC runs race on the standard modules TLC extracts to /tmp). DYNAMIC_CONTROLS = [ ("MCDynamicSupervisor", "IgnoreMax", "MaxChildrenBound"), ("MCDynamicSupervisor", "UncountedRestart", "MaxChildrenBound"), ("MCDynamicSupervisor", "RestartTemporary", "TemporaryNeverRestarted"), ("MCDynamicSupervisor", "RestartTerminated", "TerminatedNeverRestarted"), ("MCDynamicSupervisor", "AcceptDuringShutdown", "NoStartAfterShutdown"), - ("MCDynamicSupervisor", "RestartAfterClose", "NoGenerationAfterClose"), + ("MCDynamicSupervisorBackoff", "RestartAfterClose", "NoGenerationAfterClose"), ("MCDynamicSupervisor", "NoEscalate", "ExhaustionCloses"), ("MCDynamicSupervisor", "EarlyDone", "NoChildOutlivesSupervisor"), - ("MCDynamicSupervisor", "Sequential", "ConcurrentShutdown"), - # Two constructions at once need room for two children. + ("MCDynamicSupervisor", "SkipSignal", "ConcurrentShutdown"), ("MCDynamicSupervisor", "ConcurrentStart", "SerialConstruction", {"MaxChildren": "2"}), + ("MCDynamicSupervisor", "LaterOp", "FifoOrder"), ("MCDynamicSupervisor", "StuckFailsAll", "StuckIsolated"), + ("MCDynamicSupervisor", "MisclassifyStuck", "StuckIsolated"), ("MCDynamicSupervisor", "PublishAfterClose", "NoPublishAfterStop"), + ("MCDynamicSupervisor", "PublishAfterExit", "PublishedIsLive"), ("MCDynamicSupervisor", "OverlapGeneration", "AtMostOneGeneration"), ("MCDynamicSupervisor", "PublishBeforeReady", "PublishedIsLive"), ("MCDynamicSupervisor", "PrematureOk", "OkMeansReady"), ("MCDynamicSupervisor", "UnchargedRetry", "ChargedRestarts"), ("MCDynamicSupervisor", "ChargeFirstStart", "ChargedRestarts"), ("MCDynamicSupervisor", "TransientNormalRestart", "RestartFollowsPolicy"), + ("MCDynamicSupervisor", "TransientNormalRestart", "RestartFollowsPolicy", {"MaxRestarts": "0", "MaxExpire": "0"}), ("MCDynamicSupervisor", "PermanentRetires", "RestartFollowsPolicy"), ("MCDynamicSupervisor", "RetryFirstStart", "FirstStartNeverRetried"), ("MCDynamicSupervisor", "EarlyFailedReply", "FailedAfterCleanup"), + ("MCDynamicSupervisor", "EarlyRemove", "FailedAfterCleanup"), ("MCDynamicSupervisor", "LateFailedOverwrite", "FailureCauseKept"), - ("MCDynamicSupervisor", "NoRefund", "ReservationsMatch"), - ("MCDynamicSupervisor", "ExpireReserved", "ReservationsMatch"), - ("MCDynamicSupervisor", "TermReplyEarly", "TermOkMeansRemoved"), + ("MCDynamicSupervisorBackoff", "NoRefund", "ReservationsMatch"), + ("MCDynamicSupervisorBackoff", "ExpireReserved", "ReservationsMatch"), + ("MCDynamicSupervisor", "OverReserve", "BudgetBounded"), + ("MCDynamicSupervisor", "TermReplyEarly", "TermResultsSound"), + ("MCDynamicSupervisor", "SlotLookup", "TermResultsSound"), + ("MCDynamicSupervisor", "QueuedTermOk", "TermResultsSound"), + ("MCDynamicSupervisor", "StuckResultLost", "TermResultsSound"), ] DYNAMIC_TEMPORAL_CONTROLS = [ - ("NoAbort", "StartAnswered"), - ("StuckBackoff", "ClosedLeadsToDone"), + ("MCDynamicSupervisorCallers", "NoAbort", "StartAnswered"), + ("MCDynamicSupervisorCallers", "NoDeadline", "StartAnswered"), + ("MCDynamicSupervisorCallers", "TermLost", "TerminateAnswered"), + ("MCDynamicSupervisorCallers", "Sequential", "FanoutCompletes"), + ("MCDynamicSupervisorLive", "StuckBackoff", "ClosedLeadsToDone"), + ("MCDynamicSupervisorLive", "LostRestart", "RecoveryProgress"), ] DEFERRED_CONTROLS = [ @@ -313,8 +323,8 @@ def fail_instead_of_retry(events): named_temporal("MCLifecycleLive", fault, prop, args.tlc, LIFECYCLE_SMALL) for module, fault, invariant, *constants in DYNAMIC_CONTROLS: named_fault(module, fault, invariant, args.tlc, *constants) - for fault, prop in DYNAMIC_TEMPORAL_CONTROLS: - named_temporal("MCDynamicSupervisorLive", fault, prop, args.tlc) + for module, fault, prop in DYNAMIC_TEMPORAL_CONTROLS: + named_temporal(module, fault, prop, args.tlc) if __name__ == "__main__": From 007603a307dd7e39d6d94fa320f793dbf06bc774 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:39:42 -0300 Subject: [PATCH 23/52] verify: parenthesise badRestart --- verification/tla/DynamicSupervisor.tla | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index f0d58d80..bf6b8877 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -519,7 +519,7 @@ GhostUpdate == /\ lateGenClose' = [s \in Slots |-> lateGenClose[s] \/ (Grew(s) /\ ~open)] /\ lateGenTerm' = [s \in Slots |-> lateGenTerm[s] \/ (Grew(s) /\ sl[s].term)] /\ badRestart' = [s \in Slots |-> badRestart[s] \/ - ((Grew(s) /\ sl[s].gen > 0 \/ Decided(s)) /\ ~Allowed(s))] + (((Grew(s) /\ sl[s].gen > 0) \/ Decided(s)) /\ ~Allowed(s))] /\ badRetire' = [s \in Slots |-> badRetire[s] \/ (sl[s].cs = "decide" /\ sl'[s].cs = "removed" /\ open' /\ ~sl[s].term /\ Allowed(s))] From 396a422cf074879ae64dad44da322a26c2412edb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:42:21 -0300 Subject: [PATCH 24/52] docs: dynamic supervisor revision 3 (FIFO loop, lookup at handling, stuck classification, ignore, test plan) --- docs/design/dynamic-supervisor.md | 605 ++++++++++++++++++++---------- 1 file changed, 405 insertions(+), 200 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 70c18838..2203347a 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -1,9 +1,10 @@ # Dynamic supervisor: design contract -**Status: design in review (revision 2), not implemented.** ROADMAP rank 12a. +**Status: design in review (revision 3), not implemented.** ROADMAP rank 12a. Draft PR [#245](https://github.com/lambdaclass/spawned/pull/245). Revision 2 -applies the maintainer's decisions on the seven open questions of revision 1 -and resolves the first independent reviews (Codex and agy); see +applied the maintainer's decisions on revision 1's open questions and the +first reviews; revision 3 applies the decisions on round two's questions and +the second reviews (Codex, agy), plus the OTP/Elixir/Gleam comparison. See [Decisions](#decisions) and [Review resolutions](#review-resolutions). - Covers OTP's `simple_one_for_one` and Elixir's `DynamicSupervisor`: children @@ -40,6 +41,7 @@ and resolves the first independent reviews (Codex and agy); see - [Semantics contract: what this adds and touches](#semantics-contract-what-this-adds-and-touches) - [Interaction with rank 12 (scoped registries)](#interaction-with-rank-12-scoped-registries) - [What the model establishes](#what-the-model-establishes) +- [Test plan: relating the model to the code](#test-plan-relating-the-model-to-the-code) - [Alternatives considered](#alternatives-considered) - [Follow-ups](#follow-ups) - [Review resolutions](#review-resolutions) @@ -135,6 +137,21 @@ rule: | Q6 | `terminate_child` on static supervisors | Separate follow-up | | Q7 | Arguments on rejection | Returned on pre-admission rejection (`MaxChildren`, `Stopped`, placement); a distinct error for a start aborted after construction began | +Round two (revision 3): + +| # | Question | Decision | +|---|---|---| +| R1 | Loop blocked during `terminate_child` | **Kept, as OTP and Elixir:** the loop is held through the child's removal or its deadline | +| R2 | Stuck child that must restart | **Fails the supervisor**, after fixing the Transient classification (a stall is not a recorded cleanup failure). Escalation cannot make an unkillable computation disappear (see [Stuck children](#stuck-children)) | +| R3 | Atomic fan-out in the model | Kept as an ownership abstraction; a small interleaved refinement configuration checks that the fan-out completes even when siblings never finish | +| R4 | Two slots | Fast checks keep two slots; targeted three-slot instances check admission order, the cap and the shared budget. Bounded checks, no cutoff claimed | +| R5 | ethrex teardown | No message from `Drop`. Each resource's owner monitors the connection and releases on `Down`, as an Erlang owner would | +| G1 | Post-start work | Depends on the post-start continuation (a `handle_continue` equivalent) of the unified-start design (0.9), not on "the first message", which other messages could overtake | +| G2 | Restart type for connections | The ethrex mapping and the example use `temporary()` (a dropped peer must not be redialled by the supervisor); the default stays Permanent (OTP) | +| G3 | `ignore` | Accepted from the factory: nothing is stored, `start_child` returns `Ignored` (OTP `{ok, undefined}`) | +| G4 | Start data | `Started` carries the child's start data (unified start's `Init`), as Gleam's `Started(data)` | +| G5 | Introspection | `ChildInfo` and `ChildCounts` are the types introspection v1 exposes for static supervisors too; names agreed across both designs | + ## API ### Template @@ -208,25 +225,25 @@ let peers = slot.current()?; // re-resolve after the parent restarts it supervisor's `max_children <= 65536` ceiling does not apply. - `DynamicSupervisorSpec::new(name, config, spec)` takes the template; the adapter runs on the template's group. Its `shutdown_timeout` (how long the - parent waits for the whole dynamic supervisor) defaults to - **2 × the child `shutdown_timeout` + 1 s**, so the parent never gives up - before the inner kill stage ran (agy 6). OTP defaults supervisor children - to `infinity` for the same reason; Spawned keeps a finite default so a - stuck subtree is still reported (B). It can be raised with - `.shutdown_timeout()`. + parent waits for the whole dynamic supervisor) defaults to waiting for + the inner supervisor to finish, as OTP's `infinity` for supervisor + children; see [Nesting](#nesting-and-the-system). ### Operations ```rust impl DynamicSupervisor { /// OTP start_child(Sup, ExtraArgs). Queued on the control loop; resolves - /// when the first generation is ready (started() returned) or failed. - pub async fn start_child(&self, args: Args) -> Result, StartChildError>; - /// OTP terminate_child(Sup, Pid): by the child's last published ActorId. + /// when the first generation is ready, was ignored, or failed. + pub async fn start_child(&self, args: Args) + -> Result, StartChildError>; + /// OTP terminate_child(Sup, Pid): the actor id is looked up when the loop + /// handles the call; it names the slot only if it is the slot's last + /// published id at that moment. pub async fn terminate_child(&self, id: ActorId) -> Result<(), TerminateChildError>; - /// The same by slot, for a child seen in which_children (any state). + /// The same by slot (any generation). pub async fn terminate_slot(&self, id: DynamicChildId) -> Result<(), TerminateChildError>; - pub fn which_children(&self) -> Vec; + pub fn which_children(&self) -> Vec; pub fn count_children(&self) -> ChildCounts; pub fn stop(&self); pub async fn shutdown(&self, deadline: Duration) -> SupervisorReport; @@ -234,9 +251,17 @@ impl DynamicSupervisor { pub fn failure(&self) -> Option; } +pub enum StartOutcome { + Started(Started), + /// The factory returned ignore: nothing is supervised or counted + /// (OTP {ok, undefined}). + Ignored, +} + pub struct Started { pub actor: ActorRef, // the first generation, as OTP's Pid pub child: DynamicChild, // the slot, across restarts + pub data: A::StartData, // what the child's start returned (unified start, Init) } /// Like `SupervisedChild` for a dynamic slot. Holds no arguments and no @@ -249,6 +274,24 @@ impl DynamicChild { pub async fn terminate(&self) -> Result<(), TerminateChildError>; // = terminate_slot } +/// Nested form, as `SupervisedSupervisor`. +impl SupervisedDynamicSupervisor { + pub fn current(&self) -> Result, ChildUnavailable>; + pub async fn wait_ready(&self) -> Result, ChildUnavailable>; + pub async fn wait_for_generation(&self, after: u64) + -> Result, ChildUnavailable>; + pub fn snapshot(&self) -> ChildSnapshot; +} +impl DynamicSupervisorSpec { + pub fn new(name: impl Into, config: DynamicSupervisorConfig, + spec: DynamicChildSpec) -> Self; // Permanent + pub fn restart(self, restart: RestartMode) -> Self; + pub fn startup_timeout(self, limit: Duration) -> Self; + /// Default: wait for the inner supervisor to finish (OTP's `infinity` + /// for supervisor children); a finite value is a reporting bound only. + pub fn shutdown_timeout(self, limit: Duration) -> Self; +} + #[non_exhaustive] pub enum StartChildError { /// Not admitted: nothing was constructed and the arguments come back. @@ -263,22 +306,23 @@ pub enum StartChildError { #[non_exhaustive] pub enum RejectReason { - MaxChildren, // Elixir {:error, :max_children} - Stopped, // shutting down or stopped (OTP: the call exits) - Placement(SystemError),// the group is closed or out of native capacity + MaxChildren, // Elixir {:error, :max_children} + Stopped, // shutting down or stopped (OTP: the call exits) + Placement(SystemError), // the group is closed or out of native capacity } #[non_exhaustive] pub enum TerminateChildError { - NotFound, // not a current child (Elixir {:error, :not_found}) - Stopped, // called after shutdown began: the supervisor stops it + NotFound, // no current child with that id or slot (Elixir {:error, :not_found}) + Stopped, // shutdown began before the loop handled the call CleanupStuck, // the child missed its deadlines; see "Stuck children" } -pub struct DynamicChildInfo { - pub child: DynamicChildId, +/// Shared with introspection v1 (static and dynamic supervisors). +pub struct ChildInfo { + pub id: ChildKey, // DynamicChildId here; the name for static children pub actor: Option, // last published id; None before the first readiness - pub ready: bool, // false while restarting (OTP `restarting`) + pub ready: bool, // true iff state == Running pub state: ChildState, pub generation: u64, } @@ -291,25 +335,31 @@ pub struct ChildCounts { } ``` +- The factory returns `Result, E>` with + `ChildStart::Actor(a) | ChildStart::Ignore`, the unified-start design's + shape. An ignore on a first start returns `Ignored` and stores nothing; on + a restart it removes the child (OTP drops it), without charge beyond the + attempt already stamped. - `StartChildError` implements `Error` when `Args: Debug`; `into_args()` returns the arguments of a `Rejected`, and `without_args()` - maps to `StartChildError<()>` for `?` in code that does not need them. + maps to `StartChildError<()>` for `?`. - `FailedToStart` carries the generation's `ExitReason` when an actor - existed (`Panic` from `started()`, `Normal` from `ctx.stop()` in - `started()`), so callers need not match strings (agy 9). + existed, so callers need not match strings. - `ChildState` gains no variant: a dynamic child is `Starting`, `Running`, `Backoff` (restart pending), `Stopping` (including stuck), then leaves - `which_children`. `which_children` is a snapshot in no particular order. -- Removed slots are evicted from the supervisor's table at removal, after - their retained arguments are dropped: memory is O(children held), not - O(starts). + `which_children`, which is a snapshot in no particular order. +- Removed slots are evicted from the supervisor's table after slot + finalization ([Retained arguments](#retained-arguments)): memory is + O(children held). +- `ChildInfo`, `ChildCounts` and `ChildKey` are introspection v1's types; + this design only fixes their dynamic-supervisor contents. ### Restart types | | OTP | This design | |---|---|---| | Default | `permanent` | `Permanent` (`DynamicChildSpec::new`); `temporary()` for one-shot children | -| Table | `do_restart/3` | V1 with the v0.8 rule: Permanent always; Transient on an abnormal reason (`Panic`, `Kill`, `Killed`, `NoProc`) or a recorded cleanup failure; Temporary never | +| Table | `do_restart/3` | V1 with the v0.8 rule: Permanent always; Transient on an abnormal reason (`Panic`, `Kill`, `Killed`, `NoProc`) or a recorded cleanup failure (cleanup that *finished* and reported a destructor or helper failure; a cleanup that is still running past its deadline is not one); Temporary never | | A replacement failing before readiness | `try_again_restart`, retried | retried unless Temporary, each attempt charged | | `terminate_child` | never restarted | never restarted, any type | @@ -330,8 +380,13 @@ pub struct ChildCounts { when the restart attempt is admitted, and refunded if the pending restart is cancelled by `terminate_child` or shutdown. Window expiry releases only stamped grants, never reservations. -- **Backoff** is opt-in (`backoff: Some(..)`): without it the restart is - attempted as soon as the loop handles the exit, as OTP. With it, the +- **Backoff** is opt-in (`backoff: Some(..)`). Without it (the default) the + loop reserves, stamps and runs the restart attempt in the same turn in + which it handles the exit, as OTP's `restart/2` does; a failed attempt + re-queues the exit at the tail of the loop's queue (OTP's + `try_again_restart` cast) and is decided again there. With backoff the + loop reserves, releases the loop, and the timer queues a retry that the + loop takes in order. With it, the per-child jitter stream is seeded from the supervisor seed mixed with the slot's `DynamicChildId`, not only the shared template name, so siblings do not draw identical delays (agy 9). @@ -387,42 +442,77 @@ pub struct ChildCounts { the parent creates a new, empty dynamic supervisor (OTP). Old handles return `Stopped`; `SupervisedDynamicSupervisor::current()` resolves the new one. +- The adapter's own `shutdown_timeout` defaults to waiting for the inner + supervisor to finish, which its own deadlines already bound (OTP's + `infinity` for supervisor children). A finite value is a reporting bound: + when it expires the parent records a cleanup failure but keeps the + descendant join barrier. A fixed margin such as 2 × child timeout + 1 s + (revision 2) cannot cover an unbounded fan-out and the cleanup this + contract allows (Codex F7). ## Semantics ### The control loop -One logical control loop per supervisor handles, in arrival order and one at -a time, as OTP's supervisor process does: - -- a `start_child`: admission (gate, `max_children`, capacity), then the - first generation's factory and `started()`, until readiness commits or the - attempt failed and was cleaned up (or reached its cleanup deadline); -- a child's exit: the restart decision, the reservation, and (after the - optional backoff, which does not hold the loop) the restart attempt until - its readiness or failure; -- a `terminate_child`: signal, kill escalation and removal of that child. - -Not serialised: children run, exit and clean up concurrently; the backoff -timer runs off the loop; `which_children`, `count_children` and `current()` -read a snapshot without waiting for the loop; shutdown closes admission at -once and cancels the operation in progress (row 21 of the alignment table, +One logical control loop per supervisor, as OTP's supervisor process. It +owns one FIFO queue holding, in arrival order: + +- `start_child` requests; +- `terminate_child` / `terminate_slot` calls; +- exit notices of children whose owned cleanup finished (OTP's `'EXIT'`); +- backoff retries, when backoff is configured (OTP's `try_again_restart` + cast is the no-backoff counterpart: a failed attempt's exit is queued at + the tail). + +The loop takes the head and runs it to completion before the next: + +- a start: admission (gate, `max_children`, capacity), then the first + generation's factory and `started()`, until readiness commits, the factory + ignores, or the attempt failed and was cleaned up (or reached its + deadline); +- an exit notice: the restart decision, the reservation and, without + backoff, the restart attempt until readiness or failure; +- a retry: stamping and the restart attempt; +- a terminate call: lookup, signal, kill escalation and removal, or the + deadline. + +The exit of the child an operation is waiting for (the first generation of +a failing start, the child being terminated) is consumed by that operation +directly, as OTP's `start_child` returns the start function's error and +`shutdown/2` consumes the `'EXIT'` with `unlink_flush`. Every other exit +waits its turn in the queue. + +Not serialised: children run, exit and clean up concurrently; deadlines and +the backoff timer run off the loop; `which_children`, `count_children` and +`current()` read a snapshot; shutdown closes admission at once, answers the +queued requests and cancels the operation in progress (alignment row 21, B). Factories run on the target group even though the loop awaits them. -Consequences, all as in OTP: - -- A slow `started()` delays every later start, restart and termination of - that supervisor. Keep `started()` short and do slow initialisation in the - first message (OTP's `handle_continue`). `startup_timeout` bounds the - delay per start. -- With `max_children = 1`, a start that arrives while another is - constructing waits; if the first fails, the second is admitted (Codex 1). -- A `terminate_child` queued behind a start of the same child finds it. +Consequences, as in OTP: + +- Requests are served in arrival order: with `max_children = 1`, of two + queued starts the first is admitted and the second gets `MaxChildren`, + never the reverse (model: `FifoOrder`, control `LaterOp`). +- A start arriving while another constructs waits; if the first fails, the + second is admitted. +- A slow `started()` delays every later operation. Slow initialisation goes + in the post-start continuation (below). +- A slow-to-stop child holds the loop during its `terminate_child`, at most + until its deadline; the stall watchdog and introspection report "blocked + in terminate_child(id) for T" (decision R1). + +**Post-start continuation.** Slow initialisation (ethrex's RLPx handshake) +belongs in the unified-start design's continuation, OTP's +`handle_continue`: it runs right after `started()` and before any message, +including messages that other actors enqueue once they learn the +reference. That design targets 0.9 and this one depends on it; "send +yourself a message in `started()`" is not equivalent, because a message +enqueued by someone else before the self-send can run first. ### `start_child` -1. **Queued** on the loop. If the supervisor is already closed: - `Rejected { Stopped, args }` at once. +1. **Queued** on the loop, in arrival order. If the supervisor is already + closed: `Rejected { Stopped, args }` at once. 2. **Admission** when the loop reaches it: closed → `Rejected { Stopped, args }` (also for a request still queued when shutdown begins); cap reached → `Rejected { MaxChildren, args }`; native capacity → `Rejected { Placement, args }`. @@ -436,7 +526,8 @@ Consequences, all as in OTP: commit is the success: from then on `start_child` returns `Ok(Started)`, even if shutdown begins before the caller observes it (success is historical, like `ActorRef::ready()` and `start()`'s - readiness, `supervision.rs` `start_owned`). The returned `actor` may + readiness, `supervision.rs` `start_owned`). A factory that returns + ignore answers `Ok(Ignored)` and leaves nothing behind. The returned `actor` may already be dead: if the child crashed between the commit and the caller's wake-up, it is handled by the restart table and `child.current()` resolves the replacement (Codex 2). @@ -462,22 +553,25 @@ Consequences, all as in OTP: | Race | Outcome | |---|---| +| two `start_child` calls, cap reached by the first | served in arrival order: the first is admitted, the second gets `Rejected { MaxChildren, args }` | | `start_child` vs shutdown, while queued | `Rejected { Stopped, args }` | -| `start_child` vs shutdown, during construction | `Aborted`; never `Ok` unless readiness committed first (model: `NoStartAfterShutdown`) | +| `start_child` vs shutdown, during construction | `Aborted`; never `Ok` unless readiness committed first (`NoStartAfterShutdown`) | | `start_child` vs shutdown, after the commit | `Ok`; the child is then stopped with the others | -| `start_child` vs a sibling's crash | the crashed child keeps its slot through cleanup, backoff and the replacement, so at the cap the new start gets `MaxChildren` (`MaxChildrenBound`, control `UncountedRestart`) | -| `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup and argument drop | -| `start_child` vs budget exhaustion | exhaustion closes the gate; as shutdown | -| `terminate_child` vs a crash of that child | serialised by the loop. Exit handled first: if a restart is pending, `terminate_child` cancels it (refunding the reservation) and removes the child; if the restart attempt is running, `terminate_child` waits behind it and then terminates the new generation. Terminate first: the crash is recorded, no restart. Either way the child is removed and never restarted (`TerminatedNeverRestarted`, `NoPublishAfterStop`) | -| `terminate_child` vs shutdown | a call queued or in progress when shutdown begins resolves when the child is removed (`Ok`, or `CleanupStuck`); a call made after shutdown began returns `Stopped` | -| two `terminate_child` calls for one child | the second waits for the same removal and gets the same result; a call after removal gets `NotFound` | +| `start_child` vs a sibling's crash | the crashed child keeps its slot through cleanup, backoff and the replacement (`MaxChildrenBound`, control `UncountedRestart`) | +| `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup and slot finalization | +| `terminate_slot` vs a crash of that child | the loop takes them in order. Exit first: a pending restart is cancelled and refunded, or the restart attempt (which holds the loop) finishes and the call then terminates the new generation. Terminate first: the crash is consumed by the termination, no restart. Either way the child is removed and never restarted (`TerminatedNeverRestarted`, `NoPublishAfterStop`) | +| `terminate_child(id)` vs a crash of that child | the id is looked up when the loop handles the call. If the slot's last published id is still `id` (cleanup, backoff, a failed replacement), the child is terminated as above. If a replacement committed readiness before the call reached the head, `id` is stale: `NotFound`, and the replacement keeps running (Elixir; `TermResultsSound`, control `SlotLookup`) | +| `terminate_child` vs shutdown | a call still queued when shutdown begins gets `Stopped`; a call in progress resolves when the child is removed (`Ok`) or at its deadline (`CleanupStuck`) (control `QueuedTermOk`) | +| two calls for one child | the second is handled after the first completed: `NotFound` if the child was removed, `CleanupStuck` if it is stuck (Elixir; no shared waiters) | +| a terminate whose child gets stuck | `CleanupStuck` is committed at the deadline and never changes, even if the work ends later (`TermResultsSound`, `TermResultStable`, control `StuckResultLost`) | ### Restart identity and termination keys - Every generation has a new `ActorId`, mailbox and state (M4). The slot's `DynamicChildId` never changes. `current()` returns the latest ready generation; `wait_for_generation(n)` waits for a newer one (V4). -- The **termination key** of a slot is its **last published** `ActorId`. +- The **termination key** of a slot is its **last published** `ActorId`, + compared when the loop handles the call (not when it was made). It is kept through cleanup, backoff and failed replacement attempts, and switches only when a replacement's readiness commits, as OTP keeps the original pid as the `restarting` entry's key and Elixir keeps @@ -494,41 +588,54 @@ Consequences, all as in OTP: `terminate_child` is OTP's `shutdown/2` for one child, run by the loop: -1. Mark the slot terminated (no restart from now on); cancel a pending +1. Look the target up (id or slot, above). Not found: `NotFound`. Already + stuck: `CleanupStuck`. +2. Mark the slot terminated (no restart from now on); cancel a pending restart and refund its reservation. -2. Send the parent exit signal `Shutdown`: a non-trapping child begins exit +3. Send the parent exit signal `Shutdown`: a non-trapping child begins exit after its current handler and skips `stopped()` (E2, decision 2); a trapping child handles the messages queued before the signal, then runs `stopped()` (E3). -3. If the child has not reached its death point `shutdown_timeout` after the - signal, kill it (`Killed`, E1). If it misses a second `shutdown_timeout`, - or its owned cleanup misses its deadline, the child is stuck: - `terminate_child` returns `CleanupStuck`. -4. Otherwise resolve `Ok` once the child is removed (`TermOkMeansRemoved`). - -The loop is held for the whole call, as OTP's supervisor is blocked in -`terminate_child`; it is bounded by 2 × `shutdown_timeout` plus the cleanup -deadline. +4. After `shutdown_timeout` without reaching its death point, kill it + (`Killed`, E1). OTP's kill ends the process at that point. Spawned's kill + is cooperative (R), so Spawned adds a second `shutdown_timeout` after the + kill; a child that misses it, or whose owned cleanup misses its deadline, + is stuck and the call returns `CleanupStuck` at that moment. +5. Otherwise `Ok` once the child is removed. + +The loop is held for the whole call, through removal or the deadline (as +OTP's supervisor is blocked in `terminate_child` and Elixir's handles it in +its call handler); deadline monitoring and the child's cleanup run off the +loop. ### Stuck children -OTP's `kill` always ends a process, so an OTP supervisor never waits on a -child that ignores `shutdown` longer than the shutdown timeout (except with -`infinity`). Spawned's kill is cooperative (R): a child can stay in a -handler, in `stopped()` or in owned cleanup past every deadline. A child is -**stuck** when, after its stop signal, it misses the kill and the second -interval, or when its owned cleanup misses its deadline (`shutdown_timeout` -from exit begin for a child that ended on its own, E3 row 4: one interval, -no kill). Policy (agy P0): +OTP's `kill` always ends a process, so an OTP supervisor never waits past +the shutdown timeout (except with `infinity`). Spawned's kill is +cooperative (R): a child can stay in a handler, in `stopped()` or in owned +cleanup past every deadline. A child is **stuck** when, after its stop +signal, it misses the kill and the second interval, or when its owned +cleanup misses its deadline (`shutdown_timeout` from exit begin for a child +that ended on its own, E3 row 4: one interval, no kill). + +The classification uses the exit reason only: a cleanup that is still +running is not a recorded cleanup failure, so it does not make a Transient +child restartable (revision 2 contradicted itself here; Codex F3, agy 1). | The stuck child | Effect | |---|---| -| was retiring anyway (Temporary, Transient after a normal exit, removed by `terminate_child`, or a failed first start) while the supervisor is open | **only that child is affected.** Its slot stays counted (it still holds memory, sockets, capacity) and its work stays owned until it really ends, then the slot is removed. The failure is recorded in its `ChildSnapshot::last_error` and the supervisor's report; a waiting `terminate_child` gets `CleanupStuck`, a waiting `start_child` gets `FailedToStart { cleanup_stuck: true }`. The supervisor and its siblings are unaffected, as OTP's supervisor carries on after killing a child | -| would be restarted (Permanent, Transient after an abnormal exit or with the cleanup failure the stall implies) | the supervisor **fails** and exits abnormally: it cannot start the replacement while the old generation lives (V3), and OTP would have restarted it; escalation lets the parent decide (V8) | -| is being stopped by shutdown | the shutdown report is incomplete and records the failure (V9, S3); "done" waits for it | - -Model: `StuckIsolated`, control `StuckFailsAll`. A retiring stuck child is -also never restarted later; its slot simply ends when the work ends. +| would not be restarted: Temporary; Transient after a normal exit (`Normal`, `Shutdown`); removed by `terminate_child`; a failed first start | **only that child is affected.** Its slot stays counted and its work owned until it really ends, then the slot is removed. The failure is in its `ChildSnapshot::last_error` and the report; a waiting `terminate_child` gets `CleanupStuck`, a waiting `start_child` gets `FailedToStart { cleanup_stuck: true }`. The supervisor and its siblings are unaffected | +| would be restarted: Permanent; Transient after an abnormal exit; a failed replacement of a restartable child | the supervisor **fails** and exits abnormally: it cannot start the replacement while the old generation lives (V3), and OTP would have restarted it (decision R2) | +| being stopped by shutdown | the shutdown report is incomplete and records the failure (V9, S3); "done" waits for it | + +Escalation cannot make an unkillable computation disappear. The failed +dynamic supervisor's parent restarts the subtree only after the descendant +barrier (V8): the stuck generation still holds its slot, its group capacity +and whatever it touches until its code returns. What escalation gives is +visibility (the parent and the report know) and the OTP restart policy once +the work ends. Model: `StuckIsolated` checks the classification against the +table, not against the label the action chose (controls `StuckFailsAll`, +`MisclassifyStuck`). ### Shutdown of all children @@ -544,34 +651,47 @@ its parent, or a failure): `ConcurrentShutdown`, control `Sequential`. 3. **Deadlines:** each child's intervals are measured from its own signal (E3): kill after `shutdown_timeout`, stuck after a second one. OTP arms - one timer after the fan-out; measuring from each child's own signal - differs only by the fan-out time and never shortens a child's interval - (Codex 9). Shutdown therefore takes at most 2 × `shutdown_timeout` plus - the fan-out time plus owned cleanup up to its deadline, whatever N is. + one timer after the whole fan-out, so a child signalled early gets the + fan-out time on top of its interval there; here it does not. This is an + inherited E3 deviation (B: per-child bounds independent of N), listed in + the alignment table. Shutdown takes at most the fan-out time plus + 2 × `shutdown_timeout` plus owned cleanup up to its deadline. 4. **Done** means every slot was removed: every generation reached publication and its owned cleanup finished, its arguments dropped (V9; `NoChildOutlivesSupervisor`, `ClosedLeadsToDone`). 5. `shutdown(deadline)` keeps the static meaning: `deadline` is the report deadline; children still running then stay owned by the system (S3). -The model takes the fan-out as one step: the implementation sends the -signals in a loop with no await between them, and a child acting before its -own signal only ends earlier. Trace validation of the implementation will -check this abstraction (signal events precede any wait event). +The model's main configurations take the fan-out as one step, an ownership +abstraction: it establishes that every admitted child is signalled before +the supervisor waits, not timer epochs or scheduling bounds (time is +abstract). The refinement configuration `MCDynamicSupervisorCallers` runs +the fan-out one signal per step, interleaved with the children, with no +cleanup ever assumed to finish, and checks `FanoutCompletes`: every +admitted child is signalled even while signalled siblings never end +(control `Sequential` waits for each sibling and fails it). Timing claims +are for the implementation's tests. ### Retained arguments - Restartable slots keep the original `Args`; each attempt receives a clone made inside the attempt (on the target group, outside framework locks, within `startup_timeout`). A panicking or hanging clone is a failed - attempt (Codex 10). -- At removal the slot drops its retained arguments as part of its cleanup, - before its native-capacity reservation and its `max_children` slot are - released. A panicking destructor is recorded as a cleanup failure (after - the child's death it changes no exit reason); a hanging one makes the slot - stuck, under the policy above. -- `DynamicChild` handles hold no arguments and no scopes; after removal they - report terminal metadata only. + attempt. +- **Generation cleanup and slot finalization are distinct.** Generation + cleanup (the actor's death, its state and queued messages, its helpers) + ends every generation; a failure recorded there feeds the Transient rule. + Slot finalization runs once, only after the loop decided to remove the + slot (no restart can follow): it drops the retained arguments and then + releases the native-capacity reservation and the `max_children` slot. +- Finalization runs as a tracked task on the template's group, never on the + control loop and never under a supervisor lock, bounded by + `shutdown_timeout`. A panicking destructor is recorded as a finalization + failure in the report (no restart: the decision was already made). A + destructor that hangs past the deadline makes the slot stuck as a + retiring child: it stays counted, it is reported, nothing else is + affected (Codex F8). +- `DynamicChild` handles hold no arguments and no scopes. ### In-flight requests and deferred replies of a dying child @@ -608,13 +728,13 @@ As for any actor (exit-signals design, phases of an exit, with rank 11): | # | Rule | OTP / Elixir | This design | Reason for any deviation | |---|---|---|---|---| | 1 | `start_child` is synchronous and returns the child | `{ok, Pid}` after the start function returns | `Ok(Started { actor, child })` after `started()` | aligned; `child` is an addition (row 9) | -| 2 | Operations serialised by the supervisor | one process: starts, `'EXIT'` handling, restarts and `terminate_child` run one at a time | one control loop, same order and blocking | aligned (decision Q1) | +| 2 | Operations serialised by the supervisor | one process: calls and `'EXIT'` messages handled in mailbox order, one at a time | one control loop over one FIFO queue of starts, terminate calls, exits and retries | aligned (decisions Q1, R1) | | 3 | Template plus per-call args | `ExtraArgs` appended (Elixir: fixed `extra_arguments` prepended) | typed `Args` passed to the factory | T | -| 4 | `ignore` | `{ok, undefined}`, nothing stored | not supported | as the static supervisor (S1) | +| 4 | `ignore` | `{ok, undefined}`, nothing stored; on restart the child is dropped | `Ok(Ignored)`, nothing stored; on restart the child is removed | aligned (decision G3) | | 5 | Start failure | `{error, Reason}`, not counted, not retried | `FailedToStart`, not counted, not retried, after cleanup or at its deadline | aligned; the deadline release is R (S2) | | 6 | Default restart type | `permanent` | `Permanent` | aligned | | 7 | Temporary children keep no args | `mapsets` | `temporary()` moves args once, no `Clone` | aligned; R for the separate constructor | -| 8 | `terminate_child` key | pid, kept through failed restarts | last published `ActorId`, kept through failed restarts; `NotFound` (Elixir) for anything else | aligned with Elixir's `not_found`; OTP's `ok` for a dead unknown pid is a `find_child` artefact | +| 8 | `terminate_child` key | pid, looked up when the call is handled, kept through failed restarts | last published `ActorId`, looked up when the loop handles the call; `NotFound` (Elixir) otherwise, including for a second call | aligned with Elixir; OTP's `ok` for a dead unknown pid is a `find_child` artefact | | 9 | Restart identity | new pid, found through names | new `ActorId`; stable `DynamicChildId`, `current()`, `terminate_slot` | addition, consistent with `SupervisedChild` (instance-bound references) | | 10 | `restart_child`, `delete_child` | `{error, simple_one_for_one}` | not provided | aligned (absent at compile time: T) | | 11 | `which_children` | pid or `restarting`, unordered | last published id plus `ready`, unordered | aligned plus fields | @@ -622,17 +742,17 @@ As for any actor (exit-signals design, phases of an exit, with rank 11): | 13 | `max_children` | Elixir: default infinity, `0` allowed, start only, counts restarting | same; frees after cleanup | aligned; B for freeing after cleanup | | 14 | Restart budget | every restart attempt counts; Elixir default 3 in 5 s | same rule and defaults | aligned (decision Q3) | | 15 | Exhaustion | terminate children, exit `shutdown` | terminate children concurrently, exit `Shutdown`, diagnostic in `failure()` | aligned (decision Q5); static nested supervisors follow in a separate PR | -| 16 | Shutdown order | parallel, undefined, one timer after the fan-out, then `kill` | parallel, per-child intervals from each child's signal, `kill`, then stuck | aligned up to the fan-out time; the stuck stage is R | +| 16 | Shutdown order and timer | parallel, undefined order, one timer armed after the fan-out, then `kill` | parallel, each child's interval from its own signal, `kill`, then a second interval and stuck | timer epoch: inherited E3 deviation (B), early children get no fan-out time on top; the second interval is R | | 17 | Shutdown signal | `exit(Child, shutdown)`; a trapping child runs `terminate` | parent exit signal; trapping child runs `stopped()` after its backlog; non-trapping skips it | aligned (E2, E3) | | 18 | `start_child` during shutdown | the queued call exits with the supervisor's reason or `noproc` | `Rejected { Stopped, args }` or `Aborted` | R: no caller exit in Rust; T: the arguments come back | | 19 | Restarted dynamic supervisor | empty | empty | aligned | | 20 | Backoff | none | none by default; opt-in per supervisor | aligned by default (decision Q3) | | 21 | Shutdown admission | the supervisor processes its queue in order; a start queued behind the shutdown is never handled | admission closes at once, the construction in progress is cancelled | B (exit-signals design) | | 22 | When a slot frees | at `'EXIT'` | after owned cleanup and the argument drop (V3) | R and B: destructors and owned helpers outlive the last hook | -| 23 | A child that ignores `shutdown` | killed; the kill always works | killed cooperatively; if it still does not end it is stuck, isolated unless it would restart | R: Rust cannot kill a thread; policy in [Stuck children](#stuck-children) | +| 23 | A child that ignores `shutdown` | killed after the timeout; the kill always ends it | killed cooperatively after the timeout; if it misses a second interval it is stuck, isolated unless it would restart | R: Rust cannot kill a thread; policy in [Stuck children](#stuck-children) | | 24 | Calls in flight to a dying child | `'EXIT'`/monitor tells callers | `ActorStopped(reason)` at D, `NoProc` after | aligned (rank 11, v0.8) | | 25 | Heterogeneous specs | Elixir: any spec per call | one typed template | T; decision Q2 | -| 26 | Nested supervisor's shutdown wait | `infinity` by default for supervisor children | 2 × child `shutdown_timeout` + 1 s by default | B: a stuck subtree is still reported | +| 26 | Nested supervisor's shutdown wait | `infinity` by default for supervisor children | waits for the inner supervisor by default; a finite value only reports | aligned | ## Non-guarantees @@ -658,17 +778,18 @@ New guarantees (section 6 of [semantics.md](semantics.md), ids proposed): | Id | Guarantee | Checked by (planned) | |---|---|---| -| V10 | **Dynamic admission.** One control loop serialises starts, exit handling and terminations. A start is admitted atomically with the gate and the count; success commits only while admission is open and is then historical; a rejected start returns its arguments; a first-start failure is returned after cleanup or at its deadline, uncharged and unretried. | `DynamicSupervisor!SerialConstruction`, `!NoStartAfterShutdown`, `!OkMeansReady`, `!FirstStartNeverRetried`, `!FailedAfterCleanup`, `!FailureCauseKept`, `!StartAnswered`; tests | +| V10 | **Dynamic admission.** One control loop takes starts, terminate calls, exits and retries from one FIFO queue, one at a time. A start is admitted atomically with the gate and the count; success commits only while admission is open and is then historical; a rejected start returns its arguments; an ignored start stores nothing; a first-start failure is returned after cleanup or at its deadline, uncharged and unretried; every call is answered even if cleanup never ends. | `DynamicSupervisor!FifoOrder`, `!SerialConstruction`, `!NoStartAfterShutdown`, `!OkMeansReady`, `!FirstStartNeverRetried`, `!FailedAfterCleanup`, `!FailureCauseKept`, `!StartAnswered` (CallerSpec); trace validation | | V11 | **Cap counts restarting children.** A child holds its slot from admission until removal, through cleanup, backoff, replacement and a stuck cleanup. | `!MaxChildrenBound` | -| V12 | **Terminated means removed.** After `terminate_child` a child is never restarted or republished; a pending restart is cancelled and refunded; `Ok` means removed. | `!TerminatedNeverRestarted`, `!NoPublishAfterStop`, `!ReservationsMatch`, `!TermOkMeansRemoved`, `!TerminateAnswered` | -| V13 | **Concurrent shutdown.** Every live child is signalled without awaiting another; the supervisor is done only after every child finished. | `!ConcurrentShutdown`, `!NoChildOutlivesSupervisor`, `!ClosedLeadsToDone` | -| V14 | **Stuck isolation.** A child stuck while retiring fails only itself; the supervisor fails only for a stuck child it would restart, budget exhaustion, or during shutdown. | `!StuckIsolated` | +| V12 | **Terminated means removed.** The target is looked up when the loop handles the call; after a found `terminate_child` the child is never restarted or republished, a pending restart is cancelled and refunded, `Ok` means removed, `CleanupStuck` is committed at the deadline and never changes; a call still queued at shutdown gets `Stopped`. | `!TerminatedNeverRestarted`, `!NoPublishAfterStop`, `!ReservationsMatch`, `!TermResultsSound`, `TermResultStable`, `!TerminateAnswered` (CallerSpec) | +| V13 | **Concurrent shutdown.** Every admitted child is signalled without awaiting another; the supervisor is done only after every child finished. | `!ConcurrentShutdown`, `!FanoutCompletes` (refinement, CallerSpec), `!NoChildOutlivesSupervisor`, `!ClosedLeadsToDone` | +| V14 | **Stuck isolation.** A child stuck while retiring (by the exit reason; a stall is not a recorded cleanup failure) fails only itself; the supervisor fails only for a stuck child the table would restart, budget exhaustion, or during shutdown. | `!StuckIsolated` | Touched: - **V1** (restart table) per child, including the cleanup-failure rule: - `!RestartFollowsPolicy` (both directions: no forbidden restart, no - retirement of a restartable child while open), `!TemporaryNeverRestarted`, + `!RestartFollowsPolicy` (both directions, checked at the decision: + reservation, attempt or exhaustion; no retirement of a restartable child + while open), `!TemporaryNeverRestarted`, `!RecoveryProgress`. - **V3, V4** per slot: `!AtMostOneGeneration`, `!PublishedIsLive` (also requiring readiness). @@ -740,111 +861,192 @@ that generation died, not that the slot is gone. ## What the model establishes [`DynamicSupervisor.tla`](../../verification/tla/DynamicSupervisor.tla) -(revision 2) models one dynamic supervisor, N `start_child` calls made at any -moment, and one `terminate_child` per started child. A control loop (`op`) -serialises admission and first construction, exit handling and restart -attempts, and terminations. Each slot's restart type is chosen in `Init`, -so one run covers every combination. The instances use two slots: every -property relates at most a pair of children (one holds the cap or restarts -while the other races it), and with revision 2's detail three slots passed -6.7 GB of TLC state in 38 minutes without finishing. Budget reservations, stamping, refunds and window expiry -are explicit. Exits come from children (`Normal`, `Panic`), stop signals -(`Shutdown`, or `Killed` after the kill), start failures (factory error, -`Normal` before readiness), and cleanup may record a failure (the Transient -rule). A child signalled but still running, or with stuck cleanup, can -become stuck, with the policy above. - -Invariants that name an event (a commit, a publication or a generation -after close or terminate; a forbidden restart or retirement; restart -attempts) read history variables that `GhostUpdate` computes from every -step's state change, separately from the actions, so a mutated action cannot -hide its own violation (agy 3). `alive` accumulates generations and only the -one whose cleanup ended is removed (Codex 7). +(revision 3) models one dynamic supervisor, `start_child` calls made at any +moment, and `terminate_child` calls, each aimed at a slot either by the id +the caller saw or by slot. The control loop takes starts, terminate calls, +exit notices and backoff retries from one FIFO queue; the exit of the child +an operation waits for is consumed by that operation. Each slot's restart +type is chosen in `Init`. Budget reservations, stamping, refunds and window +expiry are explicit; backoff is a constant (off: the attempt runs in the +exit's turn and a failed one re-queues; on: reserve, timer, retry through +the queue). Children can ignore, fail, time out, exit, be stopped (Shutdown, +or Killed after the kill), record a cleanup failure, or get stuck before or +after their death point. + +Event invariants (late commits, publications and generations; forbidden +restart decisions or retirements; restart attempts; queue order) read +history variables that `GhostUpdate` computes from each step's state change, +separately from the actions. `alive` keeps every live generation. + +Two fairness sets. `CallerSpec` assumes only the loop's work and deadlines +(startup timeout, cleanup deadline) progress; no user code is assumed to +finish. `StartAnswered`, `TerminateAnswered` and `FanoutCompletes` hold +under it (Codex F4). `Spec` adds progress of readiness, stops, cleanup and +stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | Invariant | Meaning | |---|---| | `AtMostOneGeneration` | a slot never has two live generations | -| `PublishedIsLive` | `current()` returns only the live, latest, ready generation | +| `PublishedIsLive` | `current()` returns only the running, live, latest, ready generation | | `NoChildOutlivesSupervisor` | done means every child finished and was removed | | `MaxChildrenBound` | children held never exceed `max_children` | | `SerialConstruction` | at most one generation is being constructed | +| `FifoOrder` | the loop takes queued operations in arrival order | | `TemporaryNeverRestarted` | a Temporary child has one generation | -| `TerminatedNeverRestarted` | no generation begins after `terminate_child` | +| `TerminatedNeverRestarted` | no generation begins after a found `terminate_child` | | `NoGenerationAfterClose` | no generation begins after admission closed | -| `NoPublishAfterStop` | no generation is published after close or `terminate_child` | +| `NoPublishAfterStop` | no generation is published after close or termination | | `NoStartAfterShutdown` | no `start_child` success commits after admission closed | -| `RestartFollowsPolicy` | every restart is allowed by the table, and no restartable exit is retired while open | +| `RestartFollowsPolicy` | every restart decision (reservation, attempt, exhaustion) is allowed by the table; no restartable exit is retired while open | | `FirstStartNeverRetried` | a failed first start is never retried | -| `FailedAfterCleanup` | `FailedToStart`/`Aborted` only after cleanup or at the stuck deadline | +| `FailedAfterCleanup` | `FailedToStart`/`Aborted` only once cleanup really finished (removed, no live generation) or with the deadline failure recorded | | `FailureCauseKept` | the start error reflects what ended the attempt | | `ChargedRestarts` | every restart attempt is charged exactly once; first starts never | | `ReservationsMatch` | reservations are exactly the pending restarts | | `BudgetBounded` | stamped grants plus reservations stay within `max_restarts` | | `ExhaustionCloses` | exhaustion fails and closes the supervisor | -| `StuckIsolated` | the supervisor fails only by exhaustion, a stuck child it would restart, or while closing | -| `ConcurrentShutdown` | once stopping, every running child has its stop signal | +| `StuckIsolated` | the supervisor fails only by exhaustion, a stuck child the table would restart (checked against the table), or while closing | +| `ConcurrentShutdown` | once waiting, every running child has its stop signal | | `OkMeansReady` | `Ok` only for a child that became ready | -| `TermOkMeansRemoved` | `terminate_child` returns `Ok` only after removal | +| `TermResultsSound` | `Ok` only after removal and never for a stuck child; a pending call holds the loop; an id call reaches only the generation it named | -Action property `NeverReopens`. Liveness (fair loop, construction, stops, -cleanup and stuck children ending; crashes, start failures and getting stuck -not fair): `ClosedLeadsToDone`, `StartAnswered`, `TerminateAnswered`, -`RecoveryProgress` (a restartable exit is restarted or superseded by -terminate, shutdown, exhaustion or a stuck cleanup). +Action properties: `NeverReopens`, `TermResultStable` (a committed terminate +result never changes). Liveness: `StartAnswered`, `TerminateAnswered`, +`FanoutCompletes` (CallerSpec); `ClosedLeadsToDone`, `RecoveryProgress` +(Spec). -TLC results (TLC 2.19, fede, 4 workers, `nice -n 19`): +Configurations (TLC 2.19, fede, 4 workers, `nice -n 19`). These are +bounded checks: two slots for the broad instances, three for the targeted +one; no cutoff argument is claimed (Codex F10). | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (per PR; safety, `NeverReopens`) | 2 slots, every pair of restart types, `max_children` 1, budget 1, one window release, exits `Normal`/`Panic`, start failures factory/`Normal`, cleanup failures | 3,459,556 | 33 | 39 s | -| `MCDynamicSupervisorLive` (per PR; liveness) | as above without cleanup failures, factory start failures only | 1,628,738 | 33 | 5m21s | -| `MCDynamicSupervisorZero` (per PR; safety, liveness) | as `MCDynamicSupervisor` with budget 0 | 589,159 | 22 | 1m23s | -| `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two window releases, `max_children` 2 | 31,620,430 | 42 | 5m06s | +RESULT_ROWS Negative controls (`negative_checks.py`, run one at a time; each checks -`TypeOK` and only the named property, safety controls on -`MCDynamicSupervisor`, liveness controls on `MCDynamicSupervisorLive`): +`TypeOK` and only the named property). Every invariant and liveness +property above has at least one; the two action properties do not. -| `Fault` | Mutation | Fails | Origin | +| `Fault` | Mutation | Fails | Config | |---|---|---|---| -| `IgnoreMax` | admission ignores the cap | `MaxChildrenBound` | rev. 1 | -| `UncountedRestart` | a crashed child frees its slot until it restarts | `MaxChildrenBound` | rev. 1 | -| `RestartTemporary` | Temporary children follow the Permanent rule | `TemporaryNeverRestarted` | rev. 1 | -| `RestartTerminated` | `terminate_child` does not cancel a restart | `TerminatedNeverRestarted` | rev. 1 | -| `AcceptDuringShutdown` | a start admitted and committed after close | `NoStartAfterShutdown` | rev. 1 | -| `RestartAfterClose` | a pending restart proceeds after close | `NoGenerationAfterClose` | rev. 1 | -| `NoEscalate` | exhaustion drops only that child | `ExhaustionCloses` | rev. 1 | -| `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | rev. 1 | -| `Sequential` | children stopped one at a time | `ConcurrentShutdown` | rev. 1 | -| `ConcurrentStart` | constructions not serialised (checked with `max_children` 2: with a cap of one the cap alone serialises) | `SerialConstruction` | Codex 1 | -| `StuckFailsAll` | any stuck child fails the supervisor | `StuckIsolated` | agy P0 | -| `PublishAfterClose` | a constructing replacement publishes after close | `NoPublishAfterStop` | agy 3 | -| `OverlapGeneration` | the replacement starts before the old cleanup ended | `AtMostOneGeneration` | Codex 7 | -| `PublishBeforeReady` | a replacement is published at construction | `PublishedIsLive` | Codex 7 | -| `PrematureOk` | `Ok` committed at admission | `OkMeansReady` | Codex 7 | -| `UnchargedRetry` | a failed replacement's retry is not charged | `ChargedRestarts` | Codex 7 | -| `ChargeFirstStart` | a first start is charged | `ChargedRestarts` | agy 2 | -| `TransientNormalRestart` | Transient restarts after a normal exit | `RestartFollowsPolicy` | agy 2, Codex 8 | -| `PermanentRetires` | Permanent is not restarted | `RestartFollowsPolicy` | Codex 8 | -| `RetryFirstStart` | a failed first start is retried | `FirstStartNeverRetried` | agy 2 | -| `EarlyFailedReply` | `FailedToStart` returned before cleanup | `FailedAfterCleanup` | agy 2 | -| `LateFailedOverwrite` | a pre-shutdown failure reported as `Aborted` | `FailureCauseKept` | agy 7 | -| `NoRefund` | a cancelled restart keeps its reservation | `ReservationsMatch` | agy 4, Codex 6 | -| `ExpireReserved` | window expiry releases a reservation | `ReservationsMatch` | Codex 6 | -| `TermReplyEarly` | `terminate_child` returns before removal | `TermOkMeansRemoved` | agy 7 | -| `NoAbort` (liveness) | a start racing shutdown is not cancelled | `StartAnswered` | rev. 1 | -| `StuckBackoff` (liveness) | shutdown does not cancel a pending restart | `ClosedLeadsToDone` | rev. 1 | - -All 27 produce their counterexample on the named property when run one at a time, each in 1-5 s. (Run in parallel, TLC instances race on the standard modules TLC extracts to `/tmp`; `negative_checks.py` runs them serially.) +| `IgnoreMax` | admission ignores the cap | `MaxChildrenBound` | main | +| `UncountedRestart` | a crashed child frees its slot until it restarts | `MaxChildrenBound` | main | +| `RestartTemporary` | Temporary follows the Permanent rule | `TemporaryNeverRestarted` | main | +| `RestartTerminated` | termination does not stop a restart | `TerminatedNeverRestarted` | main | +| `AcceptDuringShutdown` | a start admitted and committed after close | `NoStartAfterShutdown` | main | +| `RestartAfterClose` | a pending restart proceeds after close | `NoGenerationAfterClose` | backoff | +| `NoEscalate` | exhaustion drops only that child | `ExhaustionCloses` | main | +| `EarlyDone` | done before every child finished | `NoChildOutlivesSupervisor` | main | +| `SkipSignal` | the fan-out skips a child | `ConcurrentShutdown` | main | +| `ConcurrentStart` | constructions not serialised | `SerialConstruction` | main, cap 2 | +| `LaterOp` | the loop admits a later start first | `FifoOrder` | main | +| `StuckFailsAll` | any stuck child fails the supervisor | `StuckIsolated` | main | +| `MisclassifyStuck` | a retiring stuck child classified as restartable | `StuckIsolated` | main | +| `PublishAfterClose` | a replacement publishes after close | `NoPublishAfterStop` | main | +| `PublishAfterExit` | a generation stays published after it exits | `PublishedIsLive` | main | +| `OverlapGeneration` | the replacement starts before the old cleanup ended | `AtMostOneGeneration` | main | +| `PublishBeforeReady` | a replacement is published at construction | `PublishedIsLive` | main | +| `PrematureOk` | `Ok` committed at admission | `OkMeansReady` | main | +| `UnchargedRetry` | a failed replacement's retry is not charged | `ChargedRestarts` | main | +| `ChargeFirstStart` | a first start is charged | `ChargedRestarts` | main | +| `TransientNormalRestart` | Transient restarts after a normal exit | `RestartFollowsPolicy` | main | +| `TransientNormalRestart` | the same at zero budget: a normal exit exhausts the budget | `RestartFollowsPolicy` | main, budget 0 | +| `PermanentRetires` | Permanent is not restarted | `RestartFollowsPolicy` | main | +| `RetryFirstStart` | a failed first start is retried | `FirstStartNeverRetried` | main | +| `EarlyFailedReply` | `FailedToStart` before cleanup | `FailedAfterCleanup` | main | +| `EarlyRemove` | a failed start removed with its generation still alive | `FailedAfterCleanup` | main | +| `LateFailedOverwrite` | a pre-shutdown failure reported as `Aborted` | `FailureCauseKept` | main | +| `NoRefund` | a cancelled restart keeps its reservation | `ReservationsMatch` | backoff | +| `ExpireReserved` | window expiry releases a reservation | `ReservationsMatch` | backoff | +| `OverReserve` | a restart reserved beyond the budget | `BudgetBounded` | main | +| `TermReplyEarly` | `terminate_child` returns before removal | `TermResultsSound` | main | +| `SlotLookup` | an id call reaches a newer generation | `TermResultsSound` | main | +| `QueuedTermOk` | a call queued at shutdown waits and may return `Ok` | `TermResultsSound` | main | +| `StuckResultLost` | `CleanupStuck` not committed; later turns into `Ok` | `TermResultsSound` | main | +| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` | callers | +| `NoDeadline` | no cleanup deadline | `StartAnswered` | callers | +| `TermLost` | a found call never signals the child | `TerminateAnswered` | callers | +| `Sequential` | the fan-out waits for each signalled child | `FanoutCompletes` | callers | +| `StuckBackoff` | shutdown does not cancel a pending restart | `ClosedLeadsToDone` | live | +| `LostRestart` | a decided restart is never attempted | `RecoveryProgress` | live | + +CONTROLS_NOTE Not modelled: handler bodies, mailboxes and requests (`Lifecycle.tla`; -rank 11's models); real time (backoff delays, the window clock, deadlines -as durations); native-capacity reservations (a counting resource like the -cap, checked by tests); nesting (a failed dynamic supervisor is one child of -`NestedSupervision.tla`'s parent); the fan-out as separate steps (see -[Shutdown](#shutdown-of-all-children)). Trace validation of real runs is part -of the implementation (verification rule). Apalache inductive invariants -are a follow-up. +rank 11's models); real time; native-capacity reservations and slot +finalization as separate steps (a counted slot and its removal stand for +them); nesting (one child of `NestedSupervision.tla`'s parent); repeated +churn of one slot (each slot is one start call; a reusable-slot model would +need fresh identities). Apalache inductive invariants are a follow-up. + +## Test plan: relating the model to the code + +Standing rule: a model counts only once real runs are checked against it. +The implementation must emit lifecycle-trace events (feature +`lifecycle-trace`, as `TraceSupervision.tla` uses today) that map one to one +onto `DynamicSupervisor.tla` actions, and ship trace-validation tests that +replay real runs as model behaviours. + +**Events.** Each carries the supervisor's trace id, the slot (`DynamicChildId`), +the generation, and where noted the queue ticket or call id. + +| Event | Emitted when | Model action | +|---|---|---| +| `DReq` (slot, ticket) | `start_child` queued, or rejected because closed | `Request` | +| `DAdmit` (slot, ticket, outcome: admitted / max) | the loop takes a start | `LoopAdmit` | +| `DReject` (slot) | shutdown answers a queued start | `RejectQueued` | +| `DReady` (slot, gen) | readiness commits | `Ready` | +| `DIgnore` (slot, gen) | the factory returned ignore | `Ignore` | +| `DStartFail` (slot, gen, reason) | factory error or panic, stop or panic in `started()` | `StartFail` | +| `DStartTimeout` (slot, gen) | `startup_timeout` fired | `StartTimeout` | +| `DAbort` (slot, gen) | shutdown cancelled a construction | `AbortStart` | +| `DExit` (slot, gen, reason) | a running generation began exiting on its own | `SelfExit` | +| `DStopSeen` (slot, gen, reason) | a signalled generation began exiting (Shutdown/Killed) | `StopObserved` | +| `DCleaned` (slot, gen, cleanup_failure, ticket) | owned cleanup finished; exit notice queued | `CleanupDone` | +| `DStuck` (slot, gen, kind) | a deadline was missed | `CleanupStuck` | +| `DStuckDone` (slot) | a stuck child finally ended | `StuckDone` | +| `DDecide` (slot, ticket?, outcome: removed / backoff / restart / exhausted) | the loop handled an exit | `Decide` | +| `DRetryQueued` (slot, ticket) | the backoff timer fired | `BackoffFire` | +| `DRetry` (slot, gen, ticket) | the loop took a retry | `BackoffDone` | +| `DCancel` (slot) | a pending restart was cancelled and refunded | `CancelBackoff` | +| `DTermCall` (call, slot, named gen, ticket) | `terminate_child`/`terminate_slot` called | `TermCall` | +| `DTermLookup` (call, outcome: found / notfound / stuck) | the loop handled the call | `LoopTerm` | +| `DTermReject` (call) | shutdown answered a queued call | `RejectQueuedTerm` | +| `DTermDone` (call, result) | the call resolved `Ok` | `TermResolve` | +| `DClose`, `DSignal` (slot), `DDone` | shutdown began, a child was signalled, all children finished | `Close`, `SupSignal`/`SupSignalStep`, `SupDone` | +| `DExpire` | a grant left the window | `WindowExpire` | + +The trace checker (`trace_check.py`) learns a `dynamic-v1` schema that +replays these events as a `TraceDynamic.tla` behaviour, with the instance +constants taken from the run (slots, modes, calls, cap, budget, backoff). +The fan-out is emitted as one `DSignal` per child; the checker accepts the +signals of one shutdown as a single `SupSignal` step when no child event +falls between them, and checks the refinement configuration otherwise. + +**Tests the implementation must ship** (`concurrency/tests/dynamic_trace.rs`, +recorded in the `tla-traces` CI job, plus generated scripts in +`supervisor_policies`): + +1. Admission order: two starts at the cap; the second rejected with its + arguments. +2. First-start failures: factory error, panic in `started()`, startup + timeout, ignore; each answered after cleanup; one with a stuck cleanup. +3. Restart table per type and reason, with and without backoff, including a + failed replacement re-queued and retried, and a recorded cleanup failure + restarting a Transient child. +4. Budget: exhaustion at zero and at one, reservation refund on terminate + and on shutdown during backoff. +5. Terminate: by id, by slot, a stale id after a successful replacement, a + second call, a call during a restart attempt, a call queued at + shutdown, a stuck child. +6. Shutdown with several children, one stuck; a start racing shutdown; a + commit racing shutdown. +7. Seeded random scripts (as `supervision_trace`) for coverage. + +**Expected action coverage** (`trace_coverage.py`): every action of +`DynamicSupervisor.tla` reached by the dedicated tests, except the fault +branches; `WindowExpire` only in paused-time tests. The coverage baseline +must list them so that a regression fails CI, as for the static models. ## Alternatives considered @@ -887,8 +1089,11 @@ are a follow-up. keeps the spec, `restart_child`, `delete_child`; `Supervision.tla` and `Cohorts.tla` changes). - **Apalache inductive invariants** for `DynamicSupervisor.tla`. -- **Implementation acceptance:** trace validation, the fan-out abstraction - check, evidence that dynamic tables, retained arguments and capacity +- **Unified start (0.9):** the post-start continuation, `ChildStart::Ignore` + and `Init` start data are that design's; this design consumes them. +- **Introspection v1:** owns `ChildInfo`, `ChildCounts`, `ChildKey` and the + "blocked in terminate_child" report. +- **Implementation acceptance:** the [test plan](#test-plan-relating-the-model-to-the-code), evidence that dynamic tables, retained arguments and capacity reservations add nothing to the ordinary actor and message paths (Codex closing note). @@ -931,7 +1136,7 @@ was checked against the code and sources before acting on it. | 8 | ethrex `Handler` lacks the monitor check on the entry; awaiting `start_child` stalls accepts; `terminate_child` also skips `stopped()` | **Accepted.** The guard is in both sketches; the handshake moves out of `started()` (required now that starts are serialised); the `stopped()` note covers `terminate_child` | | 9 (P2) | Jitter seed per slot; drop `Sync`; unconditional `Clone`, slot eviction; structured `FailedToStart` | **Accepted**, all four | -### Open questions for round 2 +### Round 1 open questions (decided as R1-R5) 1. **Holding the loop during `terminate_child`** (OTP blocks the supervisor up to 2 × `shutdown_timeout`): faithful, but a slow-to-stop child delays From 28067e7e8e118fda40fca6872a359f6bb2df3d57 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:42:46 -0300 Subject: [PATCH 25/52] docs: client mapping revision 3 (continuation, temporary rationale, owners release on Down) --- .../dynamic-supervisor-client-mapping.md | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index c5d43073..7fd21a73 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -158,14 +158,14 @@ fails only itself. | ethrex today | With the dynamic supervisor | |---|---| | `.start()`, unmanaged, on the caller's runtime | `start_child(args)` on a `DynamicSupervisor`, placed on the `p2p` group | -| handshake inside `started()` (up to `HANDSHAKE_TIMEOUT`, 10 s) | **required change:** `started()` sends itself a `Handshake` message and returns; the handshake runs as the first handler turn, still under `HANDSHAKE_TIMEOUT`, and a failure ends the actor (a Temporary exit, not a start failure). Otherwise one silent peer would hold the supervisor's control loop, and with it every accept and dial, for the whole `startup_timeout` | +| handshake inside `started()` (up to `HANDSHAKE_TIMEOUT`, 10 s) | **required change:** the handshake moves to the post-start continuation of the unified-start design (0.9, OTP's `handle_continue`): it runs right after `started()` returns and before any message, still under `HANDSHAKE_TIMEOUT`, and a failure ends the actor (a Temporary exit, not a start failure). Otherwise one silent peer would hold the supervisor's control loop, and with it every accept and dial, for the whole `startup_timeout`. A self-sent first message is not equivalent: a message enqueued by another actor first would run before it | | no owner; shutdown does not stop connections | the supervisors are owned by the system, nested under the p2p supervisor (recommended) or as roots admitted after the peer table; connections stop before the table, all at once, each killed after its `shutdown_timeout` | | `Semaphore(150)` plus `_admission_permit` in actor state | `max_children = Some(150)` on an inbound supervisor; `Rejected { MaxChildren, args }` returns the `ConnArgs`, so the accept loop can still send an RLPx Disconnect or just drop the stream, as today | | outbound uncapped | a second supervisor for dials without `max_children` | | `ConnectionId` (process-global counter) | the table's `MonitorRef` (or the connection's `ActorId`, never reused, M4) | | `stopped()` sends `remove_peer(node_id, connection_id)` | the peer table monitors the connection when it registers it; `Handler` removes the entry only if it still holds that monitor. `Down` arrives exactly once whatever ends the actor (X1), including kills and a skipped `stopped()` | | timed-out registration undone by an unconditional `remove_peer` | the table's monitor fires for that registration too; a monitor installed after the connection died gets `Down { NoProc }` at once (E4) | -| connections never restarted | `DynamicChildSpec::temporary(..)`: no restarts, no budget use, args (the `TcpStream`, the `Node`) moved without `Clone` | +| connections never restarted | `DynamicChildSpec::temporary(..)`, chosen deliberately over the Permanent default: a dropped peer must not be redialled by the supervisor (the TCP stream is gone and discovery decides whom to dial), so the connection is one-shot. No restarts, no budget use, arguments (the `TcpStream`, the `Node`) moved without `Clone` | | first-wins on duplicates | unchanged: domain policy in the table (rank 12 may later own it) | | `teardown()` and `tx_broadcaster.remove_peer` in `stopped()` | moved to owners that act on the `Down` (below): `stopped()` is skipped for a non-trapping connection on `terminate_child`, on shutdown and after a kill | | raw `tokio::spawn` outbound writer holding the socket sink ([server.rs:1530-1557](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L1530-L1557)) | **required change:** an owned helper (`ctx.spawn_helper`), cancelled at exit and joined within the cleanup deadline, with an explicit policy for frames still queued (drop them; the peer is being disconnected). Until then the writer can outlive the actor and keep the socket open after the supervisor freed the slot, so `max_children` would bound slots, not sockets | @@ -197,10 +197,10 @@ match inbound.current()?.start_child(ConnArgs::receiver(context.clone(), peer_ad } // Connection actor (server.rs): the handshake leaves started(). -#[started] -async fn started(&mut self, ctx: &Context) { - let _ = ctx.send(Handshake); // first turn, OTP's handle_continue -} +// Handshake in the post-start continuation (unified start, 0.9): runs +// after started() returns and before any message. +#[continue_start] +async fn handshake(&mut self, ctx: &Context) -> ContinueResult { /* existing code */ } // Peer table (peer_table.rs): registration installs the monitor... async fn handle_new_connected_peer(&mut self, msg: NewConnectedPeer, ctx: &Context) -> bool { @@ -227,17 +227,27 @@ impl Handler for PeerTableServer { ### Skipped teardown: who releases what `stopped()` does not run for a non-trapping connection that is terminated, -shut down or killed (v0.8, decision 2). `trap_exit` would bring it back for -`terminate_child` and shutdown, but not for a kill, so each resource the -hook releases needs an owner that also hears about the death (Codex 12): +shut down or killed (v0.8, decision 2), and `trap_exit` does not cover a +kill. So each resource the hook releases today gets an owner that holds an +exact registration and releases on the connection's `Down`, as an Erlang +process that owns a resource monitors its user. Nothing is sent from +`Drop`: state destruction runs before publication (exit phases 5 and 6), +so such a message would reach the peer table before the `Down` and could +pick the dying peer as its own alternate (agy 5), and an enqueued message +is not a completed release (Codex F9). + +| Released in `stopped()` today | Registration (identity stored by the owner) | Release on `Down` | +|---|---|---| +| peer-table entry and discv5 session | `new_connected_peer` stores the monitor reference in `PeerData` | the peer table removes the entry only if it still holds that monitor | +| tx-broadcaster index | the broadcaster's `add_peer` records the connection's `ActorId` and monitors it | the broadcaster frees the index only if it is still owned by that `ActorId` | +| in-flight transaction reservations (`clear_in_flight_txs`) | the mempool records the requesting connection's `ActorId` with each in-flight hash when the request is made | the peer table, after removing the entry, calls `mempool.release_in_flight(actor_id)`: synchronous, exact, removes only hashes still held by that id | +| re-request to alternates (`retry_on_alternates`) | the same per-hash records | best effort, done by the peer table after the removal, with a local lookup of live peers (no request to itself, so no self-request deadlock); skipped once shutdown began (`Down` reason `Shutdown` from the supervisor's shutdown) so a node stop does not issue ~150 re-requests | +| cell availability and announced-hash budget (`clear_peer_cell_availability`, `forget_peer_announcements`) | keyed today by node id with a "started session" check; becomes keyed by (node id, connection `ActorId`) | the peer table releases the entries of that `ActorId` only, so a second session of the same node keeps its own | +| disconnect metrics | | the peer table's `Down` handler (it has the final reason) | -| Released in `stopped()` today | Owner after the migration | -|---|---| -| peer-table entry and discv5 session (`remove_peer`) | the peer table's `Down` handler, guarded by its monitor | -| tx-broadcaster index (`tx_broadcaster.remove_peer`) | forwarded by the peer table from the same `Down`, carrying the node id and the monitor reference so a stale forward cannot drop a newer connection's index; or the broadcaster monitors connections itself | -| in-flight transaction reservations and re-requests to alternates (`teardown()`, [server.rs:449-511](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L449-L511)) | a guard value in the actor state whose `Drop` clears the hashes in the mempool (runs on every exit, kills included, before the slot frees); the re-request to alternates, which is async, becomes a message to the peer table sent from that `Drop`, guarded by the connection identity | -| announcement and availability bookkeeping | same guard | -| disconnect metrics | the `Down` handler (it has the final reason) | +`Drop` keeps only what is local to the actor (dropping its own buffers). +Releases happen after the connection is gone from the table, so no +re-request can target it. ### What disappears @@ -252,8 +262,9 @@ About 45-60 of the ~135 hand-written lines (counted on `main`): | inbound semaphore, `try_acquire_owned`, `_admission_permit` field and parameter | ~15 | replaced by `max_children` | | "connections are not admitted" comment and unmanaged `.start()` | | connections owned by the system | -Added: the `Handshake` first message (moving existing code), the writer as an -owned helper, the teardown guard and the broadcaster forward. Net lines are +Added: the handshake as a post-start continuation (moving existing code), +the writer as an owned helper, exact owner registrations in the mempool and +broadcaster, and the release logic in the `Down` handlers. Net lines are roughly even; what changes is that every path out of a connection is covered. #7435's integration test (171 lines) stays as a regression test, now exercising the monitor guard. From 7c6115bf0b9a15473031b5d7987848c28fff06b6 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:42:56 -0300 Subject: [PATCH 26/52] docs: continuation wording --- docs/design/dynamic-supervisor-client-mapping.md | 2 +- docs/design/dynamic-supervisor.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index 7fd21a73..de4048ab 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -284,7 +284,7 @@ now exercising the monitor guard. ### Things to watch in a migration - **Serialised starts.** Anything slow left in `started()` holds every - accept and dial. Keep `started()` to state setup and the self-send. + accept and dial. Keep `started()` to state setup; the handshake is the continuation. - **Stuck cleanup.** A connection's own exit gets one `shutdown_timeout` for its owned cleanup (E3 row 4, no kill); a terminated or shut-down one gets the signal, the kill after `shutdown_timeout` and a second interval. diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 2203347a..6f5cd083 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -129,7 +129,7 @@ rule: | # | Question | Decision | |---|---|---| -| Q1 | Concurrent or serialised construction | **Serialised, as OTP.** One control loop runs each start (admission, factory, `started()`) to completion before the next operation. Factories still run on the target group. Long work (ethrex's RLPx handshake) moves out of `started()` into the first message, OTP's `handle_continue` idiom | +| Q1 | Concurrent or serialised construction | **Serialised, as OTP.** One control loop runs each start (admission, factory, `started()`) to completion before the next operation. Factories still run on the target group. Long work (ethrex's RLPx handshake) moves out of `started()` into the post-start continuation, OTP's `handle_continue` (decision G1) | | Q2 | Heterogeneous children | The typed template is enough for v1 | | Q3 | Defaults | Elixir's: `max_restarts = 3`, `restart_window = 5 s`; backoff **opt-in** (`None` by default); `max_children = None` | | Q4 | Group capacity across restarts | The slot holds a **transferable native-capacity reservation** from admission to removal, separate from any generation's scope lease | @@ -761,7 +761,7 @@ As for any actor (exit-signals design, phases of an exit, with rank 11): - **No queueing** at `max_children`: a full supervisor rejects. Waiting for a slot is rank 17's admission. - **No parallel construction:** one slow `started()` delays the supervisor's - other operations (OTP's behaviour). Use the first message for slow work. + other operations (OTP's behaviour). Use the post-start continuation for slow work. - **No exactly-once start under cancellation:** a dropped `start_child` future may still produce a supervised child. - **No state or queue carried across a restart** (V4). From 3b1113d1d60cd824023ae749fe2cbdbce0f6cf97 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:43:20 -0300 Subject: [PATCH 27/52] docs: round-2 review resolutions and changelog --- CHANGELOG.md | 6 +++++- docs/design/dynamic-supervisor.md | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 624434ed..bd50bfbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,7 +57,11 @@ serialised starts, Elixir's 3-in-5 s budget with opt-in backoff, `Shutdown` on exhaustion, arguments returned on rejection) and both reviews (stuck-child isolation, reservation protocol, history-based - invariants), with 27 named negative controls. + invariants), with 27 named negative controls. Revision 3 applies the + second reviews and the round-two decisions: one FIFO operation queue, + terminate lookup when the loop handles the call, stuck classification by + exit reason, caller liveness without assuming cleanup ends, `ignore` and + start data, a trace-validation test plan, and 40 negative controls. ## 0.7.3 — 2026-11-10 diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 6f5cd083..8c5af551 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -1148,3 +1148,30 @@ was checked against the code and sources before acting on it. silently weaken Permanent. 3. **The one-step fan-out abstraction** in the model: acceptable with trace validation, or should the model interleave signals with child steps? + +### Round 2 + +Codex (F1-F9 P1, F10 P2) and agy (1-5 P1, 6 P2), both "approve after the +listed changes", no P0. Each was checked against the model and the doc. + +| Finding | Resolution | +|---|---| +| **Codex F1 = agy 2**: id calls looked up at call time in prose but not in the model; queued call at shutdown could become `Ok`; `Release` freed the loop before the terminate resolved; `CleanupStuck` could later become `Ok` | **Accepted.** Lookup happens when the loop handles the call; the model carries the generation the caller named and two calls (by id and by slot). A call still queued at shutdown gets `Stopped`. The loop stays held until the call resolves. `CleanupStuck` is committed at the deadline and never changes. Duplicates are not shared waiters: a second call is handled after the first and gets `NotFound` or `CleanupStuck` (Elixir). `TermResultsSound`, `TermResultStable`; controls `SlotLookup`, `QueuedTermOk`, `StuckResultLost`, `TermReplyEarly` | +| **Codex F2**: serialisation checked, arrival order not | **Accepted.** One FIFO queue of starts, terminate calls, exit notices and retries; history variable on the served ticket; `FifoOrder`, control `LaterOp`. The three-slot targeted instance has more arrivals | +| **Codex F3 = agy 1**: Transient normal exit with stalled cleanup listed as both isolated and restartable; `StuckIsolated` trusted the action's label; a wrong decision that exhausts the budget escaped `RestartFollowsPolicy` | **Accepted.** A stall is not a recorded cleanup failure: classification by exit reason only, Transient-normal stuck children are isolated. `StuckIsolated` re-derives "restart" from the table. `RestartFollowsPolicy` checks every decision (reservation, attempt, exhaustion). Controls `MisclassifyStuck` and `TransientNormalRestart` at zero budget | +| **Codex F4**: caller liveness assumed cleanup finishes | **Accepted.** `CallerSpec` gives progress only to loop work and deadlines; `StartAnswered` and `TerminateAnswered` hold under it; `ClosedLeadsToDone` and `RecoveryProgress` keep `Spec`. Control `NoDeadline` | +| **Codex F5 = agy 3**: `FailedAfterCleanup` checked a label | **Accepted.** It now requires removal with no live generation, or a recorded deadline failure; control `EarlyRemove` | +| **Codex F6 = agy 3**: `Sequential`'s witness signalled a queued, unadmitted start | **Accepted.** Fan-out targets admitted children only. `Sequential` now runs on the interleaved refinement with two admitted children (`max_children` 2) against `FanoutCompletes`; `ConcurrentShutdown` gets its own control `SkipSignal` | +| **agy 3**: `PublishedIsLive` ignored the slot state; `BudgetBounded`, `RecoveryProgress`, `TerminateAnswered` had no controls | **Accepted.** `PublishedIsLive` requires `running` (control `PublishAfterExit`); controls `OverReserve`, `LostRestart`, `TermLost`. Every invariant and liveness property now has a control; the two action properties do not, and the doc says so | +| **Codex F7**: 2 × T + 1 s cannot cover an unbounded fan-out; timer epoch is a deviation | **Accepted.** The nested adapter waits for the inner supervisor by default (OTP's `infinity`); a finite value is a reporting bound keeping the barrier. The per-child timer epoch is listed as an inherited E3 deviation (alignment row 16) | +| **Codex F8**: destruction of retained arguments | **Accepted.** Slot finalization is distinct from generation cleanup, runs as a tracked task on the group (never on the loop or under a lock), after the removal decision, so its failure never triggers a restart; a hang makes the slot stuck as retiring | +| **Codex F9 = agy 5**: teardown protocol incomplete; a message from `Drop` arrives before `Down` | **Accepted, with decision R5.** No message from `Drop`. Each owner (peer table, broadcaster, mempool records) stores the connection's identity at registration and releases exactly on `Down`; re-requests are best effort, after removal, with local lookup, skipped during shutdown | +| **agy 4**: backoff `None` in the model always went through a backoff state | **Accepted.** `BackoffOn` constant: without backoff the decision reserves, stamps and runs the attempt in the same turn; a failed attempt re-queues its exit at the tail (`try_again_restart`). Both variants are checked | +| **Codex F10 (P2)**: two slots are bounded evidence | **Accepted.** No sufficiency claim; a three-slot targeted instance (FIFO, cap with a restarting child, shared budget); repeated churn of one slot stays out of scope (it would need fresh identities) | +| **agy 6 (P2)** | **Accepted.** `SupervisedDynamicSupervisor` and spec builder methods shown; `ready` documented as "state is Running"; `latePub` catches any change to a non-zero value | + +Also from the OTP/Elixir/Gleam comparison: decisions G1-G5 (continuation, +`temporary()` for connections, `ignore`, start data, shared introspection +types), and the standing rule that models are related to the code: the +[test plan](#test-plan-relating-the-model-to-the-code) lists the events, their +action mapping, the required tests and the expected coverage. From 9b96aabaedaf85a52ad7dbe945e6e9454e58904c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:45:44 -0300 Subject: [PATCH 28/52] verify: inline fairness conditions (TLC) --- verification/tla/DynamicSupervisor.tla | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index bf6b8877..01f56cbd 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -546,25 +546,23 @@ Act == Next == Act /\ GhostUpdate -F(A) == WF_vars(A /\ GhostUpdate) - \* Loop work and deadlines only: no user cleanup is assumed to finish. CallerFairness == /\ \A s \in Slots : - /\ F(LoopAdmit(s)) /\ F(RejectQueued(s)) /\ F(StartTimeout(s)) - /\ F(AbortStart(s)) /\ F(CleanupStuck(s)) /\ F(Decide(s)) - /\ F(BackoffFire(s)) /\ F(BackoffDone(s)) /\ F(CancelBackoff(s)) - /\ \A c \in TermCalls : F(RejectQueuedTerm(c)) /\ F(LoopTerm(c)) /\ F(TermResolve(c)) - /\ F(SupSignal) /\ F(SupSignalStep) /\ F(SupDone) + /\ WF_vars(GhostUpdate /\ LoopAdmit(s)) /\ WF_vars(GhostUpdate /\ RejectQueued(s)) /\ WF_vars(GhostUpdate /\ StartTimeout(s)) + /\ WF_vars(GhostUpdate /\ AbortStart(s)) /\ WF_vars(GhostUpdate /\ CleanupStuck(s)) /\ WF_vars(GhostUpdate /\ Decide(s)) + /\ WF_vars(GhostUpdate /\ BackoffFire(s)) /\ WF_vars(GhostUpdate /\ BackoffDone(s)) /\ WF_vars(GhostUpdate /\ CancelBackoff(s)) + /\ \A c \in TermCalls : WF_vars(GhostUpdate /\ RejectQueuedTerm(c)) /\ WF_vars(GhostUpdate /\ LoopTerm(c)) /\ WF_vars(GhostUpdate /\ TermResolve(c)) + /\ WF_vars(GhostUpdate /\ SupSignal) /\ WF_vars(GhostUpdate /\ SupSignalStep) /\ WF_vars(GhostUpdate /\ SupDone) \* Plus progress of children and cleanup. Fairness == /\ CallerFairness /\ \A s \in Slots : - /\ F(Ready(s)) - /\ F(\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) - /\ F(\E f \in CleanupFails : CleanupDone(s, f)) - /\ F(StuckDone(s)) + /\ WF_vars(GhostUpdate /\ Ready(s)) + /\ WF_vars(GhostUpdate /\ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) + /\ WF_vars(GhostUpdate /\ \E f \in CleanupFails : CleanupDone(s, f)) + /\ WF_vars(GhostUpdate /\ StuckDone(s)) Spec == Init /\ [][Next]_vars /\ Fairness CallerSpec == Init /\ [][Next]_vars /\ CallerFairness From 5dae4de10bdb82db7c04782755b544bd70c8917f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:13:47 -0300 Subject: [PATCH 29/52] verify: smaller weekly DynamicSupervisor instance --- verification/tla/MCDynamicSupervisorDeep.cfg | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 7f08e524..53747eef 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -1,5 +1,7 @@ \* Weekly (--deep): budget 2, two window releases, max_children 2, -\* backoff on, two terminate calls. Safety. +\* backoff on, one terminate call, no recorded cleanup failures (with two +\* calls and cleanup failures it passed 4 GB of TLC state in 28 minutes). +\* Safety. SPECIFICATION Spec CONSTANTS N = 2 @@ -9,10 +11,10 @@ CONSTANTS MaxExpire = 2 SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} - CleanupFails = {FALSE, TRUE} - TermCalls <- MCCalls2 - CallSlot <- MCCallSlot2 - CallKind <- MCCallKind2 + CleanupFails = {FALSE} + TermCalls <- MCCalls1 + CallSlot <- MCCallSlot1 + CallKind <- MCCallKind1 BackoffOn = TRUE FanoutSteps = FALSE Fault = "None" From 4eaf491a68aa3dc2606cddfa0a70dade17fa0ea1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:14:26 -0300 Subject: [PATCH 30/52] verify: fairness actions before GhostUpdate --- verification/tla/DynamicSupervisor.tla | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 01f56cbd..2b834c7d 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -549,20 +549,20 @@ Next == Act /\ GhostUpdate \* Loop work and deadlines only: no user cleanup is assumed to finish. CallerFairness == /\ \A s \in Slots : - /\ WF_vars(GhostUpdate /\ LoopAdmit(s)) /\ WF_vars(GhostUpdate /\ RejectQueued(s)) /\ WF_vars(GhostUpdate /\ StartTimeout(s)) - /\ WF_vars(GhostUpdate /\ AbortStart(s)) /\ WF_vars(GhostUpdate /\ CleanupStuck(s)) /\ WF_vars(GhostUpdate /\ Decide(s)) - /\ WF_vars(GhostUpdate /\ BackoffFire(s)) /\ WF_vars(GhostUpdate /\ BackoffDone(s)) /\ WF_vars(GhostUpdate /\ CancelBackoff(s)) - /\ \A c \in TermCalls : WF_vars(GhostUpdate /\ RejectQueuedTerm(c)) /\ WF_vars(GhostUpdate /\ LoopTerm(c)) /\ WF_vars(GhostUpdate /\ TermResolve(c)) - /\ WF_vars(GhostUpdate /\ SupSignal) /\ WF_vars(GhostUpdate /\ SupSignalStep) /\ WF_vars(GhostUpdate /\ SupDone) + /\ WF_vars((LoopAdmit(s)) /\ GhostUpdate) /\ WF_vars((RejectQueued(s)) /\ GhostUpdate) /\ WF_vars((StartTimeout(s)) /\ GhostUpdate) + /\ WF_vars((AbortStart(s)) /\ GhostUpdate) /\ WF_vars((CleanupStuck(s)) /\ GhostUpdate) /\ WF_vars((Decide(s)) /\ GhostUpdate) + /\ WF_vars((BackoffFire(s)) /\ GhostUpdate) /\ WF_vars((BackoffDone(s)) /\ GhostUpdate) /\ WF_vars((CancelBackoff(s)) /\ GhostUpdate) + /\ \A c \in TermCalls : WF_vars((RejectQueuedTerm(c)) /\ GhostUpdate) /\ WF_vars((LoopTerm(c)) /\ GhostUpdate) /\ WF_vars((TermResolve(c)) /\ GhostUpdate) + /\ WF_vars((SupSignal) /\ GhostUpdate) /\ WF_vars((SupSignalStep) /\ GhostUpdate) /\ WF_vars((SupDone) /\ GhostUpdate) \* Plus progress of children and cleanup. Fairness == /\ CallerFairness /\ \A s \in Slots : - /\ WF_vars(GhostUpdate /\ Ready(s)) - /\ WF_vars(GhostUpdate /\ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) - /\ WF_vars(GhostUpdate /\ \E f \in CleanupFails : CleanupDone(s, f)) - /\ WF_vars(GhostUpdate /\ StuckDone(s)) + /\ WF_vars((Ready(s)) /\ GhostUpdate) + /\ WF_vars((\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) /\ GhostUpdate) + /\ WF_vars((\E f \in CleanupFails : CleanupDone(s, f)) /\ GhostUpdate) + /\ WF_vars((StuckDone(s)) /\ GhostUpdate) Spec == Init /\ [][Next]_vars /\ Fairness CallerSpec == Init /\ [][Next]_vars /\ CallerFairness From 53946c0314cbc05281dabde2687774cea393f83f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:27:13 -0300 Subject: [PATCH 31/52] docs: revision 3 TLC results --- docs/TESTING.md | 21 +++++------ docs/design/dynamic-supervisor.md | 8 +++- verification/tla/README.md | 63 +++++++++++++++---------------- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index 06504f70..68ce6056 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -925,17 +925,16 @@ found one model gap (a running child may exit with `Shutdown` while the supervisor is open, via `ChildHandle::shutdown`); no implementation deviation. `verification/tla/DynamicSupervisor.tla` models the rank 12a design (not -implemented, revision 2): `start_child` racing crashes, restarts, -`terminate_child`, `max_children`, the budget reservation protocol, stuck -children and shutdown, with OTP's serialised control loop. TLC checks 23 -safety invariants (among them: no child outlives the supervisor, the cap -counts restarting children, restarts follow the table in both directions, -every restart attempt is charged once, no commit, publication or generation -after close or terminate, a child stuck while retiring fails only itself), -`NeverReopens` and four liveness properties. `MCDynamicSupervisor`, -`MCDynamicSupervisorLive` and `MCDynamicSupervisorZero` run on every PR, `MCDynamicSupervisorDeep` weekly. -Twenty-seven named negative controls each produce a counterexample. See the -[design](design/dynamic-supervisor.md#what-the-model-establishes). +implemented, revision 3): OTP's serialised control loop with one FIFO queue, +`start_child` and `terminate_child` racing crashes, restarts, `max_children`, +the budget reservation protocol, ignore, stuck children and shutdown. TLC +checks 23 safety invariants, two action properties and five liveness +properties (callers answered even if cleanup never ends; shutdown and +recovery complete when it does). Six configurations run on every PR and one +weekly; 40 named negative controls each fail their property. The design's +[test plan](design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) +specifies the trace events and trace-validation tests the implementation +must ship. Apalache checks that the safety invariants of `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` are inductive (`Init => IndInv`, diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 8c5af551..8ac42946 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -921,7 +921,13 @@ one; no cutoff argument is claimed (Codex F10). | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -RESULT_ROWS +| `MCDynamicSupervisor` (per PR; safety, action properties; `Spec`) | 2 slots, every pair of restart types, two terminate calls (id, slot), `max_children` 1, budget 1, one window release, no backoff, exits `Normal`/`Panic`, start failures factory/`Normal`/timeout, ignore, cleanup failures | 9,237,075 | 30 | 2m13s | +| `MCDynamicSupervisorBackoff` (per PR; safety) | as above with backoff | 10,599,610 | 36 | 2m43s | +| `MCDynamicSupervisorZero` (per PR; safety) | as the first with budget 0 | 1,170,745 | 21 | 26 s | +| `MCDynamicSupervisorThree` (per PR; safety) | 3 Permanent crashing children, `max_children` 2, shared budget 1, FIFO with three arrivals | 442,239 | 30 | 11 s | +| `MCDynamicSupervisorCallers` (per PR; `StartAnswered`, `TerminateAnswered`, `FanoutCompletes` under `CallerSpec`) | 2 slots, `max_children` 2, one call, fan-out as separate steps | 3,413,878 | 29 | 8m48s | +| `MCDynamicSupervisorLive` (per PR; `ClosedLeadsToDone`, `RecoveryProgress` under `Spec`) | 2 slots, one call, backoff | 531,760 | 34 | 55 s | +| `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two releases, `max_children` 2, backoff, one call | 10,060,062 | 45 | 2m24s | Negative controls (`negative_checks.py`, run one at a time; each checks `TypeOK` and only the named property). Every invariant and liveness diff --git a/verification/tla/README.md b/verification/tla/README.md index ed0d04db..df20109f 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor{,Live,Zero,Deep}.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): `start_child` calls racing crashes, restarts, `terminate_child`, `max_children` (counting restarting children), the budget reservation protocol and its exhaustion, stuck children and concurrent shutdown, with OTP's serialised control loop. Twenty-seven named negative controls. Per PR: `MCDynamicSupervisor`, `MCDynamicSupervisorLive`, `MCDynamicSupervisorZero`; weekly: `MCDynamicSupervisorDeep`. | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor*.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): OTP's serialised control loop with one FIFO queue; `start_child` and `terminate_child` racing crashes, restarts, `max_children`, the budget reservation protocol, ignore, stuck children and shutdown. 40 named negative controls. Per PR: `MCDynamicSupervisor`, `Backoff`, `Zero`, `Three`, `Callers`, `Live`; weekly: `Deep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -584,35 +584,34 @@ All 22 named controls find their counterexample, each in a few seconds. ## Dynamic supervisor (rank 12a design) `DynamicSupervisor.tla` backs the [dynamic supervisor design](../../docs/design/dynamic-supervisor.md) -(revision 2), written before any implementation. Slots are `start_child` -calls made at any moment, each started child can get one `terminate_child`, -and one control loop serialises starts, exit handling and terminations as -OTP's supervisor process does. Each slot's restart type is chosen in `Init`. -Budget reservations, stamping, refunds and window expiry are explicit; -children can get stuck (before or after their death point) under the -stuck-isolation policy. Invariants about events (late commits, publications -and generations, forbidden restarts or retirements, restart attempts) read -history variables computed by `GhostUpdate` from each step's state change, -separately from the actions. - -Configurations: `MCDynamicSupervisor` (per PR; two slots, every pair of -restart types, `max_children` 1, budget 1; safety, `NeverReopens`), -`MCDynamicSupervisorLive` (per PR; liveness on a slightly smaller -instance), `MCDynamicSupervisorZero` (per PR; budget 0), -`MCDynamicSupervisorDeep` (weekly; budget 2, two window releases, -`max_children` 2; safety). Twenty-seven named negative controls (25 safety, -2 liveness), listed with their origin in the design's -[model section](../../docs/design/dynamic-supervisor.md#what-the-model-establishes), -run one at a time from `negative_checks.py`. - -Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`): - -| Configuration | Distinct states | Generated | Depth | Time | -|---|---|---|---|---| -| `MCDynamicSupervisor` (safety, `NeverReopens`) | 3,459,556 | 7,826,992 | 33 | 39 s | -| `MCDynamicSupervisorLive` (liveness) | 1,628,738 | 3,579,100 | 33 | 5m21s | -| `MCDynamicSupervisorZero` (safety, liveness) | 589,159 | 1,172,771 | 22 | 1m23s | -| `MCDynamicSupervisorDeep` (safety) | 31,620,430 | 95,086,511 | 42 | 5m06s | - -Not yet: Apalache inductive invariants and trace validation, which come +(revision 3), written before any implementation. `start_child` calls and +`terminate_child` calls (by the id the caller saw, or by slot) arrive at any +moment; one control loop takes starts, terminate calls, exit notices and +backoff retries from one FIFO queue, as OTP's supervisor process. Budget +reservations, stamping, refunds and window expiry are explicit; children can +ignore, fail, time out, exit, be stopped or killed, record a cleanup failure, +or get stuck. Event invariants read history variables computed by +`GhostUpdate`. `CallerSpec` assumes no user cleanup ever ends (only +deadlines fire) and checks that callers are answered; `Spec` adds cleanup +progress for shutdown completion and recovery. The design's +[model section](../../docs/design/dynamic-supervisor.md#what-the-model-establishes) +lists the 23 invariants, the properties and the 40 negative controls (34 +safety, 6 liveness), run one at a time from `negative_checks.py`; its +[test plan](../../docs/design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) +lists the trace events the implementation must emit for trace validation. + +Measured 2026-11-11 (TLC 2.19, fede, 4 workers, `nice -n 19`). Bounded +checks; no cutoff argument is claimed. + +| Configuration | Distinct states | Depth | Time | +|---|---|---|---| +| `MCDynamicSupervisor` (per PR, safety) | 9,237,075 | 30 | 2m13s | +| `MCDynamicSupervisorBackoff` (per PR, safety) | 10,599,610 | 36 | 2m43s | +| `MCDynamicSupervisorZero` (per PR, safety) | 1,170,745 | 21 | 26 s | +| `MCDynamicSupervisorThree` (per PR, safety, three slots) | 442,239 | 30 | 11 s | +| `MCDynamicSupervisorCallers` (per PR, liveness under `CallerSpec`) | 3,413,878 | 29 | 8m48s | +| `MCDynamicSupervisorLive` (per PR, liveness under `Spec`) | 531,760 | 34 | 55 s | +| `MCDynamicSupervisorDeep` (weekly, safety) | 10,060,062 | 45 | 2m24s | + +Not yet: Apalache inductive invariants, and trace validation, which comes with the implementation. From 730dfc47435ba1831e0b60fa43fa3f9835fc1a65 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:27:24 -0300 Subject: [PATCH 32/52] ci: separate TLA legs for the DynamicSupervisor configurations --- .github/workflows/verification.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index e64516c8..14a066c5 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -36,8 +36,12 @@ jobs: select: --only 'MCLifecycle*' - leg: deferred select: --only 'MCDeferred*' + - leg: dynamic-safety + select: --only 'MCDynamicSupervisor,MCDynamicSupervisorBackoff,MCDynamicSupervisorZero,MCDynamicSupervisorThree' + - leg: dynamic-liveness + select: --only 'MCDynamicSupervisorCallers,MCDynamicSupervisorLive' - leg: other - select: --exclude 'MCSupervision,MCLifecycle*,MCDeferred*' + select: --exclude 'MCSupervision,MCLifecycle*,MCDeferred*,MCDynamicSupervisor*' steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v16 From 2400e41ec5f38c6b1595e3506a90f040a68e1335 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:28:50 -0300 Subject: [PATCH 33/52] verify: terminate_slot expect fence (Stale), control IgnoreExpect --- docs/design/dynamic-supervisor.md | 4 ++-- verification/tla/DynamicSupervisor.tla | 30 +++++++++++++++--------- verification/tla/MCDynamicSupervisor.tla | 9 +++---- verification/tla/negative_checks.py | 1 + 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 8ac42946..067d9bee 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -921,12 +921,12 @@ one; no cutoff argument is claimed (Codex F10). | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (per PR; safety, action properties; `Spec`) | 2 slots, every pair of restart types, two terminate calls (id, slot), `max_children` 1, budget 1, one window release, no backoff, exits `Normal`/`Panic`, start failures factory/`Normal`/timeout, ignore, cleanup failures | 9,237,075 | 30 | 2m13s | +| `MCDynamicSupervisor` (per PR; safety, action properties; `Spec`) | 2 slots, every pair of restart types, two terminate calls (by id, by slot fenced with `expect`), `max_children` 1, budget 1, one window release, no backoff, exits `Normal`/`Panic`, start failures factory/`Normal`/timeout, ignore, cleanup failures | 9,237,075 | 30 | 2m13s | | `MCDynamicSupervisorBackoff` (per PR; safety) | as above with backoff | 10,599,610 | 36 | 2m43s | | `MCDynamicSupervisorZero` (per PR; safety) | as the first with budget 0 | 1,170,745 | 21 | 26 s | | `MCDynamicSupervisorThree` (per PR; safety) | 3 Permanent crashing children, `max_children` 2, shared budget 1, FIFO with three arrivals | 442,239 | 30 | 11 s | | `MCDynamicSupervisorCallers` (per PR; `StartAnswered`, `TerminateAnswered`, `FanoutCompletes` under `CallerSpec`) | 2 slots, `max_children` 2, one call, fan-out as separate steps | 3,413,878 | 29 | 8m48s | -| `MCDynamicSupervisorLive` (per PR; `ClosedLeadsToDone`, `RecoveryProgress` under `Spec`) | 2 slots, one call, backoff | 531,760 | 34 | 55 s | +| `MCDynamicSupervisorLive` (per PR; `ClosedLeadsToDone`, `RecoveryProgress` under `Spec`) | 2 slots, one call by slot, backoff | 531,760 | 34 | 55 s | | `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two releases, `max_children` 2, backoff, one call | 10,060,062 | 45 | 2m24s | Negative controls (`negative_checks.py`, run one at a time; each checks diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 2b834c7d..f75eb88c 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -46,7 +46,8 @@ CONSTANTS CleanupFails, \* {FALSE} or {FALSE, TRUE}: cleanup may record a failure TermCalls, \* terminate_child calls CallSlot, \* CallSlot[c]: slot the call is about - CallKind, \* CallKind[c] \in {"id", "slot"} + CallKind, \* CallKind[c] \in {"id", "slot", "expect"}: by actor id, by + \* slot, or by slot fenced on the expected id@generation BackoffOn, \* backoff configured (opt-in) FanoutSteps, \* shutdown fan-out as separate steps (refinement) Fault @@ -66,13 +67,13 @@ Faults == {"None", \* revision 3 "LaterOp", "MisclassifyStuck", "EarlyRemove", "OverReserve", "LostRestart", "TermLost", "NoDeadline", "SkipSignal", "SlotLookup", "QueuedTermOk", - "PublishAfterExit", "StuckResultLost"} + "PublishAfterExit", "StuckResultLost", "IgnoreExpect"} ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat /\ MaxRestarts \in Nat /\ MaxExpire \in Nat ASSUME ModeSet \subseteq Modes /\ ModeSet # {} ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons ASSUME CleanupFails \subseteq BOOLEAN /\ FALSE \in CleanupFails -ASSUME \A c \in TermCalls : CallSlot[c] \in Slots /\ CallKind[c] \in {"id", "slot"} +ASSUME \A c \in TermCalls : CallSlot[c] \in Slots /\ CallKind[c] \in {"id", "slot", "expect"} ASSUME BackoffOn \in BOOLEAN /\ FanoutSteps \in BOOLEAN /\ Fault \in Faults States == {"idle", "queued", "starting", "running", "cleanup", "decide", @@ -117,7 +118,7 @@ TypeOK == /\ op.k \in {"idle", "start", "restart", "term"} /\ \A s \in Slots : sl[s].cs \in States /\ sl[s].alive \subseteq Nat /\ \A c \in TermCalls : - tc[c].st \in {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped"} + tc[c].st \in {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped", "stale"} Init == /\ open = TRUE /\ failed = FALSE /\ exhausted = FALSE /\ sphase = "running" @@ -231,9 +232,16 @@ LoopTerm(c) == /\ tc[c].st = "queued" /\ op.k = "idle" /\ open /\ IsHead("term", CallSlot[c], c) /\ q' = Tail(q) /\ served' = q[1].t /\ LET s == CallSlot[c] - found == /\ sl[s].cs \in Active \ {"stuck"} - /\ CallKind[c] = "slot" \/ sl[s].lastPub = tc[c].g \/ Fault = "SlotLookup" - IN IF sl[s].cs = "stuck" /\ (CallKind[c] = "slot" \/ sl[s].lastPub = tc[c].g) + named == CallKind[c] = "slot" \/ sl[s].lastPub = tc[c].g + live == sl[s].cs \in Active \ {"stuck"} + \* expect: the slot exists but its current published child moved on + stale == CallKind[c] = "expect" /\ sl[s].cs \in Active + /\ sl[s].lastPub # tc[c].g /\ Fault # "IgnoreExpect" + found == live /\ (named \/ Fault = "SlotLookup" + \/ (CallKind[c] = "expect" /\ Fault = "IgnoreExpect")) + IN IF stale + THEN /\ tc' = [tc EXCEPT ![c].st = "stale"] /\ UNCHANGED <> + ELSE IF sl[s].cs = "stuck" /\ named THEN /\ tc' = [tc EXCEPT ![c].st = "stuck"] /\ UNCHANGED <> ELSE IF ~found THEN /\ tc' = [tc EXCEPT ![c].st = "notfound"] /\ UNCHANGED <> @@ -631,15 +639,15 @@ ConcurrentShutdown == OkMeansReady == \A s \in Slots : sl[s].result = "ok" => sl[s].everReady \* terminate results: Ok only after removal; Stuck stays Stuck; a call -\* still queued at shutdown is Stopped, never Ok; an id call never reaches a -\* generation published after the call was made. +\* still queued at shutdown is Stopped, never Ok; an id call or a fenced +\* (expect) slot call never reaches a generation other than the one named. TermResultsSound == \A c \in TermCalls : LET s == CallSlot[c] IN /\ tc[c].st = "ok" => sl[s].cs = "removed" /\ sl[s].alive = {} /\ sl[s].stuck = "none" /\ tc[c].st = "pending" => op = [k |-> "term", s |-> s, c |-> c] - /\ tc[c].st = "pending" /\ CallKind[c] = "id" => sl[s].lastPub = tc[c].g + /\ tc[c].st = "pending" /\ CallKind[c] \in {"id", "expect"} => sl[s].lastPub = tc[c].g Safety == /\ TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ NoChildOutlivesSupervisor @@ -654,7 +662,7 @@ NeverReopens == [][open' => open]_vars \* A committed terminate result never changes. TermResultStable == [][\A c \in TermCalls : - tc[c].st \in {"ok", "stuck", "notfound", "stopped"} => tc'[c].st = tc[c].st]_vars + tc[c].st \in {"ok", "stuck", "notfound", "stopped", "stale"} => tc'[c].st = tc[c].st]_vars (* Liveness *) \* Under CallerSpec (cleanup may never end): diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla index ca1407d0..47b23eca 100644 --- a/verification/tla/MCDynamicSupervisor.tla +++ b/verification/tla/MCDynamicSupervisor.tla @@ -2,14 +2,15 @@ (* Model-checking instances for DynamicSupervisor (rank 12a design). *) EXTENDS DynamicSupervisor -\* Two terminate calls on slot 1: one by the id the caller saw, one by slot. +\* Two terminate calls on slot 1: by the id the caller saw, and by slot +\* fenced on the expected id@generation (introspection's console). MCCalls2 == {1, 2} MCCallSlot2 == [c \in MCCalls2 |-> 1] -MCCallKind2 == [c \in MCCalls2 |-> IF c = 1 THEN "id" ELSE "slot"] -\* One terminate call by id. +MCCallKind2 == [c \in MCCalls2 |-> IF c = 1 THEN "id" ELSE "expect"] +\* One terminate call by slot (unfenced). MCCalls1 == {1} MCCallSlot1 == [c \in MCCalls1 |-> 1] -MCCallKind1 == [c \in MCCalls1 |-> "id"] +MCCallKind1 == [c \in MCCalls1 |-> "slot"] \* No terminate calls. MCCalls0 == {} MCCallSlot0 == [c \in MCCalls0 |-> 1] diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 8efde8dd..b55103b5 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -225,6 +225,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "SlotLookup", "TermResultsSound"), ("MCDynamicSupervisor", "QueuedTermOk", "TermResultsSound"), ("MCDynamicSupervisor", "StuckResultLost", "TermResultsSound"), + ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), ] DYNAMIC_TEMPORAL_CONTROLS = [ ("MCDynamicSupervisorCallers", "NoAbort", "StartAnswered"), From 2cdff7d74db9cf56b4d51ab42e7704cd5d214b15 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:32:25 -0300 Subject: [PATCH 34/52] docs: terminate_slot expect fence (Stale), shared introspection types --- CHANGELOG.md | 3 ++- docs/TESTING.md | 2 +- docs/design/dynamic-supervisor.md | 34 +++++++++++++++++++++++-------- verification/tla/README.md | 4 ++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd50bfbe..298d08e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,8 @@ second reviews and the round-two decisions: one FIFO operation queue, terminate lookup when the loop handles the call, stuck classification by exit reason, caller liveness without assuming cleanup ends, `ignore` and - start data, a trace-validation test plan, and 40 negative controls. + start data, a trace-validation test plan, a fenced `terminate_slot(.., expect)` for introspection, and 41 negative + controls. ## 0.7.3 — 2026-11-10 diff --git a/docs/TESTING.md b/docs/TESTING.md index 68ce6056..425f802c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -931,7 +931,7 @@ the budget reservation protocol, ignore, stuck children and shutdown. TLC checks 23 safety invariants, two action properties and five liveness properties (callers answered even if cleanup never ends; shutdown and recovery complete when it does). Six configurations run on every PR and one -weekly; 40 named negative controls each fail their property. The design's +weekly; 41 named negative controls each fail their property. The design's [test plan](design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) specifies the trace events and trace-validation tests the implementation must ship. diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 067d9bee..ceebe675 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -151,6 +151,7 @@ Round two (revision 3): | G3 | `ignore` | Accepted from the factory: nothing is stored, `start_child` returns `Ignored` (OTP `{ok, undefined}`) | | G4 | Start data | `Started` carries the child's start data (unified start's `Init`), as Gleam's `Started(data)` | | G5 | Introspection | `ChildInfo` and `ChildCounts` are the types introspection v1 exposes for static supervisors too; names agreed across both designs | +| I1 | Fenced termination (introspection #256 rev. 3) | `terminate_slot` takes an optional `expect: ActorId@generation`; a mismatch returns `Stale { current }` and does nothing, checked at the supervisor operation itself, so the console's `terminate --expect` cannot kill a child that moved on | ## API @@ -241,8 +242,11 @@ impl DynamicSupervisor { /// handles the call; it names the slot only if it is the slot's last /// published id at that moment. pub async fn terminate_child(&self, id: ActorId) -> Result<(), TerminateChildError>; - /// The same by slot (any generation). - pub async fn terminate_slot(&self, id: DynamicChildId) -> Result<(), TerminateChildError>; + /// The same by slot. With `expect`, only if the slot's current published + /// child is still that `ActorId@generation`; otherwise `Stale` and + /// nothing is done (introspection's `terminate --expect`, #256). + pub async fn terminate_slot(&self, id: DynamicChildId, expect: Option) + -> Result<(), TerminateChildError>; pub fn which_children(&self) -> Vec; pub fn count_children(&self) -> ChildCounts; pub fn stop(&self); @@ -271,7 +275,7 @@ impl DynamicChild { pub fn current(&self) -> Result, ChildUnavailable>; pub async fn wait_for_generation(&self, after: u64) -> Result, ChildUnavailable>; pub fn snapshot(&self) -> ChildSnapshot; - pub async fn terminate(&self) -> Result<(), TerminateChildError>; // = terminate_slot + pub async fn terminate(&self) -> Result<(), TerminateChildError>; // = terminate_slot(id, None) } /// Nested form, as `SupervisedSupervisor`. @@ -311,9 +315,13 @@ pub enum RejectReason { Placement(SystemError), // the group is closed or out of native capacity } +pub struct ExpectedChild { pub actor: ActorId, pub generation: u64 } + #[non_exhaustive] pub enum TerminateChildError { NotFound, // no current child with that id or slot (Elixir {:error, :not_found}) + /// `expect` did not match: the slot's current published child moved on. + Stale { current: Option }, Stopped, // shutdown began before the loop handled the call CleanupStuck, // the child missed its deadlines; see "Stuck children" } @@ -561,6 +569,7 @@ enqueued by someone else before the self-send can run first. | `start_child` vs a sibling's removal | the slot frees only after the removed child's cleanup and slot finalization | | `terminate_slot` vs a crash of that child | the loop takes them in order. Exit first: a pending restart is cancelled and refunded, or the restart attempt (which holds the loop) finishes and the call then terminates the new generation. Terminate first: the crash is consumed by the termination, no restart. Either way the child is removed and never restarted (`TerminatedNeverRestarted`, `NoPublishAfterStop`) | | `terminate_child(id)` vs a crash of that child | the id is looked up when the loop handles the call. If the slot's last published id is still `id` (cleanup, backoff, a failed replacement), the child is terminated as above. If a replacement committed readiness before the call reached the head, `id` is stale: `NotFound`, and the replacement keeps running (Elixir; `TermResultsSound`, control `SlotLookup`) | +| `terminate_slot(id, expect)` vs a restart | the expectation is compared when the loop handles the call, under the same order as everything else: if the slot's last published child is no longer `expect`, `Stale { current }` and nothing is done; a child in cleanup or backoff whose last published id still matches is terminated (`TermResultsSound`, control `IgnoreExpect`) | | `terminate_child` vs shutdown | a call still queued when shutdown begins gets `Stopped`; a call in progress resolves when the child is removed (`Ok`) or at its deadline (`CleanupStuck`) (control `QueuedTermOk`) | | two calls for one child | the second is handled after the first completed: `NotFound` if the child was removed, `CleanupStuck` if it is stuck (Elixir; no shared waiters) | | a terminate whose child gets stuck | `CleanupStuck` is committed at the deadline and never changes, even if the work ends later (`TermResultsSound`, `TermResultStable`, control `StuckResultLost`) | @@ -862,8 +871,9 @@ that generation died, not that the slot is gone. [`DynamicSupervisor.tla`](../../verification/tla/DynamicSupervisor.tla) (revision 3) models one dynamic supervisor, `start_child` calls made at any -moment, and `terminate_child` calls, each aimed at a slot either by the id -the caller saw or by slot. The control loop takes starts, terminate calls, +moment, and `terminate_child` calls, each aimed at a slot by the id the caller +saw, by slot, or by slot fenced with `expect` (the id@generation the caller +saw). The control loop takes starts, terminate calls, exit notices and backoff retries from one FIFO queue; the exit of the child an operation waits for is consumed by that operation. Each slot's restart type is chosen in `Init`. Budget reservations, stamping, refunds and window @@ -908,7 +918,7 @@ stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | `StuckIsolated` | the supervisor fails only by exhaustion, a stuck child the table would restart (checked against the table), or while closing | | `ConcurrentShutdown` | once waiting, every running child has its stop signal | | `OkMeansReady` | `Ok` only for a child that became ready | -| `TermResultsSound` | `Ok` only after removal and never for a stuck child; a pending call holds the loop; an id call reaches only the generation it named | +| `TermResultsSound` | `Ok` only after removal and never for a stuck child; a pending call holds the loop; an id call or a fenced slot call reaches only the generation it named | Action properties: `NeverReopens`, `TermResultStable` (a committed terminate result never changes). Liveness: `StartAnswered`, `TerminateAnswered`, @@ -969,6 +979,7 @@ property above has at least one; the two action properties do not. | `SlotLookup` | an id call reaches a newer generation | `TermResultsSound` | main | | `QueuedTermOk` | a call queued at shutdown waits and may return `Ok` | `TermResultsSound` | main | | `StuckResultLost` | `CleanupStuck` not committed; later turns into `Ok` | `TermResultsSound` | main | +| `IgnoreExpect` | a fenced call terminates a child that moved on | `TermResultsSound` | main | | `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` | callers | | `NoDeadline` | no cleanup deadline | `StartAnswered` | callers | | `TermLost` | a found call never signals the child | `TerminateAnswered` | callers | @@ -1016,7 +1027,7 @@ the generation, and where noted the queue ticket or call id. | `DRetry` (slot, gen, ticket) | the loop took a retry | `BackoffDone` | | `DCancel` (slot) | a pending restart was cancelled and refunded | `CancelBackoff` | | `DTermCall` (call, slot, named gen, ticket) | `terminate_child`/`terminate_slot` called | `TermCall` | -| `DTermLookup` (call, outcome: found / notfound / stuck) | the loop handled the call | `LoopTerm` | +| `DTermLookup` (call, outcome: found / notfound / stuck / stale) | the loop handled the call | `LoopTerm` | | `DTermReject` (call) | shutdown answered a queued call | `RejectQueuedTerm` | | `DTermDone` (call, result) | the call resolved `Ok` | `TermResolve` | | `DClose`, `DSignal` (slot), `DDone` | shutdown began, a child was signalled, all children finished | `Close`, `SupSignal`/`SupSignalStep`, `SupDone` | @@ -1044,7 +1055,7 @@ recorded in the `tla-traces` CI job, plus generated scripts in and on shutdown during backoff. 5. Terminate: by id, by slot, a stale id after a successful replacement, a second call, a call during a restart attempt, a call queued at - shutdown, a stuck child. + shutdown, a stuck child, a fenced call whose child moved on (`Stale`). 6. Shutdown with several children, one stuck; a start racing shutdown; a commit racing shutdown. 7. Seeded random scripts (as `supervision_trace`) for coverage. @@ -1181,3 +1192,10 @@ Also from the OTP/Elixir/Gleam comparison: decisions G1-G5 (continuation, types), and the standing rule that models are related to the code: the [test plan](#test-plan-relating-the-model-to-the-code) lists the events, their action mapping, the required tests and the expected coverage. + +**Introspection dependency (#256 revision 3).** The console's +`terminate --expect id@generation` is fenced at the supervisor operation: +`terminate_slot` takes `expect` and answers `Stale { current }` without +acting when the slot's published child moved on. Modelled as a third call +kind (`expect`) in `LoopTerm`, with control `IgnoreExpect`. `ChildInfo`, +`ChildCounts` and `ChildKey` are shared with introspection v1. diff --git a/verification/tla/README.md b/verification/tla/README.md index df20109f..0251e67b 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor*.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): OTP's serialised control loop with one FIFO queue; `start_child` and `terminate_child` racing crashes, restarts, `max_children`, the budget reservation protocol, ignore, stuck children and shutdown. 40 named negative controls. Per PR: `MCDynamicSupervisor`, `Backoff`, `Zero`, `Three`, `Callers`, `Live`; weekly: `Deep`. | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor*.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): OTP's serialised control loop with one FIFO queue; `start_child` and `terminate_child` racing crashes, restarts, `max_children`, the budget reservation protocol, ignore, stuck children and shutdown. 41 named negative controls. Per PR: `MCDynamicSupervisor`, `Backoff`, `Zero`, `Three`, `Callers`, `Live`; weekly: `Deep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -595,7 +595,7 @@ or get stuck. Event invariants read history variables computed by deadlines fire) and checks that callers are answered; `Spec` adds cleanup progress for shutdown completion and recovery. The design's [model section](../../docs/design/dynamic-supervisor.md#what-the-model-establishes) -lists the 23 invariants, the properties and the 40 negative controls (34 +lists the 23 invariants, the properties and the 41 negative controls (35 safety, 6 liveness), run one at a time from `negative_checks.py`; its [test plan](../../docs/design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) lists the trace events the implementation must emit for trace validation. From fb145e09c69d23d6ec6fd5504218e9a1a35ab52d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:46:17 -0300 Subject: [PATCH 35/52] docs: TLC results on the final revision-3 model --- docs/design/dynamic-supervisor.md | 14 +++++++------- verification/tla/README.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index ceebe675..95d2ba52 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -931,13 +931,13 @@ one; no cutoff argument is claimed (Codex F10). | Configuration | Instance | Distinct states | Depth | Time | |---|---|---|---|---| -| `MCDynamicSupervisor` (per PR; safety, action properties; `Spec`) | 2 slots, every pair of restart types, two terminate calls (by id, by slot fenced with `expect`), `max_children` 1, budget 1, one window release, no backoff, exits `Normal`/`Panic`, start failures factory/`Normal`/timeout, ignore, cleanup failures | 9,237,075 | 30 | 2m13s | -| `MCDynamicSupervisorBackoff` (per PR; safety) | as above with backoff | 10,599,610 | 36 | 2m43s | -| `MCDynamicSupervisorZero` (per PR; safety) | as the first with budget 0 | 1,170,745 | 21 | 26 s | -| `MCDynamicSupervisorThree` (per PR; safety) | 3 Permanent crashing children, `max_children` 2, shared budget 1, FIFO with three arrivals | 442,239 | 30 | 11 s | -| `MCDynamicSupervisorCallers` (per PR; `StartAnswered`, `TerminateAnswered`, `FanoutCompletes` under `CallerSpec`) | 2 slots, `max_children` 2, one call, fan-out as separate steps | 3,413,878 | 29 | 8m48s | -| `MCDynamicSupervisorLive` (per PR; `ClosedLeadsToDone`, `RecoveryProgress` under `Spec`) | 2 slots, one call by slot, backoff | 531,760 | 34 | 55 s | -| `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two releases, `max_children` 2, backoff, one call | 10,060,062 | 45 | 2m24s | +| `MCDynamicSupervisor` (per PR; safety, action properties; `Spec`) | 2 slots, every pair of restart types, two terminate calls (by id, by slot fenced with `expect`), `max_children` 1, budget 1, one window release, no backoff, exits `Normal`/`Panic`, start failures factory/`Normal`/timeout, ignore, cleanup failures | 8,488,736 | 30 | 1m26s | +| `MCDynamicSupervisorBackoff` (per PR; safety) | as above with backoff | 9,878,333 | 36 | 1m48s | +| `MCDynamicSupervisorZero` (per PR; safety) | as the first with budget 0 | 1,170,745 | 21 | 12 s | +| `MCDynamicSupervisorThree` (per PR; safety) | 3 Permanent crashing children, `max_children` 2, shared budget 1, FIFO with three arrivals | 442,239 | 30 | 7 s | +| `MCDynamicSupervisorCallers` (per PR; `StartAnswered`, `TerminateAnswered`, `FanoutCompletes` under `CallerSpec`) | 2 slots, `max_children` 2, one call, fan-out as separate steps | 3,380,536 | 29 | 6m33s | +| `MCDynamicSupervisorLive` (per PR; `ClosedLeadsToDone`, `RecoveryProgress` under `Spec`) | 2 slots, one call by slot, backoff | 581,563 | 34 | 55 s | +| `MCDynamicSupervisorDeep` (weekly; safety) | budget 2, two releases, `max_children` 2, backoff, one call | 9,992,213 | 45 | 1m35s | Negative controls (`negative_checks.py`, run one at a time; each checks `TypeOK` and only the named property). Every invariant and liveness diff --git a/verification/tla/README.md b/verification/tla/README.md index 0251e67b..2b44f8b0 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -605,13 +605,13 @@ checks; no cutoff argument is claimed. | Configuration | Distinct states | Depth | Time | |---|---|---|---| -| `MCDynamicSupervisor` (per PR, safety) | 9,237,075 | 30 | 2m13s | -| `MCDynamicSupervisorBackoff` (per PR, safety) | 10,599,610 | 36 | 2m43s | -| `MCDynamicSupervisorZero` (per PR, safety) | 1,170,745 | 21 | 26 s | -| `MCDynamicSupervisorThree` (per PR, safety, three slots) | 442,239 | 30 | 11 s | -| `MCDynamicSupervisorCallers` (per PR, liveness under `CallerSpec`) | 3,413,878 | 29 | 8m48s | -| `MCDynamicSupervisorLive` (per PR, liveness under `Spec`) | 531,760 | 34 | 55 s | -| `MCDynamicSupervisorDeep` (weekly, safety) | 10,060,062 | 45 | 2m24s | +| `MCDynamicSupervisor` (per PR, safety) | 8,488,736 | 30 | 1m26s | +| `MCDynamicSupervisorBackoff` (per PR, safety) | 9,878,333 | 36 | 1m48s | +| `MCDynamicSupervisorZero` (per PR, safety) | 1,170,745 | 21 | 12 s | +| `MCDynamicSupervisorThree` (per PR, safety, three slots) | 442,239 | 30 | 7 s | +| `MCDynamicSupervisorCallers` (per PR, liveness under `CallerSpec`) | 3,380,536 | 29 | 6m33s | +| `MCDynamicSupervisorLive` (per PR, liveness under `Spec`) | 581,563 | 34 | 55 s | +| `MCDynamicSupervisorDeep` (weekly, safety) | 9,992,213 | 45 | 1m35s | Not yet: Apalache inductive invariants, and trace validation, which comes with the implementation. From 08506b8dffc95b98b0ac49ba71d903a182ab938f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:56:35 -0300 Subject: [PATCH 36/52] verify: NoAbort control without startup deadlines (the deadline masked it) --- docs/design/dynamic-supervisor.md | 2 +- verification/tla/DynamicSupervisor.tla | 8 ++++++-- verification/tla/MCDynamicSupervisor.cfg | 1 + verification/tla/MCDynamicSupervisorBackoff.cfg | 1 + verification/tla/MCDynamicSupervisorCallers.cfg | 1 + verification/tla/MCDynamicSupervisorDeep.cfg | 1 + verification/tla/MCDynamicSupervisorLive.cfg | 1 + verification/tla/MCDynamicSupervisorThree.cfg | 1 + verification/tla/MCDynamicSupervisorZero.cfg | 1 + verification/tla/negative_checks.py | 7 ++++--- 10 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 95d2ba52..700be070 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -980,7 +980,7 @@ property above has at least one; the two action properties do not. | `QueuedTermOk` | a call queued at shutdown waits and may return `Ok` | `TermResultsSound` | main | | `StuckResultLost` | `CleanupStuck` not committed; later turns into `Ok` | `TermResultsSound` | main | | `IgnoreExpect` | a fenced call terminates a child that moved on | `TermResultsSound` | main | -| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` | callers | +| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` | callers, without startup deadlines (with them the deadline answers the start anyway, which masks the mutation) | | `NoDeadline` | no cleanup deadline | `StartAnswered` | callers | | `TermLost` | a found call never signals the child | `TerminateAnswered` | callers | | `Sequential` | the fan-out waits for each signalled child | `FanoutCompletes` | callers | diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index f75eb88c..44cc8aaf 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -50,6 +50,9 @@ CONSTANTS \* slot, or by slot fenced on the expected id@generation BackoffOn, \* backoff configured (opt-in) FanoutSteps, \* shutdown fan-out as separate steps (refinement) + StartDeadlines, \* startup_timeout exists (TRUE in every configuration; + \* FALSE only for the NoAbort control, so that shutdown's + \* cancellation is the only thing that answers a start) Fault Slots == 1..N @@ -74,7 +77,8 @@ ASSUME ModeSet \subseteq Modes /\ ModeSet # {} ASSUME SelfExits \subseteq Reasons /\ StartFails \subseteq Reasons ASSUME CleanupFails \subseteq BOOLEAN /\ FALSE \in CleanupFails ASSUME \A c \in TermCalls : CallSlot[c] \in Slots /\ CallKind[c] \in {"id", "slot", "expect"} -ASSUME BackoffOn \in BOOLEAN /\ FanoutSteps \in BOOLEAN /\ Fault \in Faults +ASSUME BackoffOn \in BOOLEAN /\ FanoutSteps \in BOOLEAN /\ StartDeadlines \in BOOLEAN +ASSUME Fault \in Faults States == {"idle", "queued", "starting", "running", "cleanup", "decide", "backoff", "restarting", "stuck", "removed", "rejected", "ignored"} @@ -321,7 +325,7 @@ StartFail(s, r) == \* startup_timeout fired (a deadline: fair in both fairness sets). StartTimeout(s) == - /\ sl[s].cs \in {"starting", "restarting"} + /\ StartDeadlines /\ sl[s].cs \in {"starting", "restarting"} /\ FailStart(s, "timeout") /\ Unchanged1 AbortStart(s) == diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index 802b0eae..1fe4d6bc 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -17,6 +17,7 @@ CONSTANTS CallSlot <- MCCallSlot2 CallKind <- MCCallKind2 BackoffOn = FALSE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorBackoff.cfg b/verification/tla/MCDynamicSupervisorBackoff.cfg index 45688b4f..a36ac773 100644 --- a/verification/tla/MCDynamicSupervisorBackoff.cfg +++ b/verification/tla/MCDynamicSupervisorBackoff.cfg @@ -14,6 +14,7 @@ CONSTANTS CallSlot <- MCCallSlot2 CallKind <- MCCallKind2 BackoffOn = TRUE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorCallers.cfg b/verification/tla/MCDynamicSupervisorCallers.cfg index bf1846a1..624ba83e 100644 --- a/verification/tla/MCDynamicSupervisorCallers.cfg +++ b/verification/tla/MCDynamicSupervisorCallers.cfg @@ -16,6 +16,7 @@ CONSTANTS CallSlot <- MCCallSlot1 CallKind <- MCCallKind1 BackoffOn = FALSE + StartDeadlines = TRUE FanoutSteps = TRUE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 53747eef..78c802e9 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -16,6 +16,7 @@ CONSTANTS CallSlot <- MCCallSlot1 CallKind <- MCCallKind1 BackoffOn = TRUE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorLive.cfg b/verification/tla/MCDynamicSupervisorLive.cfg index 81589d40..97f16aed 100644 --- a/verification/tla/MCDynamicSupervisorLive.cfg +++ b/verification/tla/MCDynamicSupervisorLive.cfg @@ -14,6 +14,7 @@ CONSTANTS CallSlot <- MCCallSlot1 CallKind <- MCCallKind1 BackoffOn = TRUE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorThree.cfg b/verification/tla/MCDynamicSupervisorThree.cfg index 199e204c..56f3bfa7 100644 --- a/verification/tla/MCDynamicSupervisorThree.cfg +++ b/verification/tla/MCDynamicSupervisorThree.cfg @@ -15,6 +15,7 @@ CONSTANTS CallSlot <- MCCallSlot0 CallKind <- MCCallKind0 BackoffOn = FALSE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg index 6c3a691c..742b5c7e 100644 --- a/verification/tla/MCDynamicSupervisorZero.cfg +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -14,6 +14,7 @@ CONSTANTS CallSlot <- MCCallSlot2 CallKind <- MCCallKind2 BackoffOn = FALSE + StartDeadlines = TRUE FanoutSteps = FALSE Fault = "None" INVARIANT TypeOK diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index b55103b5..14a3225c 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -228,7 +228,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), ] DYNAMIC_TEMPORAL_CONTROLS = [ - ("MCDynamicSupervisorCallers", "NoAbort", "StartAnswered"), + # Without startup deadlines, so the deadline cannot mask the missing abort. + ("MCDynamicSupervisorCallers", "NoAbort", "StartAnswered", {"StartDeadlines": "FALSE"}), ("MCDynamicSupervisorCallers", "NoDeadline", "StartAnswered"), ("MCDynamicSupervisorCallers", "TermLost", "TerminateAnswered"), ("MCDynamicSupervisorCallers", "Sequential", "FanoutCompletes"), @@ -324,8 +325,8 @@ def fail_instead_of_retry(events): named_temporal("MCLifecycleLive", fault, prop, args.tlc, LIFECYCLE_SMALL) for module, fault, invariant, *constants in DYNAMIC_CONTROLS: named_fault(module, fault, invariant, args.tlc, *constants) - for module, fault, prop in DYNAMIC_TEMPORAL_CONTROLS: - named_temporal(module, fault, prop, args.tlc) + for module, fault, prop, *constants in DYNAMIC_TEMPORAL_CONTROLS: + named_temporal(module, fault, prop, args.tlc, *constants) if __name__ == "__main__": From bcb145aed85f979d6ececa699dd7add100860438 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:33:14 -0300 Subject: [PATCH 37/52] docs: align with unified start #259 (StartOutcome>, ChildStartError, Init::continue_with, StartOptions) --- .../dynamic-supervisor-client-mapping.md | 14 ++-- docs/design/dynamic-supervisor.md | 65 ++++++++++++------- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index de4048ab..620a3dc6 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -158,7 +158,7 @@ fails only itself. | ethrex today | With the dynamic supervisor | |---|---| | `.start()`, unmanaged, on the caller's runtime | `start_child(args)` on a `DynamicSupervisor`, placed on the `p2p` group | -| handshake inside `started()` (up to `HANDSHAKE_TIMEOUT`, 10 s) | **required change:** the handshake moves to the post-start continuation of the unified-start design (0.9, OTP's `handle_continue`): it runs right after `started()` returns and before any message, still under `HANDSHAKE_TIMEOUT`, and a failure ends the actor (a Temporary exit, not a start failure). Otherwise one silent peer would hold the supervisor's control loop, and with it every accept and dial, for the whole `startup_timeout`. A self-sent first message is not equivalent: a message enqueued by another actor first would run before it | +| handshake inside `started()` (up to `HANDSHAKE_TIMEOUT`, 10 s) | **required change:** the handshake moves to the post-start continuation of the unified-start design (0.9, OTP's `handle_continue`): it runs right after `started()` returns and before any message, still under `HANDSHAKE_TIMEOUT`, and a failure ends the actor (a Temporary exit, not a start failure). Otherwise one silent peer would hold the supervisor's control loop, and with it every accept and dial, for the whole `startup_timeout`. A self-sent first message is not equivalent: a message enqueued by another actor first would run before it. The continuation precedes messages but cannot prevent termination: `terminate_child` or shutdown can end the connection before or during the handshake, which then ends it without registering in the peer table (nothing to clean up there) | | no owner; shutdown does not stop connections | the supervisors are owned by the system, nested under the p2p supervisor (recommended) or as roots admitted after the peer table; connections stop before the table, all at once, each killed after its `shutdown_timeout` | | `Semaphore(150)` plus `_admission_permit` in actor state | `max_children = Some(150)` on an inbound supervisor; `Rejected { MaxChildren, args }` returns the `ConnArgs`, so the accept loop can still send an RLPx Disconnect or just drop the stream, as today | | outbound uncapped | a second supervisor for dials without `max_children` | @@ -197,10 +197,14 @@ match inbound.current()?.start_child(ConnArgs::receiver(context.clone(), peer_ad } // Connection actor (server.rs): the handshake leaves started(). -// Handshake in the post-start continuation (unified start, 0.9): runs -// after started() returns and before any message. -#[continue_start] -async fn handshake(&mut self, ctx: &Context) -> ContinueResult { /* existing code */ } +// Handshake as the post-start continuation (unified start #259, 0.9): +// runs after the readiness commit and before any message. +async fn started(&mut self, _ctx: &Context) -> Init { + Init::continue_with(Handshake) +} +impl Handler for PeerConnectionServer { + async fn handle(&mut self, _: Handshake, ctx: &Context) { /* existing handshake code */ } +} // Peer table (peer_table.rs): registration installs the monitor... async fn handle_new_connected_peer(&mut self, msg: NewConnectedPeer, ctx: &Context) -> bool { diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 700be070..08437acb 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -149,8 +149,9 @@ Round two (revision 3): | G1 | Post-start work | Depends on the post-start continuation (a `handle_continue` equivalent) of the unified-start design (0.9), not on "the first message", which other messages could overtake | | G2 | Restart type for connections | The ethrex mapping and the example use `temporary()` (a dropped peer must not be redialled by the supervisor); the default stays Permanent (OTP) | | G3 | `ignore` | Accepted from the factory: nothing is stored, `start_child` returns `Ignored` (OTP `{ok, undefined}`) | -| G4 | Start data | `Started` carries the child's start data (unified start's `Init`), as Gleam's `Started(data)` | +| G4 | Start data | Deferred, as in unified start (#259): `gen_server` returns only `{ok, Pid}`; `Started` is `{ actor, child }` | | G5 | Introspection | `ChildInfo` and `ChildCounts` are the types introspection v1 exposes for static supervisors too; names agreed across both designs | +| U1 | Shared shapes with unified start (#259 rev. 2, section 18) | `StartOutcome>` with `Ignored` as an `Ok` outcome; `FailedToStart { cause: ChildStartError, cleanup_stuck }`; `ChildStart`; `StartOptions` through `DynamicChildSpec::options`; the continuation is `Init::continue_with(msg)`, which precedes messages but is no termination barrier | | I1 | Fenced termination (introspection #256 rev. 3) | `terminate_slot` takes an optional `expect: ActorId@generation`; a mismatch returns `Stale { current }` and does nothing, checked at the supervisor operation itself, so the console's `terminate --expect` cannot kill a child that moved on | ## API @@ -181,8 +182,15 @@ impl DynamicChildSpec { } impl DynamicChildSpec { - pub fn startup_timeout(self, limit: Duration) -> Self; // default 5 s + /// Covers the factory and `started()` together (#259); default 5 s; + /// `Timeout::Infinity` allowed. + pub fn startup_timeout(self, limit: impl Into) -> Self; pub fn shutdown_timeout(self, limit: Duration) -> Self; // default 5 s + /// Per-child start options shared with unified start (#259 section 2.4): + /// name, label, stack size, per-instance bounds. The template's group + /// stays the positional argument of `new`/`temporary`; a `backend` here + /// is rejected (`InvalidOptions`), as for a `ChildSpec`. + pub fn options(self, options: StartOptions) -> Self; } ``` @@ -237,7 +245,7 @@ impl DynamicSupervisor { /// OTP start_child(Sup, ExtraArgs). Queued on the control loop; resolves /// when the first generation is ready, was ignored, or failed. pub async fn start_child(&self, args: Args) - -> Result, StartChildError>; + -> Result>, StartChildError>; /// OTP terminate_child(Sup, Pid): the actor id is looked up when the loop /// handles the call; it names the slot only if it is the slot's last /// published id at that moment. @@ -255,17 +263,17 @@ impl DynamicSupervisor { pub fn failure(&self) -> Option; } -pub enum StartOutcome { - Started(Started), - /// The factory returned ignore: nothing is supervised or counted - /// (OTP {ok, undefined}). +/// Shared with unified start (#259 section 18): `T` is `ActorRef` there. +pub enum StartOutcome { + Started(T), + /// The factory returned `ChildStart::Ignore`, or `started()` returned + /// `Init::ignore()`: nothing is supervised or counted (OTP {ok, undefined}). Ignored, } pub struct Started { pub actor: ActorRef, // the first generation, as OTP's Pid pub child: DynamicChild, // the slot, across restarts - pub data: A::StartData, // what the child's start returned (unified start, Init) } /// Like `SupervisedChild` for a dynamic slot. Holds no arguments and no @@ -305,7 +313,9 @@ pub enum StartChildError { Aborted, /// The first generation failed: factory error or panic, panic or stop /// in started(), or startup_timeout. Not retried, not charged. - FailedToStart { cause: String, exit: Option, cleanup_stuck: bool }, + /// `ChildStartError` is #259's: `Factory`, `Start(StartError)` (with + /// `InitFailed`/`InitExited { reason }`), `StartupTimeout { pending }`. + FailedToStart { cause: ChildStartError, cleanup_stuck: bool }, } #[non_exhaustive] @@ -344,15 +354,19 @@ pub struct ChildCounts { ``` - The factory returns `Result, E>` with - `ChildStart::Actor(a) | ChildStart::Ignore`, the unified-start design's - shape. An ignore on a first start returns `Ignored` and stores nothing; on - a restart it removes the child (OTP drops it), without charge beyond the - attempt already stamped. + `ChildStart::Actor(a) | ChildStart::Ignore` (`From`), and `started()` + may return `Init::ignore()`; both are unified start's shapes (#259). An + ignore on a first start returns `Ok(StartOutcome::Ignored)` and stores + nothing; on a restart it removes the child (OTP drops it), without charge + beyond the attempt already stamped. +- Typed start data (`Started::data`) is deferred, as in #259: `gen_server` + returns only `{ok, Pid}`. - `StartChildError` implements `Error` when `Args: Debug`; `into_args()` returns the arguments of a `Rejected`, and `without_args()` maps to `StartChildError<()>` for `?`. -- `FailedToStart` carries the generation's `ExitReason` when an actor - existed, so callers need not match strings. +- `FailedToStart` carries #259's `ChildStartError`, so the generation's + `ExitReason` (`InitExited { reason }`) or cause (`InitFailed`, + `Factory`) is typed, not a string. - `ChildState` gains no variant: a dynamic child is `Starting`, `Running`, `Backoff` (restart pending), `Stopping` (including stuck), then leaves `which_children`, which is a snapshot in no particular order. @@ -510,12 +524,15 @@ Consequences, as in OTP: in terminate_child(id) for T" (decision R1). **Post-start continuation.** Slow initialisation (ethrex's RLPx handshake) -belongs in the unified-start design's continuation, OTP's -`handle_continue`: it runs right after `started()` and before any message, -including messages that other actors enqueue once they learn the -reference. That design targets 0.9 and this one depends on it; "send -yourself a message in `started()`" is not equivalent, because a message -enqueued by someone else before the self-send can run first. +belongs in unified start's continuation (#259 section 8, OTP's +`handle_continue`): `started()` returns `Init::continue_with(msg)` and +`Handler` runs it as the first turn after the readiness commit, before +any mailbox entry. "Send yourself a message in `started()`" is not +equivalent, because a message enqueued by someone else first would run +first. The continuation is not a termination barrier: readiness is already +committed, so `terminate_child`, a kill or shutdown can end the child before +or during it, as fatal signals end an OTP process in `handle_continue`. +This design depends on #259 (0.9). ### `start_child` @@ -1013,7 +1030,7 @@ the generation, and where noted the queue ticket or call id. | `DAdmit` (slot, ticket, outcome: admitted / max) | the loop takes a start | `LoopAdmit` | | `DReject` (slot) | shutdown answers a queued start | `RejectQueued` | | `DReady` (slot, gen) | readiness commits | `Ready` | -| `DIgnore` (slot, gen) | the factory returned ignore | `Ignore` | +| `DIgnore` (slot, gen) | the factory returned `ChildStart::Ignore` or `started()` returned `Init::ignore()` | `Ignore` | | `DStartFail` (slot, gen, reason) | factory error or panic, stop or panic in `started()` | `StartFail` | | `DStartTimeout` (slot, gen) | `startup_timeout` fired | `StartTimeout` | | `DAbort` (slot, gen) | shutdown cancelled a construction | `AbortStart` | @@ -1047,7 +1064,9 @@ recorded in the `tla-traces` CI job, plus generated scripts in 1. Admission order: two starts at the cap; the second rejected with its arguments. 2. First-start failures: factory error, panic in `started()`, startup - timeout, ignore; each answered after cleanup; one with a stuck cleanup. + timeout, `Init::error`, ignore (factory and `Init::ignore()`); each + answered after cleanup with its `ChildStartError`; one with a stuck + cleanup. 3. Restart table per type and reason, with and without backoff, including a failed replacement re-queued and retried, and a recorded cleanup failure restarting a Transient child. From 811dbbd95a2fb10b0d9a84082267a803da5423ea Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:59:32 -0300 Subject: [PATCH 38/52] verify, docs: keep the slot through ignored-attempt cleanup and slot finalization (Codex round 3, 1) --- docs/design/dynamic-supervisor.md | 26 ++++- verification/tla/DynamicSupervisor.tla | 103 ++++++++++++------ verification/tla/MCDynamicSupervisor.cfg | 1 + .../tla/MCDynamicSupervisorBackoff.cfg | 1 + verification/tla/MCDynamicSupervisorDeep.cfg | 1 + verification/tla/MCDynamicSupervisorThree.cfg | 1 + verification/tla/MCDynamicSupervisorZero.cfg | 1 + verification/tla/negative_checks.py | 2 + 8 files changed, 96 insertions(+), 40 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 08437acb..47de23ac 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -353,6 +353,14 @@ pub struct ChildCounts { } ``` +- An ignored attempt is cleaned up like any exit before anything is + released: `Init::ignore()` drops the constructed state and joins its owned + work; a factory ignore has nothing to drop. Then the slot is finalized + (retained arguments dropped) and only then freed and answered + `Ok(StartOutcome::Ignored)`. If that cleanup misses its deadline the start + is still answered `Ignored` at the deadline (the outcome was decided when + the child ignored) and the slot stays counted until the work ends, as for + any stuck retiring child (control `IgnoreFreesEarly`; Codex round 3). - The factory returns `Result, E>` with `ChildStart::Actor(a) | ChildStart::Ignore` (`From`), and `started()` may return `Init::ignore()`; both are unified start's shapes (#259). An @@ -711,8 +719,13 @@ are for the implementation's tests. slot (no restart can follow): it drops the retained arguments and then releases the native-capacity reservation and the `max_children` slot. - Finalization runs as a tracked task on the template's group, never on the - control loop and never under a supervisor lock, bounded by - `shutdown_timeout`. A panicking destructor is recorded as a finalization + control loop's executor and never under a supervisor lock, bounded by + `shutdown_timeout`. The loop awaits it (as it awaits factories) before it + takes the next operation, so a start queued behind a removal sees the + slot freed, not a transient `MaxChildren` (agy round 3). The slot stays + counted, and its native-capacity reservation held, until finalization + ended (model: state `finalizing`, invariant `RemovedAfterFinalize`, + control `EarlyFree`). A panicking destructor is recorded as a finalization failure in the report (no restart: the decision was already made). A destructor that hangs past the deadline makes the slot stuck as a retiring child: it stays counted, it is reported, nothing else is @@ -1030,7 +1043,7 @@ the generation, and where noted the queue ticket or call id. | `DAdmit` (slot, ticket, outcome: admitted / max) | the loop takes a start | `LoopAdmit` | | `DReject` (slot) | shutdown answers a queued start | `RejectQueued` | | `DReady` (slot, gen) | readiness commits | `Ready` | -| `DIgnore` (slot, gen) | the factory returned `ChildStart::Ignore` or `started()` returned `Init::ignore()` | `Ignore` | +| `DIgnore` (slot, gen) | the factory returned `ChildStart::Ignore` or `started()` returned `Init::ignore()` (cleanup follows, as for an exit) | `Ignore` | | `DStartFail` (slot, gen, reason) | factory error or panic, stop or panic in `started()` | `StartFail` | | `DStartTimeout` (slot, gen) | `startup_timeout` fired | `StartTimeout` | | `DAbort` (slot, gen) | shutdown cancelled a construction | `AbortStart` | @@ -1038,11 +1051,12 @@ the generation, and where noted the queue ticket or call id. | `DStopSeen` (slot, gen, reason) | a signalled generation began exiting (Shutdown/Killed) | `StopObserved` | | `DCleaned` (slot, gen, cleanup_failure, ticket) | owned cleanup finished; exit notice queued | `CleanupDone` | | `DStuck` (slot, gen, kind) | a deadline was missed | `CleanupStuck` | -| `DStuckDone` (slot) | a stuck child finally ended | `StuckDone` | -| `DDecide` (slot, ticket?, outcome: removed / backoff / restart / exhausted) | the loop handled an exit | `Decide` | +| `DStuckDone` (slot) | a stuck child or finalization finally ended | `StuckDone` | +| `DFinalized` (slot) | slot finalization ended: retained arguments dropped, capacity and the slot released; a pending start answered | `Finalize` | +| `DDecide` (slot, ticket?, outcome: finalize / backoff / restart / exhausted) | the loop handled an exit; "finalize" starts slot finalization, which `DFinalized` (or `DStuck`) ends | `Decide` | | `DRetryQueued` (slot, ticket) | the backoff timer fired | `BackoffFire` | | `DRetry` (slot, gen, ticket) | the loop took a retry | `BackoffDone` | -| `DCancel` (slot) | a pending restart was cancelled and refunded | `CancelBackoff` | +| `DCancel` (slot) | a pending restart was cancelled and refunded; finalization starts | `CancelBackoff` | | `DTermCall` (call, slot, named gen, ticket) | `terminate_child`/`terminate_slot` called | `TermCall` | | `DTermLookup` (call, outcome: found / notfound / stuck / stale) | the loop handled the call | `LoopTerm` | | `DTermReject` (call) | shutdown answered a queued call | `RejectQueuedTerm` | diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 44cc8aaf..ee97a082 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -70,7 +70,9 @@ Faults == {"None", \* revision 3 "LaterOp", "MisclassifyStuck", "EarlyRemove", "OverReserve", "LostRestart", "TermLost", "NoDeadline", "SkipSignal", "SlotLookup", "QueuedTermOk", - "PublishAfterExit", "StuckResultLost", "IgnoreExpect"} + "PublishAfterExit", "StuckResultLost", "IgnoreExpect", + \* round 3 + "EarlyFree", "IgnoreFreesEarly"} ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat /\ MaxRestarts \in Nat /\ MaxExpire \in Nat ASSUME ModeSet \subseteq Modes /\ ModeSet # {} @@ -81,10 +83,13 @@ ASSUME BackoffOn \in BOOLEAN /\ FanoutSteps \in BOOLEAN /\ StartDeadlines \in BO ASSUME Fault \in Faults States == {"idle", "queued", "starting", "running", "cleanup", "decide", - "backoff", "restarting", "stuck", "removed", "rejected", "ignored"} -Active == {"starting", "running", "cleanup", "decide", "backoff", + "backoff", "restarting", "stuck", "finalizing", "removed", "rejected"} +\* "finalizing": the loop decided to remove the slot; slot finalization +\* (dropping retained arguments) runs and the slot stays counted until it +\* ends (revision 3, Codex round 3). +Active == {"starting", "running", "cleanup", "decide", "backoff", "finalizing", "restarting", "stuck"} -Terminal == {"idle", "removed", "rejected", "ignored"} +Terminal == {"idle", "removed", "rejected"} Idle == [k |-> "idle", s |-> 0, c |-> 0] VARIABLES @@ -97,19 +102,19 @@ VARIABLES mode, sl, tc, \* history (GhostUpdate only) lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, - restarts, outOfOrder + restarts, outOfOrder, freedEarly sup == <> ghost == <> + restarts, outOfOrder, freedEarly>> vars == <> SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, lastPub |-> 0, reason |-> "none", cfail |-> FALSE, readyGen |-> FALSE, everReady |-> FALSE, result |-> "none", cause |-> "none", term |-> FALSE, stopSig |-> FALSE, stuck |-> "none", - retryQ |-> FALSE] + retryQ |-> FALSE, ign |-> FALSE] \* terminate call: st \in none, queued, pending, ok, stuck, notfound, stopped; \* g = the actor (generation) the caller named. CallInit == [st |-> "none", g |-> 0] @@ -119,7 +124,7 @@ TypeOK == /\ sphase \in {"running", "signal", "waiting", "done"} /\ windowed \in Nat /\ expired \in 0..MaxExpire /\ reserved \subseteq Slots /\ stamped \in Nat /\ ticket \in Nat /\ served \in Nat - /\ op.k \in {"idle", "start", "restart", "term"} + /\ op.k \in {"idle", "start", "restart", "term", "final"} /\ \A s \in Slots : sl[s].cs \in States /\ sl[s].alive \subseteq Nat /\ \A c \in TermCalls : tc[c].st \in {"none", "queued", "pending", "ok", "stuck", "notfound", "stopped", "stale"} @@ -133,7 +138,7 @@ Init == /\ lateCommit = [s \in Slots |-> FALSE] /\ latePub = [s \in Slots |-> FALSE] /\ lateGenClose = [s \in Slots |-> FALSE] /\ lateGenTerm = [s \in Slots |-> FALSE] /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] - /\ restarts = 0 /\ outOfOrder = FALSE + /\ restarts = 0 /\ outOfOrder = FALSE /\ freedEarly = [s \in Slots |-> FALSE] ----------------------------------------------------------------------------- (* Helpers *) @@ -300,12 +305,20 @@ Ready(s) == /\ UNCHANGED <> -\* The factory returned ignore: no actor, nothing stored (OTP {ok, undefined}). +\* The factory returned ChildStart::Ignore or started() returned +\* Init::ignore(): nothing will be supervised (OTP {ok, undefined}), but the +\* attempt's state and owned work are cleaned up like any exit, and the slot +\* is finalized, before the answer and before the slot frees. The outcome is +\* decided now (cause "ignored"), answered after cleanup or at its deadline. Ignore(s) == /\ sl[s].cs \in {"starting", "restarting"} - /\ sl' = [sl EXCEPT ![s].cs = "ignored", ![s].alive = {}, - ![s].result = IF sl[s].cs = "starting" THEN "ignored" ELSE sl[s].result] - /\ op' = IF op.s = s /\ op.k # "term" THEN Idle ELSE op + /\ IF Fault = "IgnoreFreesEarly" + THEN sl' = [sl EXCEPT ![s].cs = "removed", ![s].alive = {}, ![s].pub = 0, + ![s].result = IF sl[s].cs = "starting" THEN "ignored" ELSE sl[s].result] + ELSE sl' = [sl EXCEPT ![s].cs = "cleanup", ![s].reason = "Normal", ![s].pub = 0, + ![s].ign = TRUE, + ![s].cause = IF sl[s].cs = "starting" THEN "ignored" ELSE sl[s].cause] + /\ op' = IF Fault = "IgnoreFreesEarly" /\ op.s = s THEN Idle ELSE op /\ UNCHANGED <> @@ -358,7 +371,8 @@ CleanupDone(s, f) == stamped, served, tc>> \* A deadline was missed: signalled but still running after the kill and -\* the second interval, or owned cleanup past its deadline. The slot stays +\* the second interval, owned cleanup past its deadline, or slot +\* finalization past its deadline. The slot stays \* counted. Results waiting on it are committed now and stay (historical). \* Kind: "restart" only for a child the table would restart, judged from \* the exit reason alone (a stall is not a recorded cleanup failure); @@ -366,7 +380,7 @@ CleanupDone(s, f) == CleanupStuck(s) == /\ Fault # "NoDeadline" /\ sl[s].stuck = "none" - /\ \/ sl[s].cs = "cleanup" + /\ \/ sl[s].cs \in {"cleanup", "finalizing"} \/ sl[s].cs = "running" /\ sl[s].stopSig /\ LET restartable == /\ sl[s].everReady /\ ~sl[s].term /\ sl[s].cs = "cleanup" @@ -386,11 +400,25 @@ CleanupStuck(s) == /\ op' = IF op.s = s /\ (op.k # "term" \/ Fault # "StuckResultLost") THEN Idle ELSE op /\ UNCHANGED <> +\* A stuck child (or finalization) finally ended; finalization follows. StuckDone(s) == /\ sl[s].cs = "stuck" - /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].alive = {}] + /\ sl' = [sl EXCEPT ![s].cs = "finalizing", ![s].alive = {}] /\ Unchanged1 +\* Slot finalization ended: retained arguments dropped, then capacity and +\* the max_children slot released. A start's answer is given now. +Finalize(s) == + /\ sl[s].cs = "finalizing" + /\ sl' = [sl EXCEPT ![s].cs = "removed", + ![s].result = IF sl[s].result = "pending" + THEN IF Fault = "LateFailedOverwrite" /\ ~open + THEN "aborted" ELSE sl[s].cause + ELSE sl[s].result] + /\ op' = IF op = [k |-> "final", s |-> s, c |-> 0] THEN Idle ELSE op + /\ UNCHANGED <> + \* The loop handles an exit: at the head of the queue, or directly when the \* operation in progress waits for this child (start or terminate). Decide(s) == @@ -404,16 +432,18 @@ Decide(s) == \/ Fault = "OverlapGeneration" /\ sl[s].cs = "cleanup" /\ LoopIdle /\ UNCHANGED <> /\ LET ownStart == op.s = s /\ op.k = "start" + \* removal: finalize; the loop waits for it unless a terminate holds it + Remove == /\ sl' = [sl EXCEPT ![s].cs = IF Fault = "EarlyFree" THEN "removed" + ELSE "finalizing"] + /\ op' = IF op.k = "term" /\ op.s = s THEN op + ELSE IF Fault = "EarlyFree" THEN Idle + ELSE [k |-> "final", s |-> s, c |-> 0] IN IF ~sl[s].everReady /\ Fault # "RetryFirstStart" - THEN \* first-start failure: answered after cleanup; never retried - /\ sl' = [sl EXCEPT ![s].cs = "removed", - ![s].result = IF Fault = "LateFailedOverwrite" /\ ~open - THEN "aborted" ELSE sl[s].cause] - /\ op' = IF ownStart THEN Idle ELSE op + THEN \* first-start failure or ignore: answered after finalization + /\ Remove /\ UNCHANGED <> - ELSE IF ~open \/ (sl[s].term /\ Fault # "RestartTerminated") - THEN /\ sl' = [sl EXCEPT ![s].cs = "removed"] - /\ op' = IF ownStart THEN Idle ELSE op \* a terminate keeps the loop until TermResolve + ELSE IF ~open \/ (sl[s].term /\ Fault # "RestartTerminated") \/ sl[s].ign + THEN /\ Remove /\ UNCHANGED <> ELSE IF WouldRestart(s) THEN IF Budget < MaxRestarts \/ Fault = "OverReserve" @@ -435,12 +465,10 @@ Decide(s) == /\ op' = [k |-> "restart", s |-> s, c |-> 0] /\ UNCHANGED <> ELSE /\ exhausted' = TRUE - /\ sl' = [sl EXCEPT ![s].cs = "removed"] + /\ Remove /\ IF Fault = "NoEscalate" THEN UNCHANGED <> ELSE Fail - /\ op' = IF ownStart THEN Idle ELSE op /\ UNCHANGED <> - ELSE /\ sl' = [sl EXCEPT ![s].cs = "removed"] - /\ op' = IF ownStart THEN Idle ELSE op + ELSE /\ Remove /\ UNCHANGED <> /\ UNCHANGED <> @@ -476,7 +504,7 @@ CancelBackoff(s) == /\ sl[s].cs = "backoff" /\ \/ ~open /\ Fault # "StuckBackoff" \/ sl[s].term /\ Fault # "RestartTerminated" - /\ sl' = [sl EXCEPT ![s].cs = "removed", ![s].retryQ = FALSE] + /\ sl' = [sl EXCEPT ![s].cs = "finalizing", ![s].retryQ = FALSE] /\ q' = SelectSeq(q, LAMBDA e : ~(e.k = "retry" /\ e.s = s)) /\ reserved' = IF Fault = "NoRefund" THEN reserved ELSE reserved \ {s} /\ UNCHANGED < sl[s].gen Decided(s) == /\ sl[s].cs = "decide" /\ \/ sl'[s].cs \in {"backoff", "restarting"} - \/ exhausted' /\ ~exhausted /\ sl'[s].cs = "removed" + \/ exhausted' /\ ~exhausted /\ sl'[s].cs \in {"finalizing", "removed"} GhostUpdate == /\ lateCommit' = [s \in Slots |-> lateCommit[s] \/ @@ -533,10 +561,12 @@ GhostUpdate == /\ badRestart' = [s \in Slots |-> badRestart[s] \/ (((Grew(s) /\ sl[s].gen > 0) \/ Decided(s)) /\ ~Allowed(s))] /\ badRetire' = [s \in Slots |-> badRetire[s] \/ - (sl[s].cs = "decide" /\ sl'[s].cs = "removed" /\ open' + (sl[s].cs = "decide" /\ sl'[s].cs \in {"finalizing", "removed"} /\ open' /\ ~sl[s].term /\ Allowed(s))] /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) /\ outOfOrder' = (outOfOrder \/ (served' # served /\ served' < served)) + /\ freedEarly' = [s \in Slots |-> freedEarly[s] \/ + (sl'[s].cs = "removed" /\ sl[s].cs \notin {"finalizing", "removed"})] /\ UNCHANGED mode SlotStep(s) == @@ -546,7 +576,7 @@ SlotStep(s) == \/ \E r \in SelfExits : SelfExit(s, r) \/ \E r \in {"Shutdown", "Killed"} : StopObserved(s, r) \/ \E f \in CleanupFails : CleanupDone(s, f) - \/ CleanupStuck(s) \/ StuckDone(s) + \/ CleanupStuck(s) \/ StuckDone(s) \/ Finalize(s) \/ Decide(s) \/ BackoffFire(s) \/ BackoffDone(s) \/ CancelBackoff(s) CallStep(c) == TermCall(c) \/ RejectQueuedTerm(c) \/ LoopTerm(c) \/ TermResolve(c) @@ -575,6 +605,7 @@ Fairness == /\ WF_vars((\E r \in {"Shutdown", "Killed"} : StopObserved(s, r)) /\ GhostUpdate) /\ WF_vars((\E f \in CleanupFails : CleanupDone(s, f)) /\ GhostUpdate) /\ WF_vars((StuckDone(s)) /\ GhostUpdate) + /\ WF_vars((Finalize(s)) /\ GhostUpdate) Spec == Init /\ [][Next]_vars /\ Fairness CallerSpec == Init /\ [][Next]_vars /\ CallerFairness @@ -637,6 +668,10 @@ StuckIsolated == /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, FALSE) \/ ~sl[s].readyGen /\ mode[s] # "Temporary" +\* A slot is freed (removed) only after slot finalization ran: the cap and +\* capacity cover retained arguments and an ignored attempt's cleanup. +RemovedAfterFinalize == \A s \in Slots : ~freedEarly[s] + ConcurrentShutdown == sphase \in {"waiting", "done"} => \A s \in Slots : sl[s].cs = "running" => sl[s].stopSig @@ -660,7 +695,7 @@ Safety == /\ NoStartAfterShutdown /\ RestartFollowsPolicy /\ FirstStartNeverRetried /\ FailedAfterCleanup /\ FailureCauseKept /\ ChargedRestarts /\ ReservationsMatch /\ BudgetBounded /\ ExhaustionCloses /\ StuckIsolated /\ ConcurrentShutdown - /\ OkMeansReady /\ TermResultsSound + /\ OkMeansReady /\ TermResultsSound /\ RemovedAfterFinalize NeverReopens == [][open' => open]_vars \* A committed terminate result never changes. @@ -680,5 +715,5 @@ FanoutCompletes == (~open) ~> (sphase \in {"waiting", "done"}) ClosedLeadsToDone == (~open) ~> (sphase = "done") RecoveryProgress == \A s \in Slots : (sl[s].cs \in {"decide", "backoff", "restarting"} /\ sl[s].everReady) - ~> (sl[s].cs \in {"running", "removed", "stuck", "ignored"}) + ~> (sl[s].cs \in {"running", "finalizing", "removed", "stuck"}) ============================================================================= diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index 1fe4d6bc..e6806297 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -44,5 +44,6 @@ INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorBackoff.cfg b/verification/tla/MCDynamicSupervisorBackoff.cfg index a36ac773..b0a92a15 100644 --- a/verification/tla/MCDynamicSupervisorBackoff.cfg +++ b/verification/tla/MCDynamicSupervisorBackoff.cfg @@ -41,5 +41,6 @@ INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 78c802e9..7a208cdc 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -43,5 +43,6 @@ INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorThree.cfg b/verification/tla/MCDynamicSupervisorThree.cfg index 56f3bfa7..116b41e0 100644 --- a/verification/tla/MCDynamicSupervisorThree.cfg +++ b/verification/tla/MCDynamicSupervisorThree.cfg @@ -42,5 +42,6 @@ INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg index 742b5c7e..ef1a6b98 100644 --- a/verification/tla/MCDynamicSupervisorZero.cfg +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -41,5 +41,6 @@ INVARIANT StuckIsolated INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 3326b8d7..b1b442b2 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -262,6 +262,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "QueuedTermOk", "TermResultsSound"), ("MCDynamicSupervisor", "StuckResultLost", "TermResultsSound"), ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), + ("MCDynamicSupervisor", "EarlyFree", "RemovedAfterFinalize"), + ("MCDynamicSupervisor", "IgnoreFreesEarly", "RemovedAfterFinalize"), ] DYNAMIC_TEMPORAL_CONTROLS = [ # Without startup deadlines, so the deadline cannot mask the missing abort. From 88f3d3a2ba96dbd271a73d0f71633e08528e397e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:00:06 -0300 Subject: [PATCH 39/52] verify: FIFO checked at selection, queued-at-close terminate Stopped, NoAbort on a passing deadline-free config (Codex round 3, 2) --- .github/workflows/verification.yml | 2 +- docs/design/dynamic-supervisor.md | 13 ++++++---- verification/tla/DynamicSupervisor.tla | 24 +++++++++++++++---- verification/tla/MCDynamicSupervisor.cfg | 1 + verification/tla/MCDynamicSupervisorAbort.cfg | 23 ++++++++++++++++++ verification/tla/MCDynamicSupervisorAbort.tla | 3 +++ .../tla/MCDynamicSupervisorBackoff.cfg | 1 + verification/tla/MCDynamicSupervisorDeep.cfg | 1 + verification/tla/MCDynamicSupervisorThree.cfg | 1 + verification/tla/MCDynamicSupervisorZero.cfg | 1 + verification/tla/negative_checks.py | 6 ++--- 11 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorAbort.cfg create mode 100644 verification/tla/MCDynamicSupervisorAbort.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 14a066c5..192369bd 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -39,7 +39,7 @@ jobs: - leg: dynamic-safety select: --only 'MCDynamicSupervisor,MCDynamicSupervisorBackoff,MCDynamicSupervisorZero,MCDynamicSupervisorThree' - leg: dynamic-liveness - select: --only 'MCDynamicSupervisorCallers,MCDynamicSupervisorLive' + select: --only 'MCDynamicSupervisorCallers,MCDynamicSupervisorLive,MCDynamicSupervisorAbort' - leg: other select: --exclude 'MCSupervision,MCLifecycle*,MCDeferred*,MCDynamicSupervisor*' steps: diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 47de23ac..53532aaa 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -931,7 +931,7 @@ stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | `NoChildOutlivesSupervisor` | done means every child finished and was removed | | `MaxChildrenBound` | children held never exceed `max_children` | | `SerialConstruction` | at most one generation is being constructed | -| `FifoOrder` | the loop takes queued operations in arrival order | +| `FifoOrder` | every operation the loop selects is the head of the queue at selection (not merely a non-decreasing ticket: a later operation served first, then the earlier one rejected at close, is caught) | | `TemporaryNeverRestarted` | a Temporary child has one generation | | `TerminatedNeverRestarted` | no generation begins after a found `terminate_child` | | `NoGenerationAfterClose` | no generation begins after admission closed | @@ -948,11 +948,14 @@ stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | `StuckIsolated` | the supervisor fails only by exhaustion, a stuck child the table would restart (checked against the table), or while closing | | `ConcurrentShutdown` | once waiting, every running child has its stop signal | | `OkMeansReady` | `Ok` only for a child that became ready | +| `QueuedTermStopped` | a terminate call still queued when shutdown began is answered `Stopped` and stays so | +| `RemovedAfterFinalize` | a slot is freed only after slot finalization (and an ignored attempt's cleanup) | | `TermResultsSound` | `Ok` only after removal and never for a stuck child; a pending call holds the loop; an id call or a fenced slot call reaches only the generation it named | Action properties: `NeverReopens`, `TermResultStable` (a committed terminate result never changes). Liveness: `StartAnswered`, `TerminateAnswered`, -`FanoutCompletes` (CallerSpec); `ClosedLeadsToDone`, `RecoveryProgress` +`FanoutCompletes` (CallerSpec), `ShutdownAnswersStarts` (CallerSpec without +startup deadlines, `MCDynamicSupervisorAbort`); `ClosedLeadsToDone`, `RecoveryProgress` (Spec). Configurations (TLC 2.19, fede, 4 workers, `nice -n 19`). These are @@ -1007,10 +1010,12 @@ property above has at least one; the two action properties do not. | `OverReserve` | a restart reserved beyond the budget | `BudgetBounded` | main | | `TermReplyEarly` | `terminate_child` returns before removal | `TermResultsSound` | main | | `SlotLookup` | an id call reaches a newer generation | `TermResultsSound` | main | -| `QueuedTermOk` | a call queued at shutdown waits and may return `Ok` | `TermResultsSound` | main | +| `QueuedTermOk` | shutdown answers a queued call `Ok` | `QueuedTermStopped` | main | +| `EarlyFree` | removal frees the slot before finalization | `RemovedAfterFinalize` | main | +| `IgnoreFreesEarly` | an ignored attempt frees the slot at once | `RemovedAfterFinalize` | main | | `StuckResultLost` | `CleanupStuck` not committed; later turns into `Ok` | `TermResultsSound` | main | | `IgnoreExpect` | a fenced call terminates a child that moved on | `TermResultsSound` | main | -| `NoAbort` | a start racing shutdown is not cancelled | `StartAnswered` | callers, without startup deadlines (with them the deadline answers the start anyway, which masks the mutation) | +| `NoAbort` | a start racing shutdown is not cancelled | `ShutdownAnswersStarts` | abort: `CallerSpec` without startup deadlines, which passes unmutated (with deadlines the timeout would answer the start and mask the mutation) | | `NoDeadline` | no cleanup deadline | `StartAnswered` | callers | | `TermLost` | a found call never signals the child | `TerminateAnswered` | callers | | `Sequential` | the fan-out waits for each signalled child | `FanoutCompletes` | callers | diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index ee97a082..d4d8acd1 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -102,12 +102,12 @@ VARIABLES mode, sl, tc, \* history (GhostUpdate only) lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, - restarts, outOfOrder, freedEarly + restarts, outOfOrder, freedEarly, queuedAtClose sup == <> ghost == <> + restarts, outOfOrder, freedEarly, queuedAtClose>> vars == <> SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, lastPub |-> 0, @@ -139,6 +139,7 @@ Init == /\ lateGenClose = [s \in Slots |-> FALSE] /\ lateGenTerm = [s \in Slots |-> FALSE] /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] /\ restarts = 0 /\ outOfOrder = FALSE /\ freedEarly = [s \in Slots |-> FALSE] + /\ queuedAtClose = [c \in TermCalls |-> FALSE] ----------------------------------------------------------------------------- (* Helpers *) @@ -232,7 +233,7 @@ TermCall(c) == RejectQueuedTerm(c) == /\ tc[c].st = "queued" /\ ~open /\ \E i \in 1..Len(q) : q[i].k = "term" /\ q[i].c = c /\ q' = Rm(i) - /\ tc' = [tc EXCEPT ![c].st = IF Fault = "QueuedTermOk" THEN "pending" ELSE "stopped"] + /\ tc' = [tc EXCEPT ![c].st = IF Fault = "QueuedTermOk" THEN "ok" ELSE "stopped"] /\ UNCHANGED <> @@ -564,7 +565,11 @@ GhostUpdate == (sl[s].cs = "decide" /\ sl'[s].cs \in {"finalizing", "removed"} /\ open' /\ ~sl[s].term /\ Allowed(s))] /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) - /\ outOfOrder' = (outOfOrder \/ (served' # served /\ served' < served)) + \* the loop selected an operation: it must be the head at selection + /\ outOfOrder' = (outOfOrder \/ (served' # served /\ (Len(q) = 0 \/ served' # q[1].t))) + \* terminate calls still queued at the moment admission closed + /\ queuedAtClose' = [c \in TermCalls |-> queuedAtClose[c] \/ + (open /\ ~open' /\ tc[c].st = "queued")] /\ freedEarly' = [s \in Slots |-> freedEarly[s] \/ (sl'[s].cs = "removed" /\ sl[s].cs \notin {"finalizing", "removed"})] /\ UNCHANGED mode @@ -672,6 +677,11 @@ StuckIsolated == \* capacity cover retained arguments and an ignored attempt's cleanup. RemovedAfterFinalize == \A s \in Slots : ~freedEarly[s] +\* A terminate call still queued when shutdown began is answered Stopped +\* (and stays so), never Ok or NotFound. +QueuedTermStopped == + \A c \in TermCalls : queuedAtClose[c] => tc[c].st \in {"queued", "stopped"} + ConcurrentShutdown == sphase \in {"waiting", "done"} => \A s \in Slots : sl[s].cs = "running" => sl[s].stopSig @@ -695,7 +705,7 @@ Safety == /\ NoStartAfterShutdown /\ RestartFollowsPolicy /\ FirstStartNeverRetried /\ FailedAfterCleanup /\ FailureCauseKept /\ ChargedRestarts /\ ReservationsMatch /\ BudgetBounded /\ ExhaustionCloses /\ StuckIsolated /\ ConcurrentShutdown - /\ OkMeansReady /\ TermResultsSound /\ RemovedAfterFinalize + /\ OkMeansReady /\ TermResultsSound /\ RemovedAfterFinalize /\ QueuedTermStopped NeverReopens == [][open' => open]_vars \* A committed terminate result never changes. @@ -711,6 +721,10 @@ TerminateAnswered == \A c \in TermCalls : (tc[c].st \in {"queued", "pending"}) ~> (tc[c].st \notin {"queued", "pending"}) \* Every admitted child is signalled even if siblings never finish. FanoutCompletes == (~open) ~> (sphase \in {"waiting", "done"}) +\* Once shutdown began, every pending start is answered (by its abort and +\* cleanup deadline), even without startup deadlines (MCDynamicSupervisorAbort). +ShutdownAnswersStarts == \A s \in Slots : + (~open /\ sl[s].result = "pending") ~> (sl[s].result # "pending") \* Under Spec (cleanup ends): ClosedLeadsToDone == (~open) ~> (sphase = "done") RecoveryProgress == \A s \in Slots : diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index e6806297..d531e95e 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -45,5 +45,6 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorAbort.cfg b/verification/tla/MCDynamicSupervisorAbort.cfg new file mode 100644 index 00000000..a20381e9 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorAbort.cfg @@ -0,0 +1,23 @@ +\* Per PR, liveness under CallerSpec without startup deadlines: once +\* shutdown begins every pending start is answered by its abort (and the +\* cleanup deadline), not by a startup timeout. Passes as is; the NoAbort +\* control must fail here. Small: one cap, no budget, no terminate calls. +SPECIFICATION CallerSpec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 0 + MaxExpire = 0 + SelfExits = {"Normal", "Panic"} + StartFails = {"none"} + CleanupFails = {FALSE} + TermCalls <- MCCalls0 + CallSlot <- MCCallSlot0 + CallKind <- MCCallKind0 + BackoffOn = FALSE + StartDeadlines = FALSE + FanoutSteps = FALSE + Fault = "None" +INVARIANT TypeOK +PROPERTY ShutdownAnswersStarts diff --git a/verification/tla/MCDynamicSupervisorAbort.tla b/verification/tla/MCDynamicSupervisorAbort.tla new file mode 100644 index 00000000..fd8bb66b --- /dev/null +++ b/verification/tla/MCDynamicSupervisorAbort.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorAbort ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/MCDynamicSupervisorBackoff.cfg b/verification/tla/MCDynamicSupervisorBackoff.cfg index b0a92a15..64857d64 100644 --- a/verification/tla/MCDynamicSupervisorBackoff.cfg +++ b/verification/tla/MCDynamicSupervisorBackoff.cfg @@ -42,5 +42,6 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorDeep.cfg b/verification/tla/MCDynamicSupervisorDeep.cfg index 7a208cdc..e7b4db17 100644 --- a/verification/tla/MCDynamicSupervisorDeep.cfg +++ b/verification/tla/MCDynamicSupervisorDeep.cfg @@ -44,5 +44,6 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorThree.cfg b/verification/tla/MCDynamicSupervisorThree.cfg index 116b41e0..e343a08d 100644 --- a/verification/tla/MCDynamicSupervisorThree.cfg +++ b/verification/tla/MCDynamicSupervisorThree.cfg @@ -43,5 +43,6 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorZero.cfg b/verification/tla/MCDynamicSupervisorZero.cfg index ef1a6b98..562510da 100644 --- a/verification/tla/MCDynamicSupervisorZero.cfg +++ b/verification/tla/MCDynamicSupervisorZero.cfg @@ -42,5 +42,6 @@ INVARIANT ConcurrentShutdown INVARIANT OkMeansReady INVARIANT TermResultsSound INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped PROPERTY NeverReopens PROPERTY TermResultStable diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index b1b442b2..93934796 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -259,15 +259,15 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "OverReserve", "BudgetBounded"), ("MCDynamicSupervisor", "TermReplyEarly", "TermResultsSound"), ("MCDynamicSupervisor", "SlotLookup", "TermResultsSound"), - ("MCDynamicSupervisor", "QueuedTermOk", "TermResultsSound"), + ("MCDynamicSupervisor", "QueuedTermOk", "QueuedTermStopped"), ("MCDynamicSupervisor", "StuckResultLost", "TermResultsSound"), ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), ("MCDynamicSupervisor", "EarlyFree", "RemovedAfterFinalize"), ("MCDynamicSupervisor", "IgnoreFreesEarly", "RemovedAfterFinalize"), ] DYNAMIC_TEMPORAL_CONTROLS = [ - # Without startup deadlines, so the deadline cannot mask the missing abort. - ("MCDynamicSupervisorCallers", "NoAbort", "StartAnswered", {"StartDeadlines": "FALSE"}), + # MCDynamicSupervisorAbort has no startup deadlines and passes unmutated. + ("MCDynamicSupervisorAbort", "NoAbort", "ShutdownAnswersStarts"), ("MCDynamicSupervisorCallers", "NoDeadline", "StartAnswered"), ("MCDynamicSupervisorCallers", "TermLost", "TerminateAnswered"), ("MCDynamicSupervisorCallers", "Sequential", "FanoutCompletes"), From 7a0ace39da627db09fbae3e00212814a6cff1c61 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:00:28 -0300 Subject: [PATCH 40/52] docs: mempool reservations held by an owning registration; shutdown state, not Down reason, suppresses retries (Codex round 3, 3) --- .../dynamic-supervisor-client-mapping.md | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/design/dynamic-supervisor-client-mapping.md b/docs/design/dynamic-supervisor-client-mapping.md index 620a3dc6..120c22ea 100644 --- a/docs/design/dynamic-supervisor-client-mapping.md +++ b/docs/design/dynamic-supervisor-client-mapping.md @@ -232,26 +232,38 @@ impl Handler for PeerTableServer { `stopped()` does not run for a non-trapping connection that is terminated, shut down or killed (v0.8, decision 2), and `trap_exit` does not cover a -kill. So each resource the hook releases today gets an owner that holds an -exact registration and releases on the connection's `Down`, as an Erlang -process that owns a resource monitors its user. Nothing is sent from -`Drop`: state destruction runs before publication (exit phases 5 and 6), -so such a message would reach the peer table before the `Down` and could -pick the dying peer as its own alternate (agy 5), and an enqueued message -is not a completed release (Codex F9). - -| Released in `stopped()` today | Registration (identity stored by the owner) | Release on `Down` | +kill. So each resource the hook releases today needs an owner whose +lifetime covers it. Two kinds: + +- **Records an actor keeps in its own state** (peer-table entries, the + broadcaster's index) live and die with that actor. Their owner monitors + the connection and releases on `Down`, guarded by the identity it stored. + If the owner itself stops or restarts first, the records go with it, so + nothing leaks. +- **Persistent reservations outside any actor** (the mempool's in-flight + hashes, cell availability, the announced-hash budget) would outlive a + monitor whose owner stops before handling `Down` (Codex round 3: the peer + table can begin a non-trapping shutdown with the `Down` still queued). + They are held by an **owning registration**: a value the mempool hands + out when the connection reserves, kept in the connection's state, whose + destruction releases exactly that connection's entries synchronously, + under the mempool's lock, with no message and no `.await`. State + destruction runs on every exit path, kills and runtime drops included + (exit phase 5), before the connection's slot is freed, so the release + cannot be lost whoever else is alive. + +Nothing is sent from `Drop`: a message would reach the peer table before the +`Down` (phase 5 before 6) and could pick the dying peer as an alternate +(agy round 2), and an enqueued message is not a completed release. + +| Released in `stopped()` today | Owner and registration | Release | |---|---|---| -| peer-table entry and discv5 session | `new_connected_peer` stores the monitor reference in `PeerData` | the peer table removes the entry only if it still holds that monitor | -| tx-broadcaster index | the broadcaster's `add_peer` records the connection's `ActorId` and monitors it | the broadcaster frees the index only if it is still owned by that `ActorId` | -| in-flight transaction reservations (`clear_in_flight_txs`) | the mempool records the requesting connection's `ActorId` with each in-flight hash when the request is made | the peer table, after removing the entry, calls `mempool.release_in_flight(actor_id)`: synchronous, exact, removes only hashes still held by that id | -| re-request to alternates (`retry_on_alternates`) | the same per-hash records | best effort, done by the peer table after the removal, with a local lookup of live peers (no request to itself, so no self-request deadlock); skipped once shutdown began (`Down` reason `Shutdown` from the supervisor's shutdown) so a node stop does not issue ~150 re-requests | -| cell availability and announced-hash budget (`clear_peer_cell_availability`, `forget_peer_announcements`) | keyed today by node id with a "started session" check; becomes keyed by (node id, connection `ActorId`) | the peer table releases the entries of that `ActorId` only, so a second session of the same node keeps its own | -| disconnect metrics | | the peer table's `Down` handler (it has the final reason) | - -`Drop` keeps only what is local to the actor (dropping its own buffers). -Releases happen after the connection is gone from the table, so no -re-request can target it. +| peer-table entry and discv5 session | peer table; `new_connected_peer` stores the monitor reference in `PeerData` | on `Down`, only if the entry still holds that monitor; gone anyway if the peer table stops | +| tx-broadcaster index | broadcaster; its `add_peer` records the connection's `ActorId` and monitors it | on `Down`, only if the index is still owned by that `ActorId`; gone anyway if the broadcaster stops | +| in-flight transaction reservations (`clear_in_flight_txs`) | mempool registration `InFlightLease`, held in the connection's state; the mempool records each in-flight hash against the lease's `ActorId` | the lease's destruction removes the hashes still recorded against it, synchronously | +| cell availability and announced-hash budget (`clear_peer_cell_availability`, `forget_peer_announcements`) | the same lease, keyed by (node id, connection `ActorId`) instead of node id plus a "started session" check | the lease's destruction, for that `ActorId` only; a second session of the same node keeps its own | +| re-request to alternates (`retry_on_alternates`) | peer table, best effort | on `Down`, after removing the entry, with a local lookup of live peers (no request to itself). Suppressed by explicit shutdown state, the node's shutdown token that `shutdown_l1` cancels first, not by `Down.reason`: a connection killed at its shutdown deadline ends `Killed`. Lost if the peer table stops first, which is acceptable for a retry | +| disconnect metrics | peer table's `Down` handler (it has the final reason) | best effort | ### What disappears @@ -267,8 +279,9 @@ About 45-60 of the ~135 hand-written lines (counted on `main`): | "connections are not admitted" comment and unmanaged `.start()` | | connections owned by the system | Added: the handshake as a post-start continuation (moving existing code), -the writer as an owned helper, exact owner registrations in the mempool and -broadcaster, and the release logic in the `Down` handlers. Net lines are +the writer as an owned helper, the mempool's `InFlightLease` (an owning +registration released by its destruction), the broadcaster's identity check +and the release logic in the `Down` handlers. Net lines are roughly even; what changes is that every path out of a connection is covered. #7435's integration test (171 lines) stays as a regression test, now exercising the monitor guard. From 188fa50f4e57d5aa7fd64d0d2dfca326a834f30b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:00:53 -0300 Subject: [PATCH 41/52] verify, docs: Ignore only while open, ignore counted as a commit, StartOutcome/ChildStart mismatches fixed (agy round 3, 1) --- docs/design/dynamic-supervisor.md | 26 +++++++++++++++----------- verification/tla/DynamicSupervisor.tla | 9 +++++++-- verification/tla/negative_checks.py | 1 + 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 53532aaa..a5c95ee4 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -114,8 +114,6 @@ Non-goals (v1): [Interaction with rank 12](#interaction-with-rank-12-scoped-registries)). - Supervisor-type dynamic children (a template that builds a nested supervisor per call). Possible later through the nested adapter. -- `ignore` from a factory (the static supervisor does not have it either, - S1). - `Task.Supervisor` (ranks 11 and 15), `PartitionSupervisor` (sharding), significant children and `auto_shutdown` (OTP rejects them for `simple_one_for_one`). @@ -166,10 +164,11 @@ pub struct DynamicChildSpec { /* name, group, factory, restart, impl DynamicChildSpec { /// Permanent, as OTP's default. A restartable child is rebuilt from its /// arguments, hence `Clone`. - pub fn new(name: impl Into, group: &ExecutionGroup, factory: F) -> Self + pub fn new(name: impl Into, group: &ExecutionGroup, factory: F) -> Self where F: Fn(FactoryContext, Args) -> U + Send + Sync + 'static, - U: Future> + Send + 'static, + U: Future> + Send + 'static, + S: Into>, // `A` (via From) or `ChildStart::Ignore` (#259) E: std::fmt::Display; pub fn restart(self, restart: RestartMode) -> Self; } @@ -338,7 +337,7 @@ pub enum TerminateChildError { /// Shared with introspection v1 (static and dynamic supervisors). pub struct ChildInfo { - pub id: ChildKey, // DynamicChildId here; the name for static children + pub key: ChildKey, // DynamicChildId here; the name for static children pub actor: Option, // last published id; None before the first readiness pub ready: bool, // true iff state == Running pub state: ChildState, @@ -381,8 +380,10 @@ pub struct ChildCounts { - Removed slots are evicted from the supervisor's table after slot finalization ([Retained arguments](#retained-arguments)): memory is O(children held). -- `ChildInfo`, `ChildCounts` and `ChildKey` are introspection v1's types; - this design only fixes their dynamic-supervisor contents. +- `ChildInfo`, `ChildCounts` and `ChildKey` are introspection v1's types + (#256); the implementation uses its canonical declarations, which have + more fields than the ones shown here, which are those this design + depends on. ### Restart types @@ -566,8 +567,8 @@ This design depends on #259 (0.9). resolves the replacement (Codex 2). 5. **First-start failure.** Factory error or panic, panic or stop in `started()`, or `startup_timeout`: the generation is stopped and cleaned - up, the slot removed, and `FailedToStart { cause, exit, cleanup_stuck: false }` - returned. Never retried, never charged (OTP). If the cleanup misses its + up, the slot finalized and removed, and + `FailedToStart { cause: ChildStartError, cleanup_stuck: false }` returned. Never retried, never charged (OTP). If the cleanup misses its deadlines (E3 rows 3-4), `FailedToStart { .., cleanup_stuck: true }` is returned at the deadline; the slot stays counted and its work stays owned by the system until it really ends, as S2 releases `start()`'s waiter @@ -865,7 +866,9 @@ Until rank 12, the pattern depends on the restart type (Codex 13): that generation, guarded by the monitor reference. ```rust -let started = peers.start_child(args).await?; +let StartOutcome::Started(started) = peers.start_child(args).await? else { + return Ok(()); // the factory or started() ignored: nothing to track +}; let mref = ctx.monitor(&started.actor.child_handle()); self.by_key.insert(key, Entry { child: started.child, monitor: mref }); // Handler: remove the entry only if it still holds this monitor. @@ -936,7 +939,7 @@ stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | `TerminatedNeverRestarted` | no generation begins after a found `terminate_child` | | `NoGenerationAfterClose` | no generation begins after admission closed | | `NoPublishAfterStop` | no generation is published after close or termination | -| `NoStartAfterShutdown` | no `start_child` success commits after admission closed | +| `NoStartAfterShutdown` | no `start_child` outcome (`Ok` or the ignore decision) commits after admission closed | | `RestartFollowsPolicy` | every restart decision (reservation, attempt, exhaustion) is allowed by the table; no restartable exit is retired while open | | `FirstStartNeverRetried` | a failed first start is never retried | | `FailedAfterCleanup` | `FailedToStart`/`Aborted` only once cleanup really finished (removed, no live generation) or with the deadline failure recorded | @@ -1012,6 +1015,7 @@ property above has at least one; the two action properties do not. | `SlotLookup` | an id call reaches a newer generation | `TermResultsSound` | main | | `QueuedTermOk` | shutdown answers a queued call `Ok` | `QueuedTermStopped` | main | | `EarlyFree` | removal frees the slot before finalization | `RemovedAfterFinalize` | main | +| `IgnoreAfterClose` | an ignore decided after shutdown began | `NoStartAfterShutdown` | main | | `IgnoreFreesEarly` | an ignored attempt frees the slot at once | `RemovedAfterFinalize` | main | | `StuckResultLost` | `CleanupStuck` not committed; later turns into `Ok` | `TermResultsSound` | main | | `IgnoreExpect` | a fenced call terminates a child that moved on | `TermResultsSound` | main | diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index d4d8acd1..1c9a7d7f 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -72,7 +72,7 @@ Faults == {"None", "TermLost", "NoDeadline", "SkipSignal", "SlotLookup", "QueuedTermOk", "PublishAfterExit", "StuckResultLost", "IgnoreExpect", \* round 3 - "EarlyFree", "IgnoreFreesEarly"} + "EarlyFree", "IgnoreFreesEarly", "IgnoreAfterClose"} ASSUME N \in Nat \ {0} /\ MaxChildren \in Nat /\ MaxRestarts \in Nat /\ MaxExpire \in Nat ASSUME ModeSet \subseteq Modes /\ ModeSet # {} @@ -313,6 +313,9 @@ Ready(s) == \* decided now (cause "ignored"), answered after cleanup or at its deadline. Ignore(s) == /\ sl[s].cs \in {"starting", "restarting"} + \* an ignore decides the outcome: only while admission is open (agy round 3) + /\ \/ open + \/ Fault \in {"AcceptDuringShutdown", "IgnoreAfterClose"} /\ sl[s].cs = "starting" /\ IF Fault = "IgnoreFreesEarly" THEN sl' = [sl EXCEPT ![s].cs = "removed", ![s].alive = {}, ![s].pub = 0, ![s].result = IF sl[s].cs = "starting" THEN "ignored" ELSE sl[s].result] @@ -553,8 +556,10 @@ Decided(s) == \/ exhausted' /\ ~exhausted /\ sl'[s].cs \in {"finalizing", "removed"} GhostUpdate == + \* a start outcome committed after close: Ok, or the ignore decision /\ lateCommit' = [s \in Slots |-> lateCommit[s] \/ - (sl'[s].result = "ok" /\ sl[s].result # "ok" /\ ~open)] + ((sl'[s].result = "ok" /\ sl[s].result # "ok" + \/ sl'[s].cause = "ignored" /\ sl[s].cause # "ignored") /\ ~open)] /\ latePub' = [s \in Slots |-> latePub[s] \/ (sl'[s].pub # sl[s].pub /\ sl'[s].pub # 0 /\ (~open \/ sl[s].term))] /\ lateGenClose' = [s \in Slots |-> lateGenClose[s] \/ (Grew(s) /\ ~open)] diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 93934796..fc4e6ad7 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -264,6 +264,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), ("MCDynamicSupervisor", "EarlyFree", "RemovedAfterFinalize"), ("MCDynamicSupervisor", "IgnoreFreesEarly", "RemovedAfterFinalize"), + ("MCDynamicSupervisor", "IgnoreAfterClose", "NoStartAfterShutdown"), ] DYNAMIC_TEMPORAL_CONTROLS = [ # MCDynamicSupervisorAbort has no startup deadlines and passes unmutated. From af3e3833843991b6c70738c5a51c4107b09fd2d1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:01:23 -0300 Subject: [PATCH 42/52] verify, docs: stale terminate ids replayable, ExpectedChild is the last published child, no ticket for directly consumed exits, fan-out ends on its last signal (agy round 3, 2) --- docs/design/dynamic-supervisor.md | 12 ++++++++---- verification/tla/DynamicSupervisor.tla | 26 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index a5c95ee4..af673df3 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -324,6 +324,10 @@ pub enum RejectReason { Placement(SystemError), // the group is closed or out of native capacity } +/// The slot's last published child: its `ActorId` and the generation at +/// which it was published. Failed replacement attempts do not change it +/// (they advance `ChildInfo::generation`, the attempt count, but publish +/// nothing), so an `expect` taken during backoff still matches. pub struct ExpectedChild { pub actor: ActorId, pub generation: u64 } #[non_exhaustive] @@ -1058,19 +1062,19 @@ the generation, and where noted the queue ticket or call id. | `DAbort` (slot, gen) | shutdown cancelled a construction | `AbortStart` | | `DExit` (slot, gen, reason) | a running generation began exiting on its own | `SelfExit` | | `DStopSeen` (slot, gen, reason) | a signalled generation began exiting (Shutdown/Killed) | `StopObserved` | -| `DCleaned` (slot, gen, cleanup_failure, ticket) | owned cleanup finished; exit notice queued | `CleanupDone` | +| `DCleaned` (slot, gen, cleanup_failure, ticket?) | owned cleanup finished; the exit notice is queued with a ticket, or carries none when the start or terminate waiting for this child consumes it directly | `CleanupDone` | | `DStuck` (slot, gen, kind) | a deadline was missed | `CleanupStuck` | -| `DStuckDone` (slot) | a stuck child or finalization finally ended | `StuckDone` | +| `DStuckDone` (slot) | a stuck child or finalization finally ended (after `DStuck`, the generation's later end emits only this, never `DCleaned`) | `StuckDone` | | `DFinalized` (slot) | slot finalization ended: retained arguments dropped, capacity and the slot released; a pending start answered | `Finalize` | | `DDecide` (slot, ticket?, outcome: finalize / backoff / restart / exhausted) | the loop handled an exit; "finalize" starts slot finalization, which `DFinalized` (or `DStuck`) ends | `Decide` | | `DRetryQueued` (slot, ticket) | the backoff timer fired | `BackoffFire` | | `DRetry` (slot, gen, ticket) | the loop took a retry | `BackoffDone` | | `DCancel` (slot) | a pending restart was cancelled and refunded; finalization starts | `CancelBackoff` | -| `DTermCall` (call, slot, named gen, ticket) | `terminate_child`/`terminate_slot` called | `TermCall` | +| `DTermCall` (call, slot, named gen, ticket) | `terminate_child`/`terminate_slot` called; the named generation is whatever the caller passed, current or superseded | `TermCall` | | `DTermLookup` (call, outcome: found / notfound / stuck / stale) | the loop handled the call | `LoopTerm` | | `DTermReject` (call) | shutdown answered a queued call | `RejectQueuedTerm` | | `DTermDone` (call, result) | the call resolved `Ok` | `TermResolve` | -| `DClose`, `DSignal` (slot), `DDone` | shutdown began, a child was signalled, all children finished | `Close`, `SupSignal`/`SupSignalStep`, `SupDone` | +| `DClose`, `DSignal` (slot), `DDone` | external or parent shutdown began (exhaustion and a restart-required or closing stuck child close admission themselves, inside `DDecide`/`DStuck`); a child was signalled (the last signal ends the fan-out); all children finished | `Close`, `SupSignal`/`SupSignalStep`, `SupDone` | | `DExpire` | a grant left the window | `WindowExpire` | The trace checker (`trace_check.py`) learns a `dynamic-v1` schema that diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 1c9a7d7f..721a73e6 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -220,11 +220,14 @@ RejectQueued(s) == /\ UNCHANGED <> -\* terminate_child (by the id the caller saw) or terminate_slot. +\* terminate_child(id) or terminate_slot(slot, expect). The caller may name +\* any generation it ever saw published, the current one or a superseded +\* one (a stale id, an outdated expect), so stale calls replay (agy round 3). TermCall(c) == /\ tc[c].st = "none" /\ sl[CallSlot[c]].everReady /\ IF open - THEN /\ tc' = [tc EXCEPT ![c] = [st |-> "queued", g |-> sl[CallSlot[c]].lastPub]] + THEN /\ \E g \in (IF CallKind[c] = "slot" THEN {0} ELSE 1..sl[CallSlot[c]].lastPub) : + tc' = [tc EXCEPT ![c] = [st |-> "queued", g |-> g]] /\ Enq([k |-> "term", s |-> CallSlot[c], c |-> c, t |-> 0]) ELSE /\ tc' = [tc EXCEPT ![c].st = "stopped"] /\ UNCHANGED <> /\ UNCHANGED < "exit", s |-> s, c |-> 0, t |-> 0]) + /\ IF op.s = s /\ op.k \in {"start", "term"} + THEN UNCHANGED <> + ELSE Enq([k |-> "exit", s |-> s, c |-> 0, t |-> 0]) /\ op' = IF op.k = "restart" /\ op.s = s THEN Idle ELSE op /\ UNCHANGED <> @@ -433,6 +440,10 @@ Decide(s) == \/ op.s = s /\ op.k \in {"start", "term"} /\ q' = Rm(i) /\ served' = IF i = 1 /\ LoopIdle THEN q[i].t ELSE served + \* consumed directly by the operation waiting for this child + \/ /\ op.s = s /\ op.k \in {"start", "term"} /\ ExitIdx(s) = {} + /\ sl[s].cs = "decide" + /\ UNCHANGED <> \/ Fault = "OverlapGeneration" /\ sl[s].cs = "cleanup" /\ LoopIdle /\ UNCHANGED <> /\ LET ownStart == op.s = s /\ op.k = "start" @@ -535,7 +546,10 @@ SupSignalStep == /\ LET todo == {s \in Slots : ~sl[s].stopSig /\ sl[s].cs \in Active} IN IF todo = {} THEN sphase' = "waiting" /\ UNCHANGED sl - ELSE \E s \in todo : sl' = [sl EXCEPT ![s].stopSig = TRUE] /\ sphase' = "signal" + ELSE \E s \in todo : + /\ sl' = [sl EXCEPT ![s].stopSig = TRUE] + \* the last signal ends the fan-out (no extra unlogged step) + /\ sphase' = IF todo = {s} THEN "waiting" ELSE "signal" /\ UNCHANGED <> From df3960cebc4ac5d30119227bc9ff1cbd52bd441b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:02:06 -0300 Subject: [PATCH 43/52] docs: round-3 review resolutions; approved for implementation --- CHANGELOG.md | 5 ++++- docs/TESTING.md | 2 +- docs/design/dynamic-supervisor.md | 17 ++++++++++++++++- verification/tla/README.md | 6 +++--- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c628b..abffefe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -161,7 +161,10 @@ trace-validation test plan, a fenced `terminate_slot(.., expect)` for introspection, the shared shapes with unified start (#259: `StartOutcome>`, `ChildStartError`, `Init::continue_with`, - `StartOptions`), and 41 negative controls. + `StartOptions`), and 41 negative controls. Round 3: slot finalization + and ignored-attempt cleanup kept in the model, FIFO checked at selection, + queued terminate calls `Stopped` at shutdown, mempool reservations held by + an owning registration, 44 negative controls; approved for implementation. ## 0.7.3 — 2026-11-10 diff --git a/docs/TESTING.md b/docs/TESTING.md index 3ab7adbe..a246a461 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1088,7 +1088,7 @@ the budget reservation protocol, ignore, stuck children and shutdown. TLC checks 23 safety invariants, two action properties and five liveness properties (callers answered even if cleanup never ends; shutdown and recovery complete when it does). Six configurations run on every PR and one -weekly; 41 named negative controls each fail their property. The design's +weekly; 44 named negative controls each fail their property. The design's [test plan](design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) specifies the trace events and trace-validation tests the implementation must ship. diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index af673df3..02228559 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -1,6 +1,6 @@ # Dynamic supervisor: design contract -**Status: design in review (revision 3), not implemented.** ROADMAP rank 12a. +**Status: approved for implementation after review round 3 (revision 3), not implemented.** ROADMAP rank 12a. Draft PR [#245](https://github.com/lambdaclass/spawned/pull/245). Revision 2 applied the maintainer's decisions on revision 1's open questions and the first reviews; revision 3 applies the decisions on round two's questions and @@ -1245,3 +1245,18 @@ action mapping, the required tests and the expected coverage. acting when the slot's published child moved on. Modelled as a third call kind (`expect`) in `LoopTerm`, with control `IgnoreExpect`. `ChildInfo`, `ChildCounts` and `ChildKey` are shared with introspection v1. + +### Round 3 (confirmation) + +Codex (3 P1) and agy (2 P1, P2s), both "approve after the listed changes", +no P0. Each fixed in its own commit. + +| Finding | Resolution | +|---|---| +| **Codex 1**: `Ignore` and removal erased cleanup the contract keeps (an `Init::ignore()` attempt's state and helpers; slot finalization of retained arguments), so real interleavings could not replay | **Accepted.** An ignored attempt goes through cleanup like any exit; every removal goes through a `finalizing` state that keeps the slot counted and its capacity held until finalization ends (or is stuck), and the loop awaits it. New events `DFinalized`, `DDecide(finalize)`. Invariant `RemovedAfterFinalize`; controls `EarlyFree`, `IgnoreFreesEarly`. An ignored start whose cleanup misses its deadline is answered `Ignored` at the deadline, the slot stays counted | +| **Codex 2**: `FifoOrder` accepted serve-2-then-reject-1; a queued terminate answered `Ok` at shutdown passed; `NoAbort` failed on a config where `StartAnswered` fails without the mutation | **Accepted.** `FifoOrder` checks that every selection is the head at selection. `QueuedTermStopped` (history of calls queued when admission closed) with `QueuedTermOk` mutating the real wrong `Ok`. `ShutdownAnswersStarts` on `MCDynamicSupervisorAbort` (no startup deadlines), which passes unmutated; `NoAbort` fails there | +| **Codex 3**: mempool reservations released on the peer table's `Down` could be lost if the peer table stopped first; `Down.reason == Shutdown` cannot detect shutdown (escalation kills) | **Accepted.** Reservations outside actors are held by an owning registration (`InFlightLease`) in the connection's state, released synchronously by its destruction on every exit path before the slot frees. Records inside an actor's state die with it. Retries are suppressed by the node's shutdown token | +| **agy 1**: `Ignore` had no `open` guard and escaped `NoStartAfterShutdown`; leftover `StartOutcome`/`ChildStart` mismatches | **Accepted.** `Ignore` only while open; the ignore decision counts as a commit in the history; control `IgnoreAfterClose`. The non-goal line removed, the factory returns `S: Into>`, `FailedToStart { cause: ChildStartError, .. }` throughout, the rank 12 example matches `StartOutcome::Started`, `ChildInfo::key` | +| **agy 2**: stale `terminate` ids could not be replayed; `ExpectedChild::generation` vs failed replacements; tickets for directly consumed exits; an unlogged fan-out step | **Accepted.** A call names any generation the caller saw; `ExpectedChild` is the last published child (failed attempts publish nothing); an exit consumed by the waiting start or terminate takes no ticket (`DCleaned` ticket optional); the last signal ends the fan-out. `DClose` only for external or parent shutdown | +| **agy P2**: the loop must await finalization | **Accepted** (with Codex 1) | +| **Codex note**: use the canonical shared declarations | **Accepted**: `ChildInfo`/`ChildCounts`/`ChildKey` and `ChildStart` are #256's and #259's; the shapes here show only the fields this design depends on | diff --git a/verification/tla/README.md b/verification/tla/README.md index 415a5d7a..1e0e1f7d 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -17,7 +17,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | -| `DynamicSupervisor.tla`, `MCDynamicSupervisor*.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): OTP's serialised control loop with one FIFO queue; `start_child` and `terminate_child` racing crashes, restarts, `max_children`, the budget reservation protocol, ignore, stuck children and shutdown. 41 named negative controls. Per PR: `MCDynamicSupervisor`, `Backoff`, `Zero`, `Three`, `Callers`, `Live`; weekly: `Deep`. | +| `DynamicSupervisor.tla`, `MCDynamicSupervisor*.tla/.cfg` | Rank 12a design model (not implemented, [design](../../docs/design/dynamic-supervisor.md)): OTP's serialised control loop with one FIFO queue; `start_child` and `terminate_child` racing crashes, restarts, `max_children`, the budget reservation protocol, ignore, stuck children and shutdown. 44 named negative controls. Per PR: `MCDynamicSupervisor`, `Backoff`, `Zero`, `Three`, `Callers`, `Live`; weekly: `Deep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -619,8 +619,8 @@ or get stuck. Event invariants read history variables computed by deadlines fire) and checks that callers are answered; `Spec` adds cleanup progress for shutdown completion and recovery. The design's [model section](../../docs/design/dynamic-supervisor.md#what-the-model-establishes) -lists the 23 invariants, the properties and the 41 negative controls (35 -safety, 6 liveness), run one at a time from `negative_checks.py`; its +lists the 23 invariants, the properties and the 44 negative controls (37 +safety, 7 liveness), run one at a time from `negative_checks.py`; its [test plan](../../docs/design/dynamic-supervisor.md#test-plan-relating-the-model-to-the-code) lists the trace events the implementation must emit for trace validation. From d7f7d3212f640836290c2a8f770d709bdf0112e6 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:02:25 -0300 Subject: [PATCH 44/52] verify: parenthesise mixed conjunction/disjunction (TLC parse) --- verification/tla/DynamicSupervisor.tla | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 721a73e6..ea410598 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -572,8 +572,8 @@ Decided(s) == GhostUpdate == \* a start outcome committed after close: Ok, or the ignore decision /\ lateCommit' = [s \in Slots |-> lateCommit[s] \/ - ((sl'[s].result = "ok" /\ sl[s].result # "ok" - \/ sl'[s].cause = "ignored" /\ sl[s].cause # "ignored") /\ ~open)] + (((sl'[s].result = "ok" /\ sl[s].result # "ok") + \/ (sl'[s].cause = "ignored" /\ sl[s].cause # "ignored")) /\ ~open)] /\ latePub' = [s \in Slots |-> latePub[s] \/ (sl'[s].pub # sl[s].pub /\ sl'[s].pub # 0 /\ (~open \/ sl[s].term))] /\ lateGenClose' = [s \in Slots |-> lateGenClose[s] \/ (Grew(s) /\ ~open)] @@ -585,7 +585,7 @@ GhostUpdate == /\ ~sl[s].term /\ Allowed(s))] /\ restarts' = restarts + Cardinality({s \in Slots : Grew(s) /\ sl[s].gen > 0}) \* the loop selected an operation: it must be the head at selection - /\ outOfOrder' = (outOfOrder \/ (served' # served /\ (Len(q) = 0 \/ served' # q[1].t))) + /\ outOfOrder' = (outOfOrder \/ ((served' # served) /\ ((Len(q) = 0) \/ (served' # q[1].t)))) \* terminate calls still queued at the moment admission closed /\ queuedAtClose' = [c \in TermCalls |-> queuedAtClose[c] \/ (open /\ ~open' /\ tc[c].st = "queued")] From 13a2feef271d5875123e2b717d32238612d9bbb4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:03:11 -0300 Subject: [PATCH 45/52] verify: TermResultsSound targets calls pending when their child got stuck (no false alarm for a later call) --- docs/design/dynamic-supervisor.md | 2 +- verification/tla/DynamicSupervisor.tla | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 02228559..02e94af6 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -957,7 +957,7 @@ stuck children, for `ClosedLeadsToDone` and `RecoveryProgress`. | `OkMeansReady` | `Ok` only for a child that became ready | | `QueuedTermStopped` | a terminate call still queued when shutdown began is answered `Stopped` and stays so | | `RemovedAfterFinalize` | a slot is freed only after slot finalization (and an ignored attempt's cleanup) | -| `TermResultsSound` | `Ok` only after removal and never for a stuck child; a pending call holds the loop; an id call or a fenced slot call reaches only the generation it named | +| `TermResultsSound` | `Ok` only after removal; a call in progress when its child got stuck reports `CleanupStuck`; a pending call holds the loop; an id call or a fenced slot call reaches only the generation it named | Action properties: `NeverReopens`, `TermResultStable` (a committed terminate result never changes). Liveness: `StartAnswered`, `TerminateAnswered`, diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index ea410598..82d9c498 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -102,12 +102,12 @@ VARIABLES mode, sl, tc, \* history (GhostUpdate only) lateCommit, latePub, lateGenClose, lateGenTerm, badRestart, badRetire, - restarts, outOfOrder, freedEarly, queuedAtClose + restarts, outOfOrder, freedEarly, queuedAtClose, pendAtStuck sup == <> ghost == <> + restarts, outOfOrder, freedEarly, queuedAtClose, pendAtStuck>> vars == <> SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, lastPub |-> 0, @@ -140,6 +140,7 @@ Init == /\ badRestart = [s \in Slots |-> FALSE] /\ badRetire = [s \in Slots |-> FALSE] /\ restarts = 0 /\ outOfOrder = FALSE /\ freedEarly = [s \in Slots |-> FALSE] /\ queuedAtClose = [c \in TermCalls |-> FALSE] + /\ pendAtStuck = [c \in TermCalls |-> FALSE] ----------------------------------------------------------------------------- (* Helpers *) @@ -589,6 +590,10 @@ GhostUpdate == \* terminate calls still queued at the moment admission closed /\ queuedAtClose' = [c \in TermCalls |-> queuedAtClose[c] \/ (open /\ ~open' /\ tc[c].st = "queued")] + \* terminate calls in progress at the moment their child got stuck + /\ pendAtStuck' = [c \in TermCalls |-> pendAtStuck[c] \/ + (tc[c].st = "pending" /\ sl[CallSlot[c]].stuck = "none" + /\ sl'[CallSlot[c]].stuck # "none")] /\ freedEarly' = [s \in Slots |-> freedEarly[s] \/ (sl'[s].cs = "removed" /\ sl[s].cs \notin {"finalizing", "removed"})] /\ UNCHANGED mode @@ -706,14 +711,15 @@ ConcurrentShutdown == OkMeansReady == \A s \in Slots : sl[s].result = "ok" => sl[s].everReady -\* terminate results: Ok only after removal; Stuck stays Stuck; a call -\* still queued at shutdown is Stopped, never Ok; an id call or a fenced +\* terminate results: Ok only after removal; a call whose child got stuck +\* while it waited is Stuck (and stays so); an id call or a fenced \* (expect) slot call never reaches a generation other than the one named. TermResultsSound == \A c \in TermCalls : LET s == CallSlot[c] IN - /\ tc[c].st = "ok" => - sl[s].cs = "removed" /\ sl[s].alive = {} /\ sl[s].stuck = "none" + /\ tc[c].st = "ok" => sl[s].cs = "removed" /\ sl[s].alive = {} + \* a call in progress when its child got stuck reports CleanupStuck + /\ pendAtStuck[c] => tc[c].st = "stuck" /\ tc[c].st = "pending" => op = [k |-> "term", s |-> s, c |-> c] /\ tc[c].st = "pending" /\ CallKind[c] \in {"id", "expect"} => sl[s].lastPub = tc[c].g From 49f907eae71f434904cd84dc7cadfa17bd7168b7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:04:45 -0300 Subject: [PATCH 46/52] verify: an ignored restart attempt is dropped (OTP), part of the independent restart policy --- verification/tla/DynamicSupervisor.tla | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 82d9c498..9f4b0622 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -162,8 +162,10 @@ DesignAllowsRestart(m, r, cf) == CASE m = "Temporary" -> FALSE [] m = "Permanent" -> TRUE [] m = "Transient" -> r \in {"Panic", "Kill", "Killed"} \/ cf +\* An attempt that ignored is dropped, never restarted (OTP: ignore on a +\* restart removes the child). Allowed(s) == - /\ sl[s].everReady + /\ sl[s].everReady /\ ~sl[s].ign /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, sl[s].cfail) \/ ~sl[s].readyGen /\ mode[s] # "Temporary" From 0860dfd95ead0f2f4d4b92175994f058c471b068 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:21:51 -0300 Subject: [PATCH 47/52] docs: Ignored { pending } and no stopped() after a failed init, as unified start #259 rev. 3 --- docs/design/dynamic-supervisor.md | 43 +++++++++++++++++++------------ 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/design/dynamic-supervisor.md b/docs/design/dynamic-supervisor.md index 02e94af6..f25f3714 100644 --- a/docs/design/dynamic-supervisor.md +++ b/docs/design/dynamic-supervisor.md @@ -146,10 +146,10 @@ Round two (revision 3): | R5 | ethrex teardown | No message from `Drop`. Each resource's owner monitors the connection and releases on `Down`, as an Erlang owner would | | G1 | Post-start work | Depends on the post-start continuation (a `handle_continue` equivalent) of the unified-start design (0.9), not on "the first message", which other messages could overtake | | G2 | Restart type for connections | The ethrex mapping and the example use `temporary()` (a dropped peer must not be redialled by the supervisor); the default stays Permanent (OTP) | -| G3 | `ignore` | Accepted from the factory: nothing is stored, `start_child` returns `Ignored` (OTP `{ok, undefined}`) | +| G3 | `ignore` | Accepted from the factory and from `Init::ignore()`: nothing is kept once cleaned up, `start_child` returns `Ignored { pending }` (OTP `{ok, undefined}`) | | G4 | Start data | Deferred, as in unified start (#259): `gen_server` returns only `{ok, Pid}`; `Started` is `{ actor, child }` | | G5 | Introspection | `ChildInfo` and `ChildCounts` are the types introspection v1 exposes for static supervisors too; names agreed across both designs | -| U1 | Shared shapes with unified start (#259 rev. 2, section 18) | `StartOutcome>` with `Ignored` as an `Ok` outcome; `FailedToStart { cause: ChildStartError, cleanup_stuck }`; `ChildStart`; `StartOptions` through `DynamicChildSpec::options`; the continuation is `Init::continue_with(msg)`, which precedes messages but is no termination barrier | +| U1 | Shared shapes with unified start (#259 rev. 2-3, section 18) | `StartOutcome>` with `Ignored { pending: Option }` as an `Ok` outcome; a failed init never runs `stopped()`; `FailedToStart { cause: ChildStartError, cleanup_stuck }`; `ChildStart`; `StartOptions` through `DynamicChildSpec::options`; the continuation is `Init::continue_with(msg)`, which precedes messages but is no termination barrier | | I1 | Fenced termination (introspection #256 rev. 3) | `terminate_slot` takes an optional `expect: ActorId@generation`; a mismatch returns `Stale { current }` and does nothing, checked at the supervisor operation itself, so the console's `terminate --expect` cannot kill a child that moved on | ## API @@ -266,8 +266,12 @@ impl DynamicSupervisor { pub enum StartOutcome { Started(T), /// The factory returned `ChildStart::Ignore`, or `started()` returned - /// `Init::ignore()`: nothing is supervised or counted (OTP {ok, undefined}). - Ignored, + /// `Init::ignore()` (OTP {ok, undefined}). `pending` is `None` once the + /// ignoring instance was cleaned up and the slot freed; `Some(handle)` + /// when the answer came at the cleanup deadline first: the slot still + /// counts (state `Stopping`) until the instance is published and the slot + /// finalized, and `handle.wait_exit()` reaches that end (#259 rev. 3). + Ignored { pending: Option }, } pub struct Started { @@ -360,15 +364,17 @@ pub struct ChildCounts { released: `Init::ignore()` drops the constructed state and joins its owned work; a factory ignore has nothing to drop. Then the slot is finalized (retained arguments dropped) and only then freed and answered - `Ok(StartOutcome::Ignored)`. If that cleanup misses its deadline the start - is still answered `Ignored` at the deadline (the outcome was decided when - the child ignored) and the slot stays counted until the work ends, as for - any stuck retiring child (control `IgnoreFreesEarly`; Codex round 3). + `Ok(StartOutcome::Ignored { pending: None })`. If that cleanup misses its + deadline the start is answered `Ok(Ignored { pending: Some(handle) })` at + the deadline (the outcome was decided when the child ignored) and the slot + stays counted, `Stopping`, until the instance is published and the slot + finalized, as for any stuck retiring child (control `IgnoreFreesEarly`; + Codex round 3; #259 rev. 3). - The factory returns `Result, E>` with `ChildStart::Actor(a) | ChildStart::Ignore` (`From`), and `started()` may return `Init::ignore()`; both are unified start's shapes (#259). An - ignore on a first start returns `Ok(StartOutcome::Ignored)` and stores - nothing; on a restart it removes the child (OTP drops it), without charge + ignore on a first start returns `Ok(StartOutcome::Ignored { .. })` and + keeps nothing once cleaned up; on a restart it removes the child (OTP drops it), without charge beyond the attempt already stamped. - Typed start data (`Started::data`) is deferred, as in #259: `gen_server` returns only `{ok, Pid}`. @@ -564,14 +570,17 @@ This design depends on #259 (0.9). commit is the success: from then on `start_child` returns `Ok(Started)`, even if shutdown begins before the caller observes it (success is historical, like `ActorRef::ready()` and `start()`'s - readiness, `supervision.rs` `start_owned`). A factory that returns - ignore answers `Ok(Ignored)` and leaves nothing behind. The returned `actor` may + readiness, `supervision.rs` `start_owned`). An ignore answers + `Ok(Ignored { pending })` after cleanup and finalization (above). The returned `actor` may already be dead: if the child crashed between the commit and the caller's wake-up, it is handled by the restart table and `child.current()` resolves the replacement (Codex 2). 5. **First-start failure.** Factory error or panic, panic or stop in - `started()`, or `startup_timeout`: the generation is stopped and cleaned - up, the slot finalized and removed, and + `started()`, `Init::error`/`Init::stop`, or `startup_timeout`: the + generation is stopped and cleaned up without `stopped()` (a failed init + never runs it, as OTP never calls `terminate/2` after a failed `init/1`; + #259 rev. 3: `stopped()` runs only after a ready commit), its state is + dropped and its owned work joined, the slot finalized and removed, and `FailedToStart { cause: ChildStartError, cleanup_stuck: false }` returned. Never retried, never charged (OTP). If the cleanup misses its deadlines (E3 rows 3-4), `FailedToStart { .., cleanup_stuck: true }` is returned at the deadline; the slot stays counted and its work stays owned @@ -580,7 +589,8 @@ This design depends on #259 (0.9). [Stuck children](#stuck-children). 6. **Shutdown during construction**: the attempt is cancelled (state dropped without hooks if `started()` had not begun, L2; otherwise the parent exit - signal); `Aborted` is returned after its cleanup or deadline. The error + signal, applied when `started()` returns, and still no `stopped()`, since + the child never committed ready); `Aborted` is returned after its cleanup or deadline. The error reflects what ended the attempt: a factory failure recorded before shutdown stays `FailedToStart` (agy 7). 7. **Cancellation** of the `start_child` future does not cancel the start @@ -774,7 +784,7 @@ As for any actor (exit-signals design, phases of an exit, with rank 11): | 1 | `start_child` is synchronous and returns the child | `{ok, Pid}` after the start function returns | `Ok(Started { actor, child })` after `started()` | aligned; `child` is an addition (row 9) | | 2 | Operations serialised by the supervisor | one process: calls and `'EXIT'` messages handled in mailbox order, one at a time | one control loop over one FIFO queue of starts, terminate calls, exits and retries | aligned (decisions Q1, R1) | | 3 | Template plus per-call args | `ExtraArgs` appended (Elixir: fixed `extra_arguments` prepended) | typed `Args` passed to the factory | T | -| 4 | `ignore` | `{ok, undefined}`, nothing stored; on restart the child is dropped | `Ok(Ignored)`, nothing stored; on restart the child is removed | aligned (decision G3) | +| 4 | `ignore` | `{ok, undefined}`, nothing stored; on restart the child is dropped | `Ok(Ignored { pending })`, nothing kept after cleanup; on restart the child is removed | aligned (decision G3); `pending` is R (cleanup can outlive the answer) | | 5 | Start failure | `{error, Reason}`, not counted, not retried | `FailedToStart`, not counted, not retried, after cleanup or at its deadline | aligned; the deadline release is R (S2) | | 6 | Default restart type | `permanent` | `Permanent` | aligned | | 7 | Temporary children keep no args | `mapsets` | `temporary()` moves args once, no `Clone` | aligned; R for the separate constructor | @@ -1260,3 +1270,4 @@ no P0. Each fixed in its own commit. | **agy 2**: stale `terminate` ids could not be replayed; `ExpectedChild::generation` vs failed replacements; tickets for directly consumed exits; an unlogged fan-out step | **Accepted.** A call names any generation the caller saw; `ExpectedChild` is the last published child (failed attempts publish nothing); an exit consumed by the waiting start or terminate takes no ticket (`DCleaned` ticket optional); the last signal ends the fan-out. `DClose` only for external or parent shutdown | | **agy P2**: the loop must await finalization | **Accepted** (with Codex 1) | | **Codex note**: use the canonical shared declarations | **Accepted**: `ChildInfo`/`ChildCounts`/`ChildKey` and `ChildStart` are #256's and #259's; the shapes here show only the fields this design depends on | +| **Unified start #259 rev. 3** (cross-design): `Ignored { pending }` and a stuck ignore keeping its slot until publication; a failed init never runs `stopped()` | **Adopted.** `start_child` returns `Ok(StartOutcome::Ignored { pending })`; a stuck ignored attempt answers at the deadline with `Some(handle)` and its slot counts (`Stopping`) until publication and finalization, which the model already does (`CleanupStuck` commits the ignore, the slot stays counted until `StuckDone` and `Finalize`). First-start failure and shutdown-during-construction text now say `stopped()` does not run; nothing in this design depended on it | From 7c0ea43a573350a7a0b8395b788e03a13e7da3d9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:29:47 -0300 Subject: [PATCH 48/52] verify: broad configs move to weekly (WideDeep, BackoffDeep, 10-13 min); smaller per-PR configs; controls target the broad ones --- .github/workflows/verification.yml | 2 + verification/tla/MCDynamicSupervisor.cfg | 16 ++-- verification/tla/MCDynamicSupervisor.tla | 4 + .../tla/MCDynamicSupervisorBackoff.cfg | 10 +-- .../tla/MCDynamicSupervisorBackoffDeep.cfg | 48 ++++++++++++ .../tla/MCDynamicSupervisorBackoffDeep.tla | 3 + .../tla/MCDynamicSupervisorWideDeep.cfg | 50 ++++++++++++ .../tla/MCDynamicSupervisorWideDeep.tla | 3 + verification/tla/negative_checks.py | 76 +++++++++---------- 9 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorBackoffDeep.cfg create mode 100644 verification/tla/MCDynamicSupervisorBackoffDeep.tla create mode 100644 verification/tla/MCDynamicSupervisorWideDeep.cfg create mode 100644 verification/tla/MCDynamicSupervisorWideDeep.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 192369bd..e2c57a49 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -117,6 +117,8 @@ jobs: - MCLifecycleLiveDeep - MCLifecycleDeep - MCDynamicSupervisorDeep + - MCDynamicSupervisorWideDeep + - MCDynamicSupervisorBackoffDeep steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v16 diff --git a/verification/tla/MCDynamicSupervisor.cfg b/verification/tla/MCDynamicSupervisor.cfg index d531e95e..afaf08ec 100644 --- a/verification/tla/MCDynamicSupervisor.cfg +++ b/verification/tla/MCDynamicSupervisor.cfg @@ -1,8 +1,8 @@ -\* Per PR, safety: two start_child calls (every pair of restart types), -\* two terminate calls on slot 1 (by id and by slot), max_children 1, -\* budget 1 with one window release, no backoff (the default), exits -\* Normal/Panic, start failures factory/Normal/timeout, cleanup failures. -\* The safety negative controls run here unless noted. +\* Per PR, safety: two start_child calls (every pair of restart types), one +\* terminate_slot call fenced with expect (naming any generation seen), +\* max_children 1, budget 1, one window release, no backoff, exits Normal/Panic, +\* start failures factory/Normal/timeout, ignore, cleanup failures. +\* MCDynamicSupervisorWideDeep adds a terminate-by-id call. SPECIFICATION Spec CONSTANTS N = 2 @@ -13,9 +13,9 @@ CONSTANTS SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} - TermCalls <- MCCalls2 - CallSlot <- MCCallSlot2 - CallKind <- MCCallKind2 + TermCalls <- MCCalls1e + CallSlot <- MCCallSlot1e + CallKind <- MCCallKind1e BackoffOn = FALSE StartDeadlines = TRUE FanoutSteps = FALSE diff --git a/verification/tla/MCDynamicSupervisor.tla b/verification/tla/MCDynamicSupervisor.tla index 47b23eca..399a110d 100644 --- a/verification/tla/MCDynamicSupervisor.tla +++ b/verification/tla/MCDynamicSupervisor.tla @@ -11,6 +11,10 @@ MCCallKind2 == [c \in MCCalls2 |-> IF c = 1 THEN "id" ELSE "expect"] MCCalls1 == {1} MCCallSlot1 == [c \in MCCalls1 |-> 1] MCCallKind1 == [c \in MCCalls1 |-> "slot"] +\* One terminate_slot call fenced with expect. +MCCalls1e == {1} +MCCallSlot1e == [c \in MCCalls1e |-> 1] +MCCallKind1e == [c \in MCCalls1e |-> "expect"] \* No terminate calls. MCCalls0 == {} MCCallSlot0 == [c \in MCCalls0 |-> 1] diff --git a/verification/tla/MCDynamicSupervisorBackoff.cfg b/verification/tla/MCDynamicSupervisorBackoff.cfg index 64857d64..ae2a7096 100644 --- a/verification/tla/MCDynamicSupervisorBackoff.cfg +++ b/verification/tla/MCDynamicSupervisorBackoff.cfg @@ -1,5 +1,5 @@ -\* Per PR, safety with backoff configured: reservation, timer, retry -\* through the queue, refund on cancel. Reservation controls run here. +\* Per PR, safety with backoff configured and one unfenced terminate_slot +\* call. MCDynamicSupervisorBackoffDeep has two calls. SPECIFICATION Spec CONSTANTS N = 2 @@ -10,9 +10,9 @@ CONSTANTS SelfExits = {"Normal", "Panic"} StartFails = {"none", "Normal"} CleanupFails = {FALSE, TRUE} - TermCalls <- MCCalls2 - CallSlot <- MCCallSlot2 - CallKind <- MCCallKind2 + TermCalls <- MCCalls1 + CallSlot <- MCCallSlot1 + CallKind <- MCCallKind1 BackoffOn = TRUE StartDeadlines = TRUE FanoutSteps = FALSE diff --git a/verification/tla/MCDynamicSupervisorBackoffDeep.cfg b/verification/tla/MCDynamicSupervisorBackoffDeep.cfg new file mode 100644 index 00000000..e704e24c --- /dev/null +++ b/verification/tla/MCDynamicSupervisorBackoffDeep.cfg @@ -0,0 +1,48 @@ +\* Weekly (--deep): as WideDeep with backoff configured (reservation, timer, +\* retry through the queue, refund on cancel). 55M states, 13 min on fede. +\* The reservation controls run here. +SPECIFICATION Spec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = TRUE + StartDeadlines = TRUE + FanoutSteps = FALSE + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT FifoOrder +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped +PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorBackoffDeep.tla b/verification/tla/MCDynamicSupervisorBackoffDeep.tla new file mode 100644 index 00000000..a8e4c1ff --- /dev/null +++ b/verification/tla/MCDynamicSupervisorBackoffDeep.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorBackoffDeep ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/MCDynamicSupervisorWideDeep.cfg b/verification/tla/MCDynamicSupervisorWideDeep.cfg new file mode 100644 index 00000000..8755422a --- /dev/null +++ b/verification/tla/MCDynamicSupervisorWideDeep.cfg @@ -0,0 +1,50 @@ +\* Weekly (--deep): the broad instance, two terminate calls (by id and by slot +\* fenced with expect, both naming any generation seen), budget 1, no backoff, +\* every outcome. 47M states, 10 min on fede: too slow per PR. The safety +\* negative controls run here (a counterexample is found in seconds whatever +\* the instance size). +SPECIFICATION Spec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 1 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none", "Normal"} + CleanupFails = {FALSE, TRUE} + TermCalls <- MCCalls2 + CallSlot <- MCCallSlot2 + CallKind <- MCCallKind2 + BackoffOn = FALSE + StartDeadlines = TRUE + FanoutSteps = FALSE + Fault = "None" +INVARIANT TypeOK +INVARIANT AtMostOneGeneration +INVARIANT PublishedIsLive +INVARIANT NoChildOutlivesSupervisor +INVARIANT MaxChildrenBound +INVARIANT SerialConstruction +INVARIANT FifoOrder +INVARIANT TemporaryNeverRestarted +INVARIANT TerminatedNeverRestarted +INVARIANT NoGenerationAfterClose +INVARIANT NoPublishAfterStop +INVARIANT NoStartAfterShutdown +INVARIANT RestartFollowsPolicy +INVARIANT FirstStartNeverRetried +INVARIANT FailedAfterCleanup +INVARIANT FailureCauseKept +INVARIANT ChargedRestarts +INVARIANT ReservationsMatch +INVARIANT BudgetBounded +INVARIANT ExhaustionCloses +INVARIANT StuckIsolated +INVARIANT ConcurrentShutdown +INVARIANT OkMeansReady +INVARIANT TermResultsSound +INVARIANT RemovedAfterFinalize +INVARIANT QueuedTermStopped +PROPERTY NeverReopens +PROPERTY TermResultStable diff --git a/verification/tla/MCDynamicSupervisorWideDeep.tla b/verification/tla/MCDynamicSupervisorWideDeep.tla new file mode 100644 index 00000000..2dfd17ba --- /dev/null +++ b/verification/tla/MCDynamicSupervisorWideDeep.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorWideDeep ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index fc4e6ad7..9d4132ac 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -227,44 +227,44 @@ def named_temporal(module, fault, prop, tlc, constants=None): # liveness property of DynamicSupervisor.tla has at least one. Run one at a # time (parallel TLC runs race on the standard modules TLC extracts to /tmp). DYNAMIC_CONTROLS = [ - ("MCDynamicSupervisor", "IgnoreMax", "MaxChildrenBound"), - ("MCDynamicSupervisor", "UncountedRestart", "MaxChildrenBound"), - ("MCDynamicSupervisor", "RestartTemporary", "TemporaryNeverRestarted"), - ("MCDynamicSupervisor", "RestartTerminated", "TerminatedNeverRestarted"), - ("MCDynamicSupervisor", "AcceptDuringShutdown", "NoStartAfterShutdown"), - ("MCDynamicSupervisorBackoff", "RestartAfterClose", "NoGenerationAfterClose"), - ("MCDynamicSupervisor", "NoEscalate", "ExhaustionCloses"), - ("MCDynamicSupervisor", "EarlyDone", "NoChildOutlivesSupervisor"), - ("MCDynamicSupervisor", "SkipSignal", "ConcurrentShutdown"), - ("MCDynamicSupervisor", "ConcurrentStart", "SerialConstruction", {"MaxChildren": "2"}), - ("MCDynamicSupervisor", "LaterOp", "FifoOrder"), - ("MCDynamicSupervisor", "StuckFailsAll", "StuckIsolated"), - ("MCDynamicSupervisor", "MisclassifyStuck", "StuckIsolated"), - ("MCDynamicSupervisor", "PublishAfterClose", "NoPublishAfterStop"), - ("MCDynamicSupervisor", "PublishAfterExit", "PublishedIsLive"), - ("MCDynamicSupervisor", "OverlapGeneration", "AtMostOneGeneration"), - ("MCDynamicSupervisor", "PublishBeforeReady", "PublishedIsLive"), - ("MCDynamicSupervisor", "PrematureOk", "OkMeansReady"), - ("MCDynamicSupervisor", "UnchargedRetry", "ChargedRestarts"), - ("MCDynamicSupervisor", "ChargeFirstStart", "ChargedRestarts"), - ("MCDynamicSupervisor", "TransientNormalRestart", "RestartFollowsPolicy"), - ("MCDynamicSupervisor", "TransientNormalRestart", "RestartFollowsPolicy", {"MaxRestarts": "0", "MaxExpire": "0"}), - ("MCDynamicSupervisor", "PermanentRetires", "RestartFollowsPolicy"), - ("MCDynamicSupervisor", "RetryFirstStart", "FirstStartNeverRetried"), - ("MCDynamicSupervisor", "EarlyFailedReply", "FailedAfterCleanup"), - ("MCDynamicSupervisor", "EarlyRemove", "FailedAfterCleanup"), - ("MCDynamicSupervisor", "LateFailedOverwrite", "FailureCauseKept"), - ("MCDynamicSupervisorBackoff", "NoRefund", "ReservationsMatch"), - ("MCDynamicSupervisorBackoff", "ExpireReserved", "ReservationsMatch"), - ("MCDynamicSupervisor", "OverReserve", "BudgetBounded"), - ("MCDynamicSupervisor", "TermReplyEarly", "TermResultsSound"), - ("MCDynamicSupervisor", "SlotLookup", "TermResultsSound"), - ("MCDynamicSupervisor", "QueuedTermOk", "QueuedTermStopped"), - ("MCDynamicSupervisor", "StuckResultLost", "TermResultsSound"), - ("MCDynamicSupervisor", "IgnoreExpect", "TermResultsSound"), - ("MCDynamicSupervisor", "EarlyFree", "RemovedAfterFinalize"), - ("MCDynamicSupervisor", "IgnoreFreesEarly", "RemovedAfterFinalize"), - ("MCDynamicSupervisor", "IgnoreAfterClose", "NoStartAfterShutdown"), + ("MCDynamicSupervisorWideDeep", "IgnoreMax", "MaxChildrenBound"), + ("MCDynamicSupervisorWideDeep", "UncountedRestart", "MaxChildrenBound"), + ("MCDynamicSupervisorWideDeep", "RestartTemporary", "TemporaryNeverRestarted"), + ("MCDynamicSupervisorWideDeep", "RestartTerminated", "TerminatedNeverRestarted"), + ("MCDynamicSupervisorWideDeep", "AcceptDuringShutdown", "NoStartAfterShutdown"), + ("MCDynamicSupervisorBackoffDeep", "RestartAfterClose", "NoGenerationAfterClose"), + ("MCDynamicSupervisorWideDeep", "NoEscalate", "ExhaustionCloses"), + ("MCDynamicSupervisorWideDeep", "EarlyDone", "NoChildOutlivesSupervisor"), + ("MCDynamicSupervisorWideDeep", "SkipSignal", "ConcurrentShutdown"), + ("MCDynamicSupervisorWideDeep", "ConcurrentStart", "SerialConstruction", {"MaxChildren": "2"}), + ("MCDynamicSupervisorWideDeep", "LaterOp", "FifoOrder"), + ("MCDynamicSupervisorWideDeep", "StuckFailsAll", "StuckIsolated"), + ("MCDynamicSupervisorWideDeep", "MisclassifyStuck", "StuckIsolated"), + ("MCDynamicSupervisorWideDeep", "PublishAfterClose", "NoPublishAfterStop"), + ("MCDynamicSupervisorWideDeep", "PublishAfterExit", "PublishedIsLive"), + ("MCDynamicSupervisorWideDeep", "OverlapGeneration", "AtMostOneGeneration"), + ("MCDynamicSupervisorWideDeep", "PublishBeforeReady", "PublishedIsLive"), + ("MCDynamicSupervisorWideDeep", "PrematureOk", "OkMeansReady"), + ("MCDynamicSupervisorWideDeep", "UnchargedRetry", "ChargedRestarts"), + ("MCDynamicSupervisorWideDeep", "ChargeFirstStart", "ChargedRestarts"), + ("MCDynamicSupervisorWideDeep", "TransientNormalRestart", "RestartFollowsPolicy"), + ("MCDynamicSupervisorWideDeep", "TransientNormalRestart", "RestartFollowsPolicy", {"MaxRestarts": "0", "MaxExpire": "0"}), + ("MCDynamicSupervisorWideDeep", "PermanentRetires", "RestartFollowsPolicy"), + ("MCDynamicSupervisorWideDeep", "RetryFirstStart", "FirstStartNeverRetried"), + ("MCDynamicSupervisorWideDeep", "EarlyFailedReply", "FailedAfterCleanup"), + ("MCDynamicSupervisorWideDeep", "EarlyRemove", "FailedAfterCleanup"), + ("MCDynamicSupervisorWideDeep", "LateFailedOverwrite", "FailureCauseKept"), + ("MCDynamicSupervisorBackoffDeep", "NoRefund", "ReservationsMatch"), + ("MCDynamicSupervisorBackoffDeep", "ExpireReserved", "ReservationsMatch"), + ("MCDynamicSupervisorWideDeep", "OverReserve", "BudgetBounded"), + ("MCDynamicSupervisorWideDeep", "TermReplyEarly", "TermResultsSound"), + ("MCDynamicSupervisorWideDeep", "SlotLookup", "TermResultsSound"), + ("MCDynamicSupervisorWideDeep", "QueuedTermOk", "QueuedTermStopped"), + ("MCDynamicSupervisorWideDeep", "StuckResultLost", "TermResultsSound"), + ("MCDynamicSupervisorWideDeep", "IgnoreExpect", "TermResultsSound"), + ("MCDynamicSupervisorWideDeep", "EarlyFree", "RemovedAfterFinalize"), + ("MCDynamicSupervisorWideDeep", "IgnoreFreesEarly", "RemovedAfterFinalize"), + ("MCDynamicSupervisorWideDeep", "IgnoreAfterClose", "NoStartAfterShutdown"), ] DYNAMIC_TEMPORAL_CONTROLS = [ # MCDynamicSupervisorAbort has no startup deadlines and passes unmutated. From c9c6457103ecfc550e678c41d2f7f61ef9f0e787 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:38:12 -0300 Subject: [PATCH 49/52] verify: slot finalization has its own deadline even after an earlier stuck generation (Callers liveness counterexample) --- verification/tla/DynamicSupervisor.tla | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 9f4b0622..313ce064 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -114,7 +114,7 @@ SlotInit == [cs |-> "idle", gen |-> 0, alive |-> {}, pub |-> 0, lastPub |-> 0, reason |-> "none", cfail |-> FALSE, readyGen |-> FALSE, everReady |-> FALSE, result |-> "none", cause |-> "none", term |-> FALSE, stopSig |-> FALSE, stuck |-> "none", - retryQ |-> FALSE, ign |-> FALSE] + retryQ |-> FALSE, ign |-> FALSE, fstuck |-> FALSE] \* terminate call: st \in none, queued, pending, ok, stuck, notfound, stopped; \* g = the actor (generation) the caller named. CallInit == [st |-> "none", g |-> 0] @@ -393,9 +393,12 @@ CleanupDone(s, f) == \* "closing" during shutdown; otherwise "retire" (fails only the child). CleanupStuck(s) == /\ Fault # "NoDeadline" - /\ sl[s].stuck = "none" - /\ \/ sl[s].cs \in {"cleanup", "finalizing"} - \/ sl[s].cs = "running" /\ sl[s].stopSig + \* generation deadlines once per slot life; the finalization deadline has + \* its own flag, so a slot that was stuck before and then reached + \* finalization can still miss (and report) that deadline + /\ \/ sl[s].stuck = "none" /\ sl[s].cs = "cleanup" + \/ sl[s].stuck = "none" /\ sl[s].cs = "running" /\ sl[s].stopSig + \/ sl[s].cs = "finalizing" /\ ~sl[s].fstuck /\ LET restartable == /\ sl[s].everReady /\ ~sl[s].term /\ sl[s].cs = "cleanup" /\ \/ DesignAllowsRestart(mode[s], sl[s].reason, FALSE) @@ -404,6 +407,7 @@ CleanupStuck(s) == ELSE IF restartable \/ (Fault = "MisclassifyStuck" /\ ~sl[s].term) THEN "restart" ELSE "retire" IN /\ sl' = [sl EXCEPT ![s].cs = "stuck", ![s].stuck = kind, ![s].pub = 0, + ![s].fstuck = sl[s].fstuck \/ sl[s].cs = "finalizing", ![s].result = IF sl[s].result = "pending" THEN sl[s].cause ELSE sl[s].result] /\ IF kind # "retire" \/ Fault = "StuckFailsAll" From 0955358dd9b2337fdb9cce5405737bcce35064cb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:38:26 -0300 Subject: [PATCH 50/52] verify: parenthesise fstuck update --- verification/tla/DynamicSupervisor.tla | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 313ce064..286dddab 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -407,7 +407,7 @@ CleanupStuck(s) == ELSE IF restartable \/ (Fault = "MisclassifyStuck" /\ ~sl[s].term) THEN "restart" ELSE "retire" IN /\ sl' = [sl EXCEPT ![s].cs = "stuck", ![s].stuck = kind, ![s].pub = 0, - ![s].fstuck = sl[s].fstuck \/ sl[s].cs = "finalizing", + ![s].fstuck = (sl[s].fstuck \/ sl[s].cs = "finalizing"), ![s].result = IF sl[s].result = "pending" THEN sl[s].cause ELSE sl[s].result] /\ IF kind # "retire" \/ Fault = "StuckFailsAll" From 1dc47e18ee5fde767f93aeaafdb7248bea8aea75 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:42:02 -0300 Subject: [PATCH 51/52] verify: a stuck finalization that ends completes finalization (second Callers counterexample) --- verification/tla/DynamicSupervisor.tla | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/verification/tla/DynamicSupervisor.tla b/verification/tla/DynamicSupervisor.tla index 286dddab..49d5f4cd 100644 --- a/verification/tla/DynamicSupervisor.tla +++ b/verification/tla/DynamicSupervisor.tla @@ -418,10 +418,12 @@ CleanupStuck(s) == /\ op' = IF op.s = s /\ (op.k # "term" \/ Fault # "StuckResultLost") THEN Idle ELSE op /\ UNCHANGED <> -\* A stuck child (or finalization) finally ended; finalization follows. +\* A stuck child finally ended: finalization follows. A stuck finalization +\* that ends is finalization done: the slot is removed. StuckDone(s) == /\ sl[s].cs = "stuck" - /\ sl' = [sl EXCEPT ![s].cs = "finalizing", ![s].alive = {}] + /\ sl' = [sl EXCEPT ![s].cs = IF sl[s].fstuck THEN "removed" ELSE "finalizing", + ![s].alive = {}] /\ Unchanged1 \* Slot finalization ended: retained arguments dropped, then capacity and @@ -601,7 +603,8 @@ GhostUpdate == (tc[c].st = "pending" /\ sl[CallSlot[c]].stuck = "none" /\ sl'[CallSlot[c]].stuck # "none")] /\ freedEarly' = [s \in Slots |-> freedEarly[s] \/ - (sl'[s].cs = "removed" /\ sl[s].cs \notin {"finalizing", "removed"})] + (sl'[s].cs = "removed" /\ sl[s].cs \notin {"finalizing", "removed"} + /\ ~(sl[s].cs = "stuck" /\ sl[s].fstuck))] /\ UNCHANGED mode SlotStep(s) == From d75a596c717423ee5017cbc12fb9a93dc44c109a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:29:38 -0300 Subject: [PATCH 52/52] verify: full caller-liveness instance weekly (CallersDeep); smaller per-PR Callers --- .github/workflows/verification.yml | 1 + .../tla/MCDynamicSupervisorCallers.cfg | 8 +++--- .../tla/MCDynamicSupervisorCallersDeep.cfg | 26 +++++++++++++++++++ .../tla/MCDynamicSupervisorCallersDeep.tla | 3 +++ verification/tla/negative_checks.py | 6 ++--- 5 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 verification/tla/MCDynamicSupervisorCallersDeep.cfg create mode 100644 verification/tla/MCDynamicSupervisorCallersDeep.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 44f3e4dc..f3fa506e 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -121,6 +121,7 @@ jobs: - MCDynamicSupervisorDeep - MCDynamicSupervisorWideDeep - MCDynamicSupervisorBackoffDeep + - MCDynamicSupervisorCallersDeep - MCStartDeep steps: - uses: actions/checkout@v4 diff --git a/verification/tla/MCDynamicSupervisorCallers.cfg b/verification/tla/MCDynamicSupervisorCallers.cfg index 624ba83e..3665eca7 100644 --- a/verification/tla/MCDynamicSupervisorCallers.cfg +++ b/verification/tla/MCDynamicSupervisorCallers.cfg @@ -1,11 +1,9 @@ -\* Per PR, liveness under CallerSpec (no user cleanup ever assumed to -\* finish, only deadlines fire): every start_child and terminate_child is -\* answered, and the fan-out, as separate steps (refinement of the atomic -\* SupSignal), signals every admitted child even if siblings never finish. +\* Per PR: as MCDynamicSupervisorCallersDeep with two restart types +\* (Permanent, Temporary) instead of three. SPECIFICATION CallerSpec CONSTANTS N = 2 - ModeSet = {"Permanent", "Transient", "Temporary"} + ModeSet = {"Permanent", "Temporary"} MaxChildren = 2 MaxRestarts = 1 MaxExpire = 1 diff --git a/verification/tla/MCDynamicSupervisorCallersDeep.cfg b/verification/tla/MCDynamicSupervisorCallersDeep.cfg new file mode 100644 index 00000000..df08b8fd --- /dev/null +++ b/verification/tla/MCDynamicSupervisorCallersDeep.cfg @@ -0,0 +1,26 @@ +\* Weekly (--deep): liveness under CallerSpec (no user cleanup ever assumed +\* to finish, only deadlines fire), every pair of restart types: every +\* start_child and terminate_child is answered, and the fan-out, as separate +\* steps, signals every admitted child even if siblings never finish. +\* 11.7M states, 47 min on a loaded fede. The liveness controls run here. +SPECIFICATION CallerSpec +CONSTANTS + N = 2 + ModeSet = {"Permanent", "Transient", "Temporary"} + MaxChildren = 2 + MaxRestarts = 1 + MaxExpire = 1 + SelfExits = {"Normal", "Panic"} + StartFails = {"none"} + CleanupFails = {FALSE} + TermCalls <- MCCalls1 + CallSlot <- MCCallSlot1 + CallKind <- MCCallKind1 + BackoffOn = FALSE + StartDeadlines = TRUE + FanoutSteps = TRUE + Fault = "None" +INVARIANT TypeOK +PROPERTY StartAnswered +PROPERTY TerminateAnswered +PROPERTY FanoutCompletes diff --git a/verification/tla/MCDynamicSupervisorCallersDeep.tla b/verification/tla/MCDynamicSupervisorCallersDeep.tla new file mode 100644 index 00000000..1d062954 --- /dev/null +++ b/verification/tla/MCDynamicSupervisorCallersDeep.tla @@ -0,0 +1,3 @@ +---- MODULE MCDynamicSupervisorCallersDeep ---- +EXTENDS MCDynamicSupervisor +==== diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index c01a1f05..df60ccb6 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -269,9 +269,9 @@ def named_temporal(module, fault, prop, tlc, constants=None): DYNAMIC_TEMPORAL_CONTROLS = [ # MCDynamicSupervisorAbort has no startup deadlines and passes unmutated. ("MCDynamicSupervisorAbort", "NoAbort", "ShutdownAnswersStarts"), - ("MCDynamicSupervisorCallers", "NoDeadline", "StartAnswered"), - ("MCDynamicSupervisorCallers", "TermLost", "TerminateAnswered"), - ("MCDynamicSupervisorCallers", "Sequential", "FanoutCompletes"), + ("MCDynamicSupervisorCallersDeep", "NoDeadline", "StartAnswered"), + ("MCDynamicSupervisorCallersDeep", "TermLost", "TerminateAnswered"), + ("MCDynamicSupervisorCallersDeep", "Sequential", "FanoutCompletes"), ("MCDynamicSupervisorLive", "StuckBackoff", "ClosedLeadsToDone"), ("MCDynamicSupervisorLive", "LostRestart", "RecoveryProgress"), ]