From 90029ed5d7c8adfdd0ea2c6a211bedc79c02713b Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 23 Oct 2026 09:11:01 +0000 Subject: [PATCH 01/36] fix(controller): end a message that quotes the engine with one full stop The engine writes what a person reads as sentences, each ending with a full stop: a refusal's message, detail and hint, and its 503 sentences. An error of Go's or of the operator's own has none. Some conditions and events end by quoting one of those, and each site decided on its own whether to add a full stop, so they disagreed: - Sites that added one ended in two whenever the quote was the engine's. These are the EngineUnavailable condition ("...or ask another node.."), SecretsSynced after a refused store write, a pause that may not have landed, ApplyFailed's account and its unreadable fleet record, and the Reactivated warning of a reload the operator could not read back. The last quotes mid-sentence, so it ended in ".. It takes". - Sites that added none ended with no stop whenever the quote had none: Go's errors at EngineHealthy, IdentityClaimed and a Seat's Ready when the health or the fleet could not be read, and at a pause refusal the engine's account of the refused call when it ended without one. - A Paused message quotes the reason the person who paused the seat gave. That reason is their own words, so "Hold the deploy." became "Hold the deploy.. It starts no new turn". There is now one rule, ended: add a full stop where the text has none, and leave text that already ends with ".", "!" or "?", or with a parenthesis that closes on one, as it is. The first half is the rule the engine ends its own details and hints by (httpjson.Sentence). The second is for the client's quote of a refusal, which puts the engine's hint in parentheses after its message: the hint is a sentence with its own full stop, so a message quoting one already ends inside them, and a stop after them made it "....).". sentences now uses the rule too. Every site above ends through it, and so does each message that ended with a list or a render finding plus a "." of its own, so the package has one way to end a sentence. TestQuotedEndingsGoThroughEnded walks the package and refuses a message that ends with `+ "."` or with a format ending ": %s." or ": %v.", so a new site can't add the old shapes back. Its comment says what it can't see: a quote in the middle of a message, which the tests beside it hold one by one. TestEndedEndsASentenceOnce covers the rule, a closing parenthesis included. These cover the sites, each with a quote that ends in a full stop and one that doesn't, and the conditions' test with a quote that ends in the engine's hint too (the 503 a starting node answers): - TestAConditionThatQuotesAnErrorEndsItOnce - TestAFailedApplyAccountEndsWhatItQuotesOnce - TestASyncThatFailsEndsWhatItQuotesOnce - TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce - TestAPauseVerdictEndsWhatItQuotesOnce - TestAPauseReasonIsQuotedAsWrittenOnce Each site test and the walk fail against the sites as they were, with ended defined beside them, and pass with this change; the conditions' hint case and the rule's own fail with ended blind to the parenthesis. The fake engines gained what those tests need: a /fleet status, a store and a document read that refuse as an engine that can't decide who is asking does (503 identity_unavailable, its request guard's answer), and pause tools that answer an unknown outcome. Their refusal now carries the engine's words, not a sentence the engine never wrote. Signed-off-by: Ali Yousefian --- internal/controller/company_runtime.go | 5 +- internal/controller/config_apply.go | 6 +- internal/controller/config_apply_test.go | 8 +- internal/controller/config_fleet.go | 6 +- internal/controller/config_optional.go | 4 +- internal/controller/config_parts.go | 36 +++- internal/controller/config_pause.go | 25 ++- internal/controller/config_pause_test.go | 91 ++++++-- internal/controller/config_run.go | 23 +- internal/controller/config_serving.go | 6 +- internal/controller/config_status.go | 2 +- internal/controller/config_withdraw.go | 15 +- internal/controller/config_withdraw_test.go | 49 ++++- internal/controller/sentence_test.go | 228 ++++++++++++++++++++ 14 files changed, 433 insertions(+), 71 deletions(-) create mode 100644 internal/controller/sentence_test.go diff --git a/internal/controller/company_runtime.go b/internal/controller/company_runtime.go index cf83b945..f5f3c292 100644 --- a/internal/controller/company_runtime.go +++ b/internal/controller/company_runtime.go @@ -177,7 +177,7 @@ func (r *CompanyReconciler) reconcileWorkload( } if len(in.missing) > 0 { return holdBack(crewletaiv1alpha1.ReasonSecretNotFound, - "The engine's pods cannot be given the values they read from Secrets: "+strings.Join(in.missing, "; ")+".") + ended("The engine's pods cannot be given the values they read from Secrets: "+strings.Join(in.missing, "; "))) } return r.runWorkload(ctx, c, class, in, existing, pools) } @@ -318,7 +318,8 @@ func (r *CompanyReconciler) runWorkload( } return suspendedCondition(""), nil case refusal != nil: - return runtimeCondition(metav1.ConditionFalse, refusal.reason, refusal.message+" The data pods keep running "+image+"."), nil + return runtimeCondition(metav1.ConditionFalse, refusal.reason, + refusal.message+" "+ended("The data pods keep running "+image)), nil case progress.ready: return runtimeCondition(metav1.ConditionTrue, crewletaiv1alpha1.ReasonPodsReady, progress.message), nil default: diff --git a/internal/controller/config_apply.go b/internal/controller/config_apply.go index c0ad28f9..5d786bcf 100644 --- a/internal/controller/config_apply.go +++ b/internal/controller/config_apply.go @@ -184,8 +184,8 @@ func (run *configRun) failureAccount(ctx context.Context, eng *engine.Client, he health.Status, health.Posture, health.AppliedEpoch) fleet, err := run.fleetOf(ctx, eng) if err != nil { - return fmt.Sprintf("The engine reports that it did not apply the activated epoch, and its record of the attempt "+ - "could not be read: %v.", err) + runs + return ended(fmt.Sprintf("The engine reports that it did not apply the activated epoch, and its record of the attempt "+ + "could not be read: %v", err)) + runs } record := fleet.LastApply() if record.Serving != "" { @@ -197,7 +197,7 @@ func (run *configRun) failureAccount(ctx context.Context, eng *engine.Client, he if account == "" { account = "the engine gave no account of the failure" } - return fmt.Sprintf("The engine failed to apply epoch %d (revision %s): %s.", record.Epoch, record.RevisionID, account) + runs + return ended(fmt.Sprintf("The engine failed to apply epoch %d (revision %s): %s", record.Epoch, record.RevisionID, account)) + runs } epoch := max(fleet.TargetEpoch, target) return fmt.Sprintf("The engine reports that it did not apply epoch %d and records no failed attempt at it: its apply "+ diff --git a/internal/controller/config_apply_test.go b/internal/controller/config_apply_test.go index 75c23f70..2d1549d4 100644 --- a/internal/controller/config_apply_test.go +++ b/internal/controller/config_apply_test.go @@ -23,10 +23,11 @@ import ( // never puts the real engine in. The real engine's ordinary path, an apply // that reaches ConfigApplied=True, is the envtest suite's (config_test.go). -// fakeEngine answers /health and /fleet with the bodies it holds, and counts -// the /fleet reads. +// fakeEngine answers /health and /fleet with the bodies it holds, /fleet +// with fleetStatus where it is set, and counts the /fleet reads. type fakeEngine struct { health, fleet string + fleetStatus int fleetReads int } @@ -39,6 +40,9 @@ func (f *fakeEngine) client(t *testing.T) *engine.Client { _, _ = w.Write([]byte(f.health)) case fleetRoute: f.fleetReads++ + if f.fleetStatus != 0 { + w.WriteHeader(f.fleetStatus) + } _, _ = w.Write([]byte(f.fleet)) default: http.NotFound(w, r) diff --git a/internal/controller/config_fleet.go b/internal/controller/config_fleet.go index 5228ce54..1d8722bf 100644 --- a/internal/controller/config_fleet.go +++ b/internal/controller/config_fleet.go @@ -153,7 +153,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { setCondition(conditions, gen, crewletaiv1alpha1.ConditionReady, false, crewletaiv1alpha1.ReasonSuspended, fmt.Sprintf("Company %s is suspended: no engine node runs its seats, and their messages wait.", run.company.Name)) case p.executor != nil && p.executor.Reason != "": - message := p.executor.Field + ": " + p.executor.Message + "." + message := ended(p.executor.Field + ": " + p.executor.Message) lease, held := engine.SeatLease{}, false if run.fleet != nil { lease, held = run.leaseOf(handle) @@ -174,7 +174,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { s.Status.Placement.Node = lease.Node message := heldMessage(lease, workload) if p.executor != nil { - message += " It " + p.executor.Message + "." + message += " " + ended("It "+p.executor.Message) } setCondition(conditions, gen, crewletaiv1alpha1.ConditionReady, true, crewletaiv1alpha1.ReasonLeaseHeld, message) return @@ -192,7 +192,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { s.Status.Placement.Node = p.base.(*crewletaiv1alpha1.Seat).Status.Placement.Node meta.SetStatusCondition(conditions, metav1.Condition{ Type: crewletaiv1alpha1.ConditionReady, Status: metav1.ConditionUnknown, Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: bounded("The engine's fleet, which says which node holds the seat, could not be read: " + run.fleetErr.Error()), + Message: bounded(ended("The engine's fleet, which says which node holds the seat, could not be read: " + run.fleetErr.Error())), ObservedGeneration: gen, }) default: diff --git a/internal/controller/config_optional.go b/internal/controller/config_optional.go index 6d787214..33701eed 100644 --- a/internal/controller/config_optional.go +++ b/internal/controller/config_optional.go @@ -214,6 +214,6 @@ func (run *configRun) leftOutLines(values []leftOutValue) []string { // leftOutMessage is a part's ResolvedRefs=True message when it gives values // it lets go without whose Secret or key does not exist or holds nothing. func leftOutMessage(leftOut []string) string { - return "Every object it references exists, and every Secret key but these, each a value it gives under a key " + - "an optionalKeys list lets go without it: " + strings.Join(leftOut, "; ") + "." + return ended("Every object it references exists, and every Secret key but these, each a value it gives under a key " + + "an optionalKeys list lets go without it: " + strings.Join(leftOut, "; ")) } diff --git a/internal/controller/config_parts.go b/internal/controller/config_parts.go index 62903a13..2001088a 100644 --- a/internal/controller/config_parts.go +++ b/internal/controller/config_parts.go @@ -231,15 +231,39 @@ func bounded(message string) string { return message[:cut] + ellipsis } -// sentences joins findings into one message, each ending with a period. +// sentences joins findings into one message, each ending as ended ends it. func sentences(findings []string) string { out := make([]string, len(findings)) for i, f := range findings { - f = capitalize(strings.TrimSpace(f)) - if !strings.HasSuffix(f, ".") { - f += "." - } - out[i] = f + out[i] = ended(capitalize(strings.TrimSpace(f))) } return strings.Join(out, " ") } + +// ended is text as the end of a sentence: with a full stop added where it +// has none, and as it is where it already ends with one, or with "!" or "?", +// or with a parenthesis that closes on one. +// +// THE ONE RULE FOR A MESSAGE THAT ENDS IN WORDS IT QUOTES. A condition or an +// event often ends with what somebody else said: the engine's refusal, its +// account of an apply that failed, or an error that carries either. The +// engine writes those as sentences, each with its full stop, while an error of +// the operator's own or of Go's has none. So a message that added a full stop +// to what it quoted ended "...won't change that.." whenever the quote was the +// engine's, and one that added none ended without a stop whenever it was not. +// The engine ends its own details and hints by the same rule (crewlet's +// internal/api/httpjson, Sentence). The parenthesis is the client's: an +// *engine.APIError quotes the engine's hint in parentheses after its message, +// and the hint is a sentence with its own full stop, so a message quoting one +// already ends inside them and a stop after them would end it twice ("….)."). +// TestQuotedEndingsGoThroughEnded holds every message of the package to it. +func ended(text string) string { + text = strings.TrimSpace(text) + if text == "" { + return text + } + if body := strings.TrimSuffix(text, ")"); body != "" && strings.ContainsRune(".!?", rune(body[len(body)-1])) { + return text + } + return text + "." +} diff --git a/internal/controller/config_pause.go b/internal/controller/config_pause.go index c4163c4b..f93a1b11 100644 --- a/internal/controller/config_pause.go +++ b/internal/controller/config_pause.go @@ -159,8 +159,8 @@ func (run *configRun) syncPause( case errors.Is(err, engine.ErrOutcomeUnknown): return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseUnknown, - message: fmt.Sprintf("The engine may or may not have recorded the operator's %s of the seat; the operator asks again in %s: %s.", - changeWord(*want), engineRetryInterval, err), + message: ended(fmt.Sprintf("The engine may or may not have recorded the operator's %s of the seat; the operator asks again in %s: %s", + changeWord(*want), engineRetryInterval, err)), }, engineRetryInterval case errors.As(err, &refused): // A refusal is often contention (the record lost to other writers @@ -168,15 +168,15 @@ func (run *configRun) syncPause( // for is asked for again soon rather than at the next resync. return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseRefused, - message: fmt.Sprintf("The engine refused the operator's %s of the seat, which it asks for again in %s: %s", - changeWord(*want), engineRetryInterval, refused.Message), + message: ended(fmt.Sprintf("The engine refused the operator's %s of the seat, which it asks for again in %s: %s", + changeWord(*want), engineRetryInterval, refused.Message)), }, engineRetryInterval case err != nil: logf.FromContext(ctx).V(1).Info("Failed to change a seat's pause", "seat", seat.Name, "error", err.Error()) return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseUnknown, - message: fmt.Sprintf("The operator's %s of the seat may not have reached the engine; it tries again in %s: %s.", - changeWord(*want), engineRetryInterval, err), + message: ended(fmt.Sprintf("The operator's %s of the seat may not have reached the engine; it tries again in %s: %s", + changeWord(*want), engineRetryInterval, err)), }, engineRetryInterval } if answer.Changed { @@ -221,16 +221,19 @@ func reported(state engine.SeatPause) pauseVerdict { message: "The engine runs the seat: nobody has it paused.", } } - message := fmt.Sprintf("The engine holds the seat paused since %s, paused by %s", state.At.Format(time.RFC3339), state.By) + held := fmt.Sprintf("The engine holds the seat paused since %s, paused by %s", state.At.Format(time.RFC3339), state.By) if state.Reason != "" { - message += ": " + state.Reason + // The reason is whoever paused the seat's own words, which may end + // a sentence of their own. + held += ": " + state.Reason } - message += ". It starts no new turn and its mail waits on its inbox" + waits := "It starts no new turn and its mail waits on its inbox" if state.StopRunning { - message += "; its running work was stopped: the turn it was on, and its executor run at once" + waits += "; its running work was stopped: the turn it was on, and its executor run at once" } return pauseVerdict{ - status: metav1.ConditionTrue, reason: crewletaiv1alpha1.ReasonSeatPaused, message: message + ".", pause: state, + status: metav1.ConditionTrue, reason: crewletaiv1alpha1.ReasonSeatPaused, + message: ended(held) + " " + ended(waits), pause: state, } } diff --git a/internal/controller/config_pause_test.go b/internal/controller/config_pause_test.go index ac9fd949..5552a1f1 100644 --- a/internal/controller/config_pause_test.go +++ b/internal/controller/config_pause_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -30,12 +31,23 @@ type pauseEngine struct { roster map[string]bool rosterDown bool // record is whether the coordination store holds each seat paused, which - // the tools answer from and change; refuse makes them refuse. - record map[string]bool - refuse bool - calls []string + // the tools answer from and change; refusal, when set, makes them refuse + // with it as the engine's account, and unknown makes them answer that the + // write may or may not have landed, with it as the engine's detail. + record map[string]bool + refusal, unknown string + calls []string } +// The engine's words for a pause call that lost the record to other writers +// every time it tried, and for one whose write got no answer (crewlet's +// internal/agent/builtin, seatpause.go). +const ( + pauseLostRaces = "pause_seat lost 4 races in a row for ana to other changes to its pause. " + + "Read the seat again before retrying." + pauseMayHaveLanded = "the write may or may not have landed; reading the seat again says which" +) + func (f *pauseEngine) client(t *testing.T) *engine.Client { t.Helper() server := mcp.NewServer(&mcp.Implementation{Name: "fake", Version: "1"}, nil) @@ -51,16 +63,22 @@ func (f *pauseEngine) client(t *testing.T) *engine.Client { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, name+" "+args.Handle) - if f.refuse { + if f.refusal != "" { return &mcp.CallToolResult{IsError: true, Content: []mcp.Content{&mcp.TextContent{ - Text: "lost the record to other writers four times in a row", + Text: f.refusal, }}}, nil } - changed := f.record[args.Handle] != paused - f.record[args.Handle] = paused - answer := map[string]any{"handle": args.Handle, "outcome": "applied", "paused": paused, "changed": changed} - if paused { - answer["paused_by"], answer["paused_at"] = engine.TokenLogin(render.TokenOperator), "2026-10-09T08:00:00Z" + answer := map[string]any{"handle": args.Handle} + switch { + case f.unknown != "": + answer["outcome"], answer["detail"] = "unknown", f.unknown + default: + answer["outcome"], answer["paused"] = "applied", paused + answer["changed"] = f.record[args.Handle] != paused + f.record[args.Handle] = paused + if paused { + answer["paused_by"], answer["paused_at"] = engine.TokenLogin(render.TokenOperator), "2026-10-09T08:00:00Z" + } } text, err := json.Marshal(answer) if err != nil { @@ -145,7 +163,8 @@ func TestAPauseThatCannotBeMadeIsAskedForAgainSoon(t *testing.T) { } }) t.Run("the engine refuses the call", func(t *testing.T) { - f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, refuse: true} + f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, + refusal: pauseLostRaces} run, p := pauseRun(new(true)) result := run.syncPauses(context.Background(), f.client(t), ctrl.Result{RequeueAfter: 5 * time.Minute}) if p.pause == nil || p.pause.reason != crewletaiv1alpha1.ReasonPauseRefused { @@ -176,3 +195,51 @@ func TestASeatTheEngineDoesNotRunHasNoPause(t *testing.T) { t.Errorf("the operator called %v for a seat the engine does not run", f.calls) } } + +// TestAPauseVerdictEndsWhatItQuotesOnce holds each pause verdict that quotes +// the engine to one full stop at its end, whether the engine's words end +// with one or not: its refusal is a sentence with its own, and its detail of +// a write that may not have landed is a clause with none, which either could +// become. Each verdict used to add a full stop of its own, or none, whatever +// it quoted, so one of the two shapes ended with two or none. +func TestAPauseVerdictEndsWhatItQuotesOnce(t *testing.T) { + const clause = "the record moved under the call each time it tried" + for _, c := range []struct { + name, refusal, unknown, reason string + }{ + {name: "a refusal the engine ends", refusal: pauseLostRaces, reason: crewletaiv1alpha1.ReasonPauseRefused}, + {name: "a refusal the engine does not end", refusal: clause, reason: crewletaiv1alpha1.ReasonPauseRefused}, + {name: "a detail the engine does not end", unknown: pauseMayHaveLanded, + reason: crewletaiv1alpha1.ReasonPauseUnknown}, + {name: "a detail the engine ends", unknown: "The write may or may not have landed. " + + "Reading the seat again says which.", reason: crewletaiv1alpha1.ReasonPauseUnknown}, + } { + t.Run(c.name, func(t *testing.T) { + f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, + refusal: c.refusal, unknown: c.unknown} + run, p := pauseRun(new(true)) + run.syncPauses(context.Background(), f.client(t), ctrl.Result{}) + if p.pause == nil || p.pause.reason != c.reason { + t.Fatalf("the verdict is %+v, want %s", p.pause, c.reason) + } + if message := p.pause.message; !strings.HasSuffix(message, ".") || strings.HasSuffix(message, "..") { + t.Errorf("the verdict's message does not end with one full stop: %q", message) + } + }) + } +} + +// TestAPauseReasonIsQuotedAsWrittenOnce holds the Paused message to one full +// stop after the reason whoever paused the seat gave, which is their own +// words and may end a sentence of its own, before the operator's next one. +func TestAPauseReasonIsQuotedAsWrittenOnce(t *testing.T) { + at := time.Date(2026, 10, 9, 8, 0, 0, 0, time.UTC) + for _, reason := range []string{"Hold the deploy.", "hold the deploy", ""} { + for _, stop := range []bool{false, true} { + verdict := reported(engine.SeatPause{Paused: true, By: humanJane, At: at, Reason: reason, StopRunning: stop}) + if strings.Contains(verdict.message, "..") || !endsOnce(verdict.message) { + t.Errorf("reason %q, stop %t: the message is %q, want each sentence ended once", reason, stop, verdict.message) + } + } + } +} diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 63b7df17..10533247 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -450,9 +450,9 @@ func (run *configRun) enforce( if notOwned.Controller == nil || !isPreviousCompany(notOwned.Controller, c) { run.secretsSynced = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionSecretsSynced, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonNameConflict, Message: capitalize(notOwned.Error()) + ".", + Reason: crewletaiv1alpha1.ReasonNameConflict, Message: capitalize(ended(notOwned.Error())), } - run.block(crewletaiv1alpha1.ReasonNameConflict, capitalize(notOwned.Error())+".") + run.block(crewletaiv1alpha1.ReasonNameConflict, capitalize(ended(notOwned.Error()))) return ctrl.Result{}, nil } state, err = secretsync.State{}, nil @@ -710,7 +710,7 @@ func (run *configRun) sync( if syncErr != nil { run.secretsSynced = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionSecretsSynced, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: capitalize(syncErr.Error()) + ".", + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: capitalize(ended(syncErr.Error())), } return next, result, syncErr } @@ -750,8 +750,8 @@ func (run *configRun) sync( message += run.retainedSentence() message += run.heldSentence() if len(run.companyLeftOut) > 0 { - message += " The Company's own setup steps give values they let go without: " + - strings.Join(run.leftOutLines(run.companyLeftOut), "; ") + "." + message += " " + ended("The Company's own setup steps give values they let go without: "+ + strings.Join(run.leftOutLines(run.companyLeftOut), "; ")) } if len(run.companyEmpty) > 0 { message += " " + sentences(run.companyEmpty) @@ -1082,8 +1082,8 @@ func (run *configRun) announce(p *part) { } if p.executor != nil { if c := turned(crewletaiv1alpha1.ConditionReady, p.executor.Reason); c != nil { - run.Recorder.Eventf(p.obj, nil, corev1.EventTypeWarning, c.Reason, "CheckExecutor", "%s: %s.", - p.executor.Field, p.executor.Message) + run.Recorder.Eventf(p.obj, nil, corev1.EventTypeWarning, c.Reason, "CheckExecutor", "%s", + ended(p.executor.Field+": "+p.executor.Message)) } } if len(p.unresolved) == 0 { @@ -1238,8 +1238,9 @@ func (run *configRun) block(reason, message string) { func (run *configRun) unavailable(err error) { run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s.", retryAfter(err), err), + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, + Message: ended(fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s", + retryAfter(err), err)), } } @@ -1538,7 +1539,7 @@ func engineHealthy(alarms *engine.Alarms, err error) *metav1.Condition { switch { case err != nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonAlarmsNotReported - c.Message = "The engine's health could not be read: " + err.Error() + c.Message = ended("The engine's health could not be read: " + err.Error()) case alarms == nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonAlarmsNotReported c.Message = "The engine has not evaluated its alarms yet." @@ -1620,7 +1621,7 @@ func identityClaimed(health engine.Health, err error, humanSeats []string, acces switch { case err != nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonIdentityNotReported - c.Message = "The engine's health could not be read: " + err.Error() + c.Message = ended("The engine's health could not be read: " + err.Error()) case health.Identity == engine.IdentityReady: c.Status, c.Reason = metav1.ConditionTrue, crewletaiv1alpha1.ReasonPersonEnrolled c.Message = "A person has joined the company: the engine's identity directory holds someone on a Human seat, " + diff --git a/internal/controller/config_serving.go b/internal/controller/config_serving.go index 8abdcb51..487d2f0c 100644 --- a/internal/controller/config_serving.go +++ b/internal/controller/config_serving.go @@ -396,7 +396,7 @@ func (run *configRun) heldSentence() string { if len(run.serving.unread) > 0 { why = append(why, "a revision the operator does not know of "+strings.Join(run.serving.unread, ", ")) } - return fmt.Sprintf(" It keeps %d %s the revision the engine is to run no longer needs, for %s, until no pod runs or "+ - "boots on a revision that does: %s.", len(held), plural(len(held), "value", "values"), strings.Join(why, "; and "), - strings.Join(held, ", ")) + return " " + ended(fmt.Sprintf("It keeps %d %s the revision the engine is to run no longer needs, for %s, until no pod "+ + "runs or boots on a revision that does: %s", len(held), plural(len(held), "value", "values"), strings.Join(why, "; and "), + strings.Join(held, ", "))) } diff --git a/internal/controller/config_status.go b/internal/controller/config_status.go index 52b0eb5d..f0b68bde 100644 --- a/internal/controller/config_status.go +++ b/internal/controller/config_status.go @@ -169,7 +169,7 @@ func (run *configRun) decideApplied(p *part) { switch { case len(p.drifted) > 0: setCondition(conditions, gen, crewletaiv1alpha1.ConditionApplied, false, crewletaiv1alpha1.ReasonDrifted, - "The engine's document differs from this object's part of the rendered one at: "+strings.Join(p.drifted, ", ")+".") + ended("The engine's document differs from this object's part of the rendered one at: "+strings.Join(p.drifted, ", "))) default: setCondition(conditions, gen, crewletaiv1alpha1.ConditionApplied, true, crewletaiv1alpha1.ReasonInRevision, fmt.Sprintf("In revision %s, which the engine runs.", verdict.revision)) diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index d5cb0e25..0bee9fca 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -534,13 +534,16 @@ func (run *configRun) reactivatedUnconfirmed( active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string, why string, ) { who, action := reactivatedBy(by) - run.event(corev1.EventTypeWarning, "Reactivated", action, - "%s the engine's document again as revision %s (epoch %d), so that the engine reads its secret store without "+ - "%s, whose Secret or key no longer exists and which revision %s lets go without; but the operator cannot confirm "+ - "that revision %s is revision %s's document, as %s. It takes revision %s to read every value as required, and "+ - "one that reads a value it removed so goes without what it starts with it until the Secret exists again.", + // why ends with what the read back met, which may be the engine's own + // sentence: the sentence that quotes it is ended by ended. + activated := ended(fmt.Sprintf("%s the engine's document again as revision %s (epoch %d), so that the engine "+ + "reads its secret store without %s, whose Secret or key no longer exists and which revision %s lets go without; "+ + "but the operator cannot confirm that revision %s is revision %s's document, as %s", who, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", "), active.RevisionID, reloaded.RevisionID, - active.RevisionID, why, reloaded.RevisionID) + active.RevisionID, why)) + run.event(corev1.EventTypeWarning, "Reactivated", action, + "%s It takes revision %s to read every value as required, and one that reads a value it removed so goes without "+ + "what it starts with it until the Secret exists again.", activated, reloaded.RevisionID) } // reactivatedBy is who an activation again is said to be by, and the action diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index ccb705b7..cc7ba036 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -62,7 +62,9 @@ import ( // said); runsNone names nodes beside those that refused the fleet's revision // and run no company, as a seat pod that started on nothing and refused it // does; fleetDown makes GET /fleet unanswerable, and missing names the -// revisions GET /config/revisions does not find. +// revisions GET /config/revisions does not find. configDown and secretsDown +// make it answer a read of the fleet's document, and a write to the store, as +// an engine that cannot decide who is asking does. // // Each node's row says what it starts on as the engine's does: a data node // (dataNode) the revision it runs, and a seat node, whose store is scratch, @@ -74,6 +76,7 @@ type storeEngine struct { revisions []storeRevision store map[string]string refuse bool + configDown, secretsDown bool beforeReload, afterReload func(e *storeEngine) afterApply func(e *storeEngine) reloadAnswer *map[string]any @@ -95,6 +98,29 @@ func dataNode(node string) bool { // otherwise. const storeNode = "acme-data-0" +// down reports whether r is a request configDown or secretsDown makes the +// engine refuse: a read of the fleet's document, or a write to the store, one +// name's or a batch's. +func (e *storeEngine) down(r *http.Request) bool { + if r.Method == http.MethodGet { + return r.URL.Path == configRoute && e.configDown + } + return (r.URL.Path == secretsRoute || strings.HasPrefix(r.URL.Path, secretsRoute+"/")) && e.secretsDown +} + +// cannotDecide is the engine's answer to a request it cannot decide who is +// asking: its request guard's, which cannot read who presented the credential +// (crewlet's internal/api/auth, Caller). The authority table's own unknown +// never meets the operator's tokens on /config or /secrets, since they hold +// admin and admin is decided before the chart. +func cannotDecide(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"identity_unavailable",` + + `"message":"Can't check who you are right now. Try again in a moment."}`)) +} + type storeRevision struct { id, summary string document []byte @@ -128,6 +154,10 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { activated := func(revision storeRevision) { answer(http.StatusCreated, map[string]any{revisionIDField: revision.id, "epoch": len(e.revisions)}) } + if e.down(r) { + cannotDecide(w) + return + } if r.Method == http.MethodGet && e.serveServing(w, r.URL.Path, active) { return } @@ -161,7 +191,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { hook(e) } activated(reloaded) - case r.Method == http.MethodGet && r.URL.Path == "/secrets": + case r.Method == http.MethodGet && r.URL.Path == secretsRoute: listing := make([]map[string]string, 0, len(e.store)) for _, name := range slices.Sorted(maps.Keys(e.store)) { listing = append(listing, map[string]string{"name": name, "updated_by": operatorLogin, "operator_id": operatorLogin}) @@ -182,7 +212,7 @@ func (e *storeEngine) serveSecretWrite( reads := func(name string) bool { return strings.Contains(string(active.document), "${"+name+"}") } name, isSecret := strings.CutPrefix(r.URL.Path, "/secrets/") switch { - case r.Method == http.MethodPut && r.URL.Path == "/secrets": + case r.Method == http.MethodPut && r.URL.Path == secretsRoute: var batch struct { Values map[string]string `json:"values"` } @@ -377,13 +407,14 @@ func (e *storeEngine) activateOther(document []byte) { } // configRoute is the engine's route of the company document, fleetRoute that -// of its fleet, and dataRole and seatsRole the roles of a data pod's node and -// a seat pod's there. +// of its fleet, secretsRoute that of its secret store, and dataRole and +// seatsRole the roles of a data pod's node and a seat pod's there. const ( - configRoute = "/config" - fleetRoute = "/fleet" - dataRole = "data" - seatsRole = "seats" + configRoute = "/config" + fleetRoute = "/fleet" + secretsRoute = "/secrets" + dataRole = "data" + seatsRole = "seats" ) // running is the revision the engine runs when a withdrawal's reconcile diff --git a/internal/controller/sentence_test.go b/internal/controller/sentence_test.go new file mode 100644 index 00000000..03ce4e4b --- /dev/null +++ b/internal/controller/sentence_test.go @@ -0,0 +1,228 @@ +package controller + +import ( + "context" + "errors" + "go/ast" + "go/parser" + "go/token" + "net/http" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// The three shapes of what a message quotes at its end: the engine's words, +// written as a sentence with its full stop (its 503 unavailable, as crewlet's +// main and crewlet#142 word it under a Retry-After); the engine's words with +// its hint, which the client puts in parentheses after the message, both +// sentences with their full stops (the 503 a node answers while it starts, +// detail and hint as the engine's httpjson.Sentence ends them); and an error +// of Go's, which has none. +var ( + quotedSentence = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "unavailable", + Message: "Can't answer right now. Try again in a moment, or ask another node.", RetryAfter: 2 * time.Second} + quotedHint = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "starting", + Message: "This node is recovering what it holds — its store, its broker and its copy of the estate — " + + "and serves nothing until it has started.", + Hint: `Retry after the Retry-After: once the node serves, /health stops answering status "starting" ` + + `and /ready stops naming reason "starting".`, + RetryAfter: 10 * time.Second} + errQuotedClause = errors.New("dial tcp 10.0.0.7:8080: connect: connection refused") +) + +// endsOnce reports whether message ends with exactly one full stop: one not +// after another, or one a closing parenthesis follows, the end of a quoted +// hint, with nothing after it. +func endsOnce(message string) bool { + if body, quoted := strings.CutSuffix(message, ")"); quoted { + return strings.HasSuffix(body, ".") && !strings.HasSuffix(body, "..") + } + return strings.HasSuffix(message, ".") && !strings.HasSuffix(message, "..") && !strings.HasSuffix(message, ".).") +} + +func TestEndedEndsASentenceOnce(t *testing.T) { + for _, c := range []struct{ text, want string }{ + {"the engine refused it", "the engine refused it."}, + {"Try again in a moment.", "Try again in a moment."}, + {"Is it running?", "Is it running?"}, + {"It stopped!", "It stopped!"}, + {" padded ", "padded."}, + {"", ""}, + {"the value (Secret ns/a)", "the value (Secret ns/a)."}, + {"It failed. (Try again later.)", "It failed. (Try again later.)"}, + {"It failed (try again later)", "It failed (try again later)."}, + {"Is it running? (Ask the node!)", "Is it running? (Ask the node!)"}, + {")", ")."}, + } { + if got := ended(c.text); got != c.want { + t.Errorf("ended(%q) = %q, want %q", c.text, got, c.want) + } + } +} + +// TestAConditionThatQuotesAnErrorEndsItOnce holds every condition that ends +// with an error the operator met to one full stop, whether the error ends in +// the engine's sentence or in Go's words. +func TestAConditionThatQuotesAnErrorEndsItOnce(t *testing.T) { + for _, err := range []error{quotedSentence, quotedHint, errQuotedClause} { + t.Run(err.Error(), func(t *testing.T) { + run := &configRun{CompanyConfigReconciler: &CompanyConfigReconciler{}, company: &crewletaiv1alpha1.Company{}} + run.unavailable(err) + seat := &crewletaiv1alpha1.Seat{Namespace: "ns", Name: seatAna, Generation: 1} + p := newPart(seat) + run.fleetErr = err + run.decideReady(p, seat) + ready := seat.Status.Conditions[0] + for name, message := range map[string]string{ + "ConfigApplied": run.configApplied.Message, + "EngineHealthy": engineHealthy(nil, err).Message, + "IdentityClaimed": identityClaimed(engine.Health{}, err, nil, "", "").Message, + "a Seat's Ready": ready.Message, + } { + if !endsOnce(message) { + t.Errorf("%s does not end with one full stop: %q", name, message) + } + } + }) + } +} + +// TestAFailedApplyAccountEndsWhatItQuotesOnce holds ApplyFailed's account to +// one full stop after what it quotes: the engine's answer when its record of +// the attempt cannot be read, and its account of an apply that failed, which +// is a clause or a sentence as the engine had it. +func TestAFailedApplyAccountEndsWhatItQuotesOnce(t *testing.T) { + written := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, c := range []struct { + name, fleet string + status int + }{ + {name: "a record that cannot be read", status: http.StatusServiceUnavailable, + fleet: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`}, + {name: "an account the engine ends", fleet: fleet(2, "r2", "error", "Something on this node failed.")}, + {name: "an account the engine does not end", fleet: fleet(2, "r2", "error", "engine: apply: no sandbox cell")}, + } { + t.Run(c.name, func(t *testing.T) { + cl := (&fakeEngine{health: health("isolated", "isolated", 1), fleet: c.fleet, fleetStatus: c.status}).client(t) + run, _ := verifyRun(written.Add(time.Second), written, crewletaiv1alpha1.ReasonApplied, nil) + h, err := cl.Health(context.Background()) + run.verifyApplied(context.Background(), cl, h, err, ctrl.Result{}) + account, _, _ := strings.Cut(run.configApplied.Message, " It reports status") + if run.configApplied.Reason != crewletaiv1alpha1.ReasonApplyFailed || !endsOnce(account) { + t.Errorf("ConfigApplied is %s: %q; want ApplyFailed, its account ending with one full stop", + run.configApplied.Reason, run.configApplied.Message) + } + }) + } +} + +// TestASyncThatFailsEndsWhatItQuotesOnce holds SecretsSynced to one full stop +// after the engine's refusal of a write to its secret store. +func TestASyncThatFailsEndsWhatItQuotesOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.secretsDown = true + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + if _, _, err := w.run.sync(ctx, w.client, w.resolve(t), w.state(t), ref); err == nil { + t.Fatal("the sync succeeded against a store that refused it") + } + if c := w.run.secretsSynced; c == nil || c.Status != metav1.ConditionFalse || !endsOnce(c.Message) { + t.Errorf("SecretsSynced is %+v, want False, its message ending with one full stop", c) + } +} + +// TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce holds the Warning of an +// activation the operator could not read back to one full stop after the +// engine's answer to the read, before the sentence that follows it. +func TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // The operator asks for the activation: the engine did not at the removal. + w.engine.reloadAnswer = new(map[string]any(nil)) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + if err != nil { + t.Fatal(err) + } + w.engine.afterReload = func(e *storeEngine) { e.configDown = true } + if _, err := w.run.reactivate(ctx, w.client, nil, res.Reload, res.ReloadAction, state); err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Warning Reactivated ") }) + if i < 0 { + t.Fatalf("events %q, want a Warning Reactivated", recorded) + } + if strings.Contains(recorded[i], "..") { + t.Errorf("the Warning quotes the engine with two full stops: %q", recorded[i]) + } +} + +// quotedAtTheEnd is a format whose last verb quotes text and ends the +// sentence: "…: %s." and "…: %v.". +var quotedAtTheEnd = regexp.MustCompile(`: %[sv]\.$`) + +// TestQuotedEndingsGoThroughEnded holds the package's messages to ended: no +// message ends a sentence with a full stop of its own, neither as +// `text + "."` nor as a format ending ": %s." or ": %v.". Either is right only +// while what comes before it has none, and an error that carries the +// engine's words, which the engine ends with one, makes it two. A quote in +// the middle of a message (a pause's reason, a read that failed, followed by +// a sentence of the operator's) is not a shape this can see: those end the +// sentence that quotes them through ended too, and the tests beside this one +// hold each. +func TestQuotedEndingsGoThroughEnded(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + var found []string + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + t.Fatal(err) + } + // joined are the "." joins inside a longer join, such as a host + // name's labels: a full stop that ends nothing. + joined := map[ast.Expr]bool{} + ast.Inspect(f, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncDecl: + // ended's own body is where the rule is kept. + return n.Name.Name != "ended" + case *ast.BinaryExpr: + if inner, ok := n.X.(*ast.BinaryExpr); ok && n.Op == token.ADD { + joined[inner] = true + } + if lit, ok := n.Y.(*ast.BasicLit); ok && n.Op == token.ADD && lit.Value == `"."` && !joined[n] { + found = append(found, fset.Position(n.Pos()).String()+`: ends a message with a "." of its own`) + } + case *ast.BasicLit: + if text, err := strconv.Unquote(n.Value); n.Kind == token.STRING && err == nil && quotedAtTheEnd.MatchString(text) { + found = append(found, fset.Position(n.Pos()).String()+": ends a quote with a full stop of its own") + } + } + return true + }) + } + if len(found) == 0 { + return + } + t.Errorf("end these through ended:\n%s", strings.Join(found, "\n")) +} From 29bf8f1750ae8182f8c88cb3d04397ab62730a6d Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 23 Oct 2026 09:42:50 +0000 Subject: [PATCH 02/36] test(controller): answer the pause tests' roster 503 as the engine does The pause tests' engine answered a roster that cannot be read with http.Error: a text/plain body holding {"error":"unavailable"} and no Retry-After. The engine never gives that answer. Once a node serves, GET /agents fails only where it cannot decide who is asking (its auth.Caller), and then answers 503 identity_unavailable as JSON, with its sentence and Retry-After: 2, on crewlet's main and at crewlet#142 alike. The client read the double's answer as a proxy's page rather than as the engine's refusal, so a change to how the operator treats the engine's answer would have been tested against a page instead. The double now answers as the engine words it. TestAPauseThatCannotBeMadeIsAskedForAgainSoon holds what it held: the pause is taken without the roster, and a look comes again within ten seconds, which is what the operator does with any roster it cannot read. Signed-off-by: Ali Yousefian --- internal/controller/config_pause_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/controller/config_pause_test.go b/internal/controller/config_pause_test.go index 5552a1f1..d19d3c8b 100644 --- a/internal/controller/config_pause_test.go +++ b/internal/controller/config_pause_test.go @@ -27,7 +27,8 @@ import ( type pauseEngine struct { mu sync.Mutex // roster is what GET /agents answers, by handle; rosterDown makes it - // fail. + // fail as a serving engine's does, which answers 503 only when it + // cannot decide who is asking (its auth.Caller). roster map[string]bool rosterDown bool // record is whether the coordination store holds each seat paused, which @@ -96,7 +97,11 @@ func (f *pauseEngine) client(t *testing.T) *engine.Client { f.mu.Lock() defer f.mu.Unlock() if f.rosterDown { - http.Error(w, `{"error":"unavailable"}`, http.StatusServiceUnavailable) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"identity_unavailable",` + + `"message":"Can't check who you are right now. Try again in a moment."}`)) return } rows := []map[string]any{} From cc8b0934f317ff0ae0ea4e937515797782e8475b Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 23 Oct 2026 09:42:50 +0000 Subject: [PATCH 03/36] test(engine): answer a refused secret write as the engine does TestPutSecretErrorsNeverContainTheValue refused the write as 503 no_keyring with a detail and no message, and failed it as 500 internal_error with no message. The engine gives neither. Every node holds the keyring now: its secret surface refuses to start without one, so no route of it answers no_keyring, on crewlet's main and at crewlet#142 alike. And every refusal carries the sentence the engine writes for its code. A test of what an error may say was checking an error the client never builds. The refusal is now the 413 body_too_large the route answers a value past what it takes, and the 500 carries the engine's sentence, as the engine words both. The test holds what it held: no error carries the value. Signed-off-by: Ali Yousefian --- internal/engine/client_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 6006090d..8ee729c9 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -731,13 +731,15 @@ func TestPutSecretSendsTheRawValue(t *testing.T) { // TestPutSecretErrorsNeverContainTheValue holds the package's promise on every // way a write can fail: a refusal, an error answer, a proxy's page and a -// transport failure. +// transport failure. The engine's answers are as it words them: a value past +// what the route takes, and a store it could not write. func TestPutSecretErrorsNeverContainTheValue(t *testing.T) { const value = "sk-live-do-not-leak" cases := map[string]http.HandlerFunc{ - "refused": answer(http.StatusServiceUnavailable, - `{"error":"no_keyring","detail":"this node has no secrets.keys"}`), - "internal error": answer(http.StatusInternalServerError, `{"error":"internal_error"}`), + "refused": answer(http.StatusRequestEntityTooLarge, `{"error":"body_too_large","message":"The request body is `+ + `larger than this endpoint accepts. Send a smaller document, or split the change across more than one request."}`), + "internal error": answer(http.StatusInternalServerError, `{"error":"internal_error",`+ + `"message":"Something went wrong inside the engine. The reason is in this node's log."}`), "a proxy's page": page(http.StatusBadGateway), } for name, h := range cases { From dab08fa39d26d83091b4928917045f05cfd0c759 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 04:49:16 +0000 Subject: [PATCH 04/36] fix(controller): report a revision the engine can't open as that The engine answers a read of the fleet's company document 409 unreadable_revision when that revision is sealed under a key its keyring no longer holds, and 409 unsealed_revision when the revision is stored without a seal, which no node reads. It answers a write the same way, since a write reads the revision it replaces first. Each carries the revision, the engine's account as its detail, its remedy as its hint, and no Retry-After. The code's own sentence says to put the key back in the node's keyring (crewlet's main and crewlet#142 alike). The operator read that as an engine it could not reach: ConfigApplied False EngineUnavailable, "the operator's request to the company's engine failed, and it tries again in 10s", and a look every ten seconds. No look clears it, and nothing the operator sends does. What does is given to the data pods: a key put back in Secret -keyring, which the operator renders into every pod and never rewrites, and which the pods read only when they start. So the client tells it apart (ErrRevisionUnreadable, matched by the two codes, with the revision decoded into APIError.RevisionID), and a reconcile it ends reports ConfigApplied False RevisionUnreadable, with one Warning event: the revision, the engine's refusal quoted whole, and what restores it. For unreadable_revision that is the key put back in the keyring Secret from the backup kept with the volumes. For unsealed_revision, which no key opens, it is the engine's hint. The next look is at the resync interval, as for an engine with nothing to do, rather than ten seconds on. The data pods restarting with a restored key changes RuntimeReady, which runs the reconcile at once. The engine's failures that end a reconcile are now decided in one place (engineFailed), and their next look in one (engineRetry). The withdrawal of a deleted Secret's value met the same answer and said it would try again in ten seconds. It now waits as engineRetry says, and follows a Retry-After the engine sends, which it used to ignore. Its SecretWithdrawalFailed event now ends through ended(), like the conditions that quote the engine. The troubleshooting guide gains the reason's row, its branch of the flowchart, and "The engine cannot open its company document"; applying.md lists the reason. TestARevisionTheEngineCannotOpenBlocksTheDocument fails against the old reconcile: EngineUnavailable, a promise of a retry, and a look ten seconds on. Its doubles send the detail and the hint as the engine's httpjson.Sentence ends them, and it holds the message to quoting the hint, which closes on its own full stop, without a second one after it ("....)."). It also fails with the reason's branch dropped, with the next look back on retryAfter, and with one remedy for both codes. TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt fails with the withdrawal back on engineRetryInterval. TestAnUnreadableRevisionNamesItAndItsRemedy fails with the revision left undecoded, and TestErrorMapping holds both codes to the sentinel and a proxy's 409 to none. Signed-off-by: Ali Yousefian --- api/v1alpha1/common_types.go | 11 ++ docs/concepts/applying.md | 8 +- docs/guides/troubleshooting.md | 33 ++++ internal/controller/config_run.go | 67 +++++++- internal/controller/config_unreadable_test.go | 146 ++++++++++++++++++ internal/controller/config_withdraw.go | 22 +-- internal/controller/config_withdraw_test.go | 22 ++- internal/engine/client.go | 15 +- internal/engine/client_test.go | 68 +++++++- internal/engine/errors.go | 35 ++++- 10 files changed, 397 insertions(+), 30 deletions(-) create mode 100644 internal/controller/config_unreadable_test.go diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 8aa2350d..2b55f584 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -635,6 +635,17 @@ const ( // still exists and is held: one whose kind turned from Human to Agent. ReasonSeatHeld = "SeatHeld" + // ReasonRevisionUnreadable is ConfigApplied=False when the engine's + // keyring does not open the fleet's revision of the company document + // (409 unreadable_revision or unsealed_revision): it is sealed under a + // key the keyring no longer holds, or stored without a seal. The engine + // then refuses to show the document and to take another, so the + // operator writes nothing; no wait changes that, and somebody has to + // give the engine what opens the revision, such as the key put back in + // the Secret -keyring. The message names the revision and + // carries the engine's account and remedy. + ReasonRevisionUnreadable = "RevisionUnreadable" + // ReasonInvalid is Applied=False (or ConfigApplied=False) when the // engine's dry run refuses the rendered document; the message carries // the engine's own words. diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index f31487dc..2ea9186b 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -355,7 +355,11 @@ reports `ConfigApplied=False` with the reason `EngineUnavailable`, and the operator tries again every ten seconds, or sooner where the engine's `Retry-After` says so. Because the address is the cluster's DNS name of the pod, an operator running outside -the cluster (`make run`) cannot reach any engine. +the cluster (`make run`) cannot reach any engine. When the engine answers but +its keyring does not open the fleet's revision, the Company reports +`RevisionUnreadable` instead, which no retry clears, and the operator looks +again when the data pods restart and every five minutes besides +([The engine cannot open its company document](../guides/troubleshooting.md#the-engine-cannot-open-its-company-document)). --- @@ -516,7 +520,7 @@ takes no work for a person that a pause could hold. | Object | Condition | When | |---|---|---| | Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know (it names those, and which of the two it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | -| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine) or `EngineUnavailable`. | +| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine's keyring does not open the fleet's revision, so it neither shows the document nor takes another) or `EngineUnavailable`. | | Company | `EngineHealthy` | The engine's alarm table, read each time the engine is reached: `True`, `NoAlarms`; `False`, `Alarms`, naming the one standing longest; `Unknown`, `AlarmsNotReported`, before the engine has evaluated them. Not part of `Ready`. | | Company | `IdentityClaimed` | Whether anybody has joined the company, read from the engine's `/health` each time the engine is reached, and every 30 seconds while nobody has, since a person joins where no watch sees it: `True`, `PersonEnrolled`, once its identity directory holds a person; `False`, `Unclaimed`, while nobody has joined, naming up to five of the company's Human seats and the access token to bring the first person in with; `False`, `NoHumanSeat`, when nobody can, for want of a Human seat; `Unknown`, `IdentityNotReported`, when the engine could not read its directory, did not say, or could not be asked. Not part of `Ready`: a company nobody has joined runs its agent seats. | | every part | `Accepted` | `True` once its Company exists; `False` with `CompanyNotFound`, or `Conflict` when it collides with another object of its company. | diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 20ff8af7..75c7a099 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -33,6 +33,7 @@ flowchart TD EX -->|True| CA{"ConfigApplied"} CA -->|EngineUnavailable| UN["The operator cannot reach the engine"] CA -->|"WriteRefused, NotGranted, SeatHeld"| WR["The engine refuses a change
to the document"] + CA -->|RevisionUnreadable| RU["The engine cannot open
its company document"] CA -->|"Applying, ApplyFailed"| AF["The engine did not apply a revision"] SYM -->|"IdentityClaimed=False"| NS["Nobody can sign in"] CA -->|"any other reason"| NA["Changes are not applied"] @@ -122,6 +123,7 @@ with your last change yet. | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | +| | False | `RevisionUnreadable` | The engine's keyring does not open the fleet's revision of the company document (`409 unreadable_revision` or `unsealed_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | ### EngineClass @@ -439,6 +441,37 @@ account, or cancels the invitation. `crewlet iam seats` lists what holds each seat. The operator does neither for you: whoever holds a seat is a person, or something a person set up. Or keep the seat: restore its Seat as a Human seat. +## The engine cannot open its company document + +`ConfigApplied=False RevisionUnreadable` means the engine's keyring does not +open the fleet's revision of the company document. The engine refuses every +read of the document with `409`, and every write too, since a write reads the +revision it replaces; it keeps running what it applied, and shows and takes +no document until it can open that revision. No wait changes that, and +nothing the operator sends does, so the operator writes nothing and does not +ask again every ten seconds: it looks again when the data pods restart, and +every five minutes besides. The message names the revision, quotes the +engine's account and its remedy, and says which of two it is: + +- `unreadable_revision`: the revision is sealed under a key the data pod's + keyring does not hold. The keyring is the Secret `-keyring`, which + the operator generates once and never rewrites, so a key gone from it was + taken out by somebody else, or the Secret was restored from an older backup + than the volumes. Put the key back in that Secret from the backup kept + with the volumes, with its `crewlet.ai/active-key-id` annotation as it was: + the data pods restart with it, and their restart runs the reconcile again. +- `unsealed_revision`: the revision is stored without a seal, which no keyring + opens and no engine reads; only an engine older than the keyring every + engine now requires stored one. The engine's hint says what brings its + document back. + +To see which keys the keyring holds without printing them: + +```bash +kubectl get secret acme-keyring -n acme -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' +kubectl get events -n acme --field-selector involvedObject.name=acme,reason=RevisionUnreadable +``` + ## The engine did not apply a revision `ConfigApplied=False ApplyFailed` means the engine stored and activated the diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 10533247..b83cf57b 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -281,11 +281,7 @@ func (run *configRun) reconcileDocument(ctx context.Context) (ctrl.Result, error if isAPIServerError(err) { return ctrl.Result{}, err } - if !run.writeRefused(err) && !run.notGranted(err) && !run.seatHeld(err) { - run.unavailable(err) - } - _, _ = run.fleetOf(ctx, eng) - return ctrl.Result{RequeueAfter: retryAfter(err)}, nil + return run.engineFailed(ctx, eng, err), nil } health, healthErr := run.readHealth(ctx, eng) result = run.verifyApplied(ctx, eng, health, healthErr, result) @@ -1233,6 +1229,67 @@ func (run *configRun) block(reason, message string) { why, run.company.Name)} } +// engineFailed records err, the engine's, or a request's to it, which ended a +// reconcile before it could decide anything, as what it is: a refusal of the +// operator's token, a seat the document takes away while somebody holds it, a +// revision the engine cannot read, or an engine that could not be reached or +// answer now. It reads the fleet for the Seats' placement, and returns when to +// look at the engine again (engineRetry). +func (run *configRun) engineFailed(ctx context.Context, eng *engine.Client, err error) ctrl.Result { + if !run.writeRefused(err) && !run.notGranted(err) && !run.seatHeld(err) && !run.revisionUnreadable(err) { + run.unavailable(err) + } + _, _ = run.fleetOf(ctx, eng) + return ctrl.Result{RequeueAfter: run.engineRetry(err)} +} + +// engineRetry is how soon the reconcile looks at a Company's engine again +// after err. A revision the engine's keyring does not open is looked at again +// at the resync interval, as an engine with nothing to do is: the engine sends +// no Retry-After for it, no wait opens it, and nothing the operator sends +// does. What does is given to the data pods, a key put back in the keyring +// Secret, which they read only when they start, and the data pods restarting +// changes RuntimeReady, which runs the reconcile at once. Anything else is +// asked again as retryAfter says. +func (run *configRun) engineRetry(err error) time.Duration { + if errors.Is(err, engine.ErrRevisionUnreadable) { + return run.resyncInterval() + } + return retryAfter(err) +} + +// revisionUnreadable records an engine whose keyring does not open the +// fleet's revision of the company document (engine.ErrRevisionUnreadable), and +// reports false for any other error. The engine refuses then to show the +// document and to take another, since a write reads the revision it replaces, +// so the operator can neither see the document nor change it. It is +// reported as what blocks the document, in the engine's words, with what +// restores the revision: a key the revision is sealed under is one of the +// keyring Secret's, which the operator renders into every pod and never +// rewrites, so a key gone from it is put back there, from the backup kept with +// the data volumes. +func (run *configRun) revisionUnreadable(err error) bool { + var apiErr *engine.APIError + if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) { + return false + } + revision := "the revision" + if apiErr.RevisionID != "" { + revision = "revision " + apiErr.RevisionID + } + remedy := fmt.Sprintf("The keyring is Secret %s, which the operator never rewrites: put the key %s is sealed under "+ + "back in it from the backup kept with the data volumes, and the data pods restart with it.", + keyringSecretName(run.company), revision) + if apiErr.Code == "unsealed_revision" { + remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + + "requires, and no keyring opens it; the engine's hint says what brings its document back." + } + run.blockAndTell(crewletaiv1alpha1.ReasonRevisionUnreadable, "ReadConfig", ended(fmt.Sprintf( + "The engine's keyring does not open %s of the company document, so the engine neither shows the document "+ + "nor takes another, and no wait changes that: %s", revision, err))+" "+remedy) + return true +} + // unavailable records an engine that could not be reached, or that failed a // request for a reason of its own. func (run *configRun) unavailable(err error) { diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go new file mode 100644 index 00000000..c33b7f65 --- /dev/null +++ b/internal/controller/config_unreadable_test.go @@ -0,0 +1,146 @@ +package controller + +import ( + "context" + "fmt" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" +) + +// These tests hold what the operator does with an engine whose keyring does +// not open the fleet's revision of the company document: the engine answers +// every read of the document, and every write, which reads the revision it +// replaces, 409 with no Retry-After, and nothing clears it but somebody giving +// the engine what opens the revision. So it is reported as what blocks the +// document, in the engine's words and with the remedy, and looked at again at +// the resync interval, not ten seconds on with a promise of a retry that +// changes nothing. + +// keyringLost and sealMissing are the engine's answers to a read of revision, +// sealed under a key its keyring no longer holds or stored without a seal, as +// crewlet's configapi words them (refuseUnreadable) and its httpjson.Envelope +// sends them, the detail and the hint each ended as a sentence. +func keyringLost(revision string) string { + return fmt.Sprintf(`{"error":"unreadable_revision","revision_id":%q,`+ + `"message":"That revision is sealed under a key this deployment no longer holds, so it cannot be read. `+ + `Put the key back in the node's keyring first.",`+ + `"detail":"secrets: open document: secret could not be decrypted.",`+ + `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's `+ + `secrets.keys first."}`, revision) +} + +func sealMissing(revision string) string { + return fmt.Sprintf(`{"error":"unsealed_revision","revision_id":%q,`+ + `"message":"That revision is stored without a seal, which a build older than the mandatory keyring wrote, `+ + `and a node reads only sealed revisions because the seal is what authenticates one. The hint says what `+ + `brings its document back.",`+ + `"detail":"secrets: this document is stored unsealed, and a node holding a keyring reads only sealed ones `+ + `— a plaintext payload is one anything that can write the store or the coordination bucket could have `+ + `authored.",`+ + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown nor `+ + `reverted to; to have its document again, import it from your own copy with PUT /config or `+ + "`crewlet config import`"+`, which stores it sealed."}`, revision) +} + +// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine whose +// keyring does not open the fleet's revision to ConfigApplied=False +// RevisionUnreadable: the message names the revision, quotes the engine's +// account and its remedy, and says what restores it in the Company's terms; +// the operator writes nothing; and the next look is at the resync interval. +// Reported as an engine that is unavailable, it promised a retry in ten +// seconds that no retry could keep. +func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { + for _, c := range []struct { + name string + answer func(revision string) string + says []string + }{ + { + name: "a key the keyring lost", answer: keyringLost, + says: []string{"revision r1", "secrets: open document: secret could not be decrypted", + "restore that key to the node's secrets.keys first", "Secret acme-keyring"}, + }, + { + name: "a revision stored without a seal", answer: sealMissing, + says: []string{"revision r1", "stored unsealed", "import it from your own copy", + "the engine's hint says what brings its document back"}, + }, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.unreadable = c.answer + _, err := w.run.converge(ctx, w.client) + if err == nil { + t.Fatal("converge() = nil over a revision the engine's keyring does not open") + } + result := w.run.engineFailed(ctx, w.client, err) + + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable { + t.Fatalf("ConfigApplied = %+v, want False RevisionUnreadable", got) + } + for _, s := range c.says { + if !strings.Contains(got.Message, s) { + t.Errorf("ConfigApplied does not say %q: %s", s, got.Message) + } + } + if strings.Contains(got.Message, "tries again") || strings.Contains(got.Message, ".).") || + !endsOnce(got.Message) { + t.Errorf("ConfigApplied promises a retry, or ends the engine's words or its own with a second "+ + "full stop: %s", got.Message) + } + if result.RequeueAfter != w.run.resyncInterval() { + t.Errorf("the next look is %s on, want the resync interval, %s", result.RequeueAfter, w.run.resyncInterval()) + } + if ids := w.engine.ids(); len(ids) != 1 { + t.Errorf("the engine holds revisions %v: the operator wrote over a revision the engine cannot open, "+ + "want nothing written", ids) + } + if recorded := drained(w.events); !warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want a Warning RevisionUnreadable", recorded) + } + + // The next reconcile meets the same answer with the condition + // already recorded: it is not told again. + meta.SetStatusCondition(&w.run.company.Status.Conditions, *got) + w.again(t, w.run.company) + if _, err := w.run.converge(ctx, w.client); err != nil { + w.run.engineFailed(ctx, w.client, err) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want the refusal ConfigApplied already reports told once", recorded) + } + }) + } +} + +// TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt holds the +// withdrawal of a deleted Secret's value to the same next look: the engine +// cannot take its revision again while its keyring does not open it either, +// so the operator says so and looks again at the resync interval. +func TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.unreadable = keyringLost + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != w.run.resyncInterval() { + t.Errorf("the next look is %s on, want the resync interval, %s", result.RequeueAfter, w.run.resyncInterval()) + } + recorded := drained(w.events) + if !warned(recorded, "SecretWithdrawalFailed") || + !strings.Contains(strings.Join(recorded, "\n"), "unreadable_revision") { + t.Errorf("events %q, want SecretWithdrawalFailed quoting the engine's refusal", recorded) + } + if !w.engine.holds(w.registry) || len(w.engine.ids()) != 1 { + t.Errorf("the store holds the registry: %t, and the revisions are %v; want both as they were", + w.engine.holds(w.registry), w.engine.ids()) + } +} diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index 0bee9fca..8c10e2d5 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -126,8 +126,8 @@ import ( // // An error is the API server's, for the reconcile to retry. One of the // engine's, or one reaching it, is reported (withdrawalFailed) and tried -// again engineRetryInterval later: the document's conditions already say why -// it is held. +// again when engineRetry says: the document's conditions already say why it +// is held. func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) (ctrl.Result, error) { c := run.company if run.policy() != crewletaiv1alpha1.ManagementPolicyEnforce || suspended(c) || @@ -159,13 +159,11 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) } eng, err := run.Engines.Engine(ctx, c) if err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + return run.withdrawalFailed(ctx, err), nil } active, err := activeConfig(ctx, eng) if err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + return run.withdrawalFailed(ctx, err), nil } if active != nil { if run.doc != nil { @@ -208,8 +206,7 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) return ctrl.Result{}, err } if err := errors.Join(withdrawErr, reactivateErr); err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + return run.withdrawalFailed(ctx, err), nil } return run.heldResync(ctrl.Result{}), nil } @@ -398,11 +395,14 @@ func (run *configRun) settleRefused( // withdrawalFailed reports err, which kept the engine from reading its store // without a value whose Secret no longer exists: until the operator gets // through, the engine may still start a tool server or an executor run with it. -func (run *configRun) withdrawalFailed(ctx context.Context, err error) { +// It returns when to try again, as engineRetry says for err. +func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Result { + retry := run.engineRetry(err) logf.FromContext(ctx).Error(err, "Failed to keep the value of a deleted Secret from the engine") - run.event(corev1.EventTypeWarning, "SecretWithdrawalFailed", "ReloadConfig", + run.event(corev1.EventTypeWarning, "SecretWithdrawalFailed", "ReloadConfig", "%s", ended(fmt.Sprintf( "The engine may still start a tool server or an executor run with the value of a Secret that no longer exists, "+ - "which the revision it runs lets go without; the operator tries again in %s: %s", engineRetryInterval, err) + "which the revision it runs lets go without; the operator tries again in %s: %s", retry, err))) + return ctrl.Result{RequeueAfter: retry} } // reactivate activates the engine's running revision again when state records diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index cc7ba036..9709abfb 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -64,7 +64,9 @@ import ( // does; fleetDown makes GET /fleet unanswerable, and missing names the // revisions GET /config/revisions does not find. configDown and secretsDown // make it answer a read of the fleet's document, and a write to the store, as -// an engine that cannot decide who is asking does. +// an engine that cannot decide who is asking does; unreadable, when set, gives +// the body it answers a read of the fleet's document with, as an engine whose +// keyring does not open that revision does (config_unreadable_test.go). // // Each node's row says what it starts on as the engine's does: a data node // (dataNode) the revision it runs, and a seat node, whose store is scratch, @@ -86,6 +88,7 @@ type storeEngine struct { runsNone map[string]bool fleetDown bool missing map[string]bool + unreadable func(revision string) string } // dataNode reports whether node is a pod of the Company's data StatefulSet, @@ -168,8 +171,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodGet && r.URL.Path == "/health": answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": "serving", "applied_epoch": len(e.revisions)}) case r.Method == http.MethodGet && r.URL.Path == configRoute: - w.Header().Set("ETag", `"`+active.id+`"`) - _, _ = w.Write(active.document) + e.readActive(w, active) case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: answer(http.StatusUnprocessableEntity, map[string]any{"error": "validation_error", "detail": "refused", "problems": []map[string]string{{"path": "roles", "kind": "invalid", "message": "refused"}}}) @@ -373,6 +375,20 @@ func refusal(code, detail string) map[string]any { return map[string]any{"error": code, "detail": detail} } +// readActive answers a read of the fleet's document, active: the document and +// its ETag, or, where unreadable is set, the engine's refusal of a revision +// its keyring does not open. +func (e *storeEngine) readActive(w http.ResponseWriter, active storeRevision) { + if e.unreadable != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(e.unreadable(active.id))) + return + } + w.Header().Set("ETag", `"`+active.id+`"`) + _, _ = w.Write(active.document) +} + // ids are the engine's revisions, oldest first. func (e *storeEngine) ids() []string { e.mu.Lock() diff --git a/internal/engine/client.go b/internal/engine/client.go index 7779f3be..dcf28522 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -13,9 +13,10 @@ // that takes a held human seat away a *SeatHeldError, a lost compare-and-set // matches ErrRevisionAdvanced, a refused token ErrUnauthorized, a token the // engine will not let do this ErrForbidden, a write a managed document -// refuses this token ErrConfigManaged, a request the engine asks to be sent -// again ErrUnavailable, a draining or unconfigured engine ErrNotReady, and -// anything else an *APIError with the engine's code and message. +// refuses this token ErrConfigManaged, a revision the engine's keyring does +// not open ErrRevisionUnreadable, a request the engine asks to be sent again +// ErrUnavailable, a draining or unconfigured engine ErrNotReady, and anything +// else an *APIError with the engine's code and message. // // No error this package returns carries a secret value: the engine never // echoes one, and a secret written with WriteSecret or PutSecrets travels @@ -270,7 +271,9 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { // goes on running the revision it ran before, and starts on the revision its // own store names where that store outlives a restart, as a data pod's does // (a seat pod's does not: it starts on nothing and applies the fleet's -// revision). Before the first apply it fails with ErrNoActiveRevision. +// revision). Before the first apply it fails with ErrNoActiveRevision, and +// on an engine whose keyring does not open the fleet's revision with +// ErrRevisionUnreadable. // // An engine that cannot read the activation pointer from its coordination // store answers 503 with a Retry-After, which is ErrUnavailable: "the fleet @@ -353,7 +356,9 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // A document the engine refuses fails with a *ValidationError locating each // problem; a stale req.IfMatch, or a write that lost a race with another // writer, with ErrRevisionAdvanced; req.CreateOnly on a configured engine with -// ErrAlreadyConfigured. +// ErrAlreadyConfigured; and a write while the engine's keyring does not open +// the fleet's revision, which the write reads first, with +// ErrRevisionUnreadable. func (c *Client) ApplyConfig(ctx context.Context, req ApplyRequest) (ApplyResult, error) { if req.IfMatch != "" && req.CreateOnly { return ApplyResult{}, errors.New("apply the company document: IfMatch and CreateOnly are mutually exclusive: " + diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 8ee729c9..55b1166f 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -456,6 +456,21 @@ func TestErrorMapping(t *testing.T) { handler: page(http.StatusForbidden), wantStatus: http.StatusForbidden, }, + { + name: "a revision sealed under a key the keyring lost", + handler: answer(http.StatusConflict, unreadableRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unreadable_revision", wantStatus: http.StatusConflict, + }, + { + name: "a revision stored without a seal", + handler: answer(http.StatusConflict, unsealedRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_revision", wantStatus: http.StatusConflict, + }, + { + name: "a proxy's 409", + handler: page(http.StatusConflict), + wantStatus: http.StatusConflict, + }, { name: "an engine that could not decide", handler: answer(http.StatusServiceUnavailable, `{"error":"unavailable",`+ @@ -477,7 +492,7 @@ func TestErrorMapping(t *testing.T) { sentinels := []error{ engine.ErrRevisionAdvanced, engine.ErrAlreadyConfigured, engine.ErrNoActiveRevision, engine.ErrUnauthorized, engine.ErrNotReady, engine.ErrConfigManaged, engine.ErrForbidden, - engine.ErrUnavailable, + engine.ErrUnavailable, engine.ErrRevisionUnreadable, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -502,6 +517,57 @@ func TestErrorMapping(t *testing.T) { } } +// unreadableRevision and unsealedRevision are the engine's answers to a read of +// a revision its keyring does not open, as crewlet's configapi words them +// (refuseUnreadable) and its httpjson.Envelope sends them, the detail and the +// hint each ended as a sentence (httpjson.Sentence): the code's own sentence, +// the revision, the keyring's account of the failure as the detail, and the +// remedy as the hint. +const ( + unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + + `"revision_id":"rev-4","detail":"secrets: open document: secret could not be decrypted.",` + + `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's ` + + `secrets.keys first."}` + unsealedRevision = `{"error":"unsealed_revision","message":"That revision is stored without a seal, which ` + + `a build older than the mandatory keyring wrote, and a node reads only sealed revisions because the seal ` + + `is what authenticates one. The hint says what brings its document back.","revision_id":"rev-4",` + + `"detail":"secrets: this document is stored unsealed, and a node holding a keyring reads only sealed ones ` + + `— a plaintext payload is one anything that can write the store or the coordination bucket could have ` + + `authored.",` + + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown nor ` + + `reverted to; to have its document again, import it from your own copy with PUT /config or ` + + "`crewlet config import`" + `, which stores it sealed."}` +) + +// TestAnUnreadableRevisionNamesItAndItsRemedy reads the engine's answer to a +// read of the fleet's document its keyring does not open: ErrRevisionUnreadable +// and no other refusal, naming the revision, with the engine's account as the +// message and its remedy as the hint, which is everything a person has to act +// on, since no wait and no request clears it. +func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { + for _, body := range []string{unreadableRevision, unsealedRevision} { + f := newFakeEngine(t) + f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) + f.handle("GET /config", answer(http.StatusConflict, body)) + _, err := f.client().ActiveConfig(context.Background()) + var apiErr *engine.APIError + if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) || + errors.Is(err, engine.ErrUnavailable) || errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("ActiveConfig() = %v, want ErrRevisionUnreadable and no other refusal", err) + } + var want struct{ Detail, Hint string } + if err := json.Unmarshal([]byte(body), &want); err != nil { + t.Fatal(err) + } + if apiErr.RevisionID != "rev-4" || apiErr.Message != want.Detail || apiErr.Hint != want.Hint || + apiErr.RetryAfter != 0 { + t.Errorf("APIError = {revision %q, message %q, hint %q, retry after %s}, want {rev-4, the detail, "+ + "the hint, 0}", apiErr.RevisionID, apiErr.Message, apiErr.Hint, apiErr.RetryAfter) + } + } +} + func TestRevisionAdvancedNamesTheCurrentRevision(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", answer(http.StatusConflict, diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 13bf6f76..df787ef3 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -64,6 +64,18 @@ var ( // engine said when. ErrUnavailable = errors.New("the engine could not answer now") + // ErrRevisionUnreadable is a revision of the company document that the + // engine's keyring does not open (409 unreadable_revision or + // unsealed_revision): sealed under a key the keyring no longer holds, or + // stored without a seal, which no node reads. The engine answers it to a + // read of the document and to a write, which reads the revision it + // replaces, and sends no Retry-After: no wait opens the revision, and + // nothing the caller sends does, since what does is given to the engine + // (a key put back in its keyring, which it reads when it starts). The + // *APIError behind it carries the revision in RevisionID, the engine's + // account as its Message and the remedy as its Hint. + ErrRevisionUnreadable = errors.New("the engine's keyring does not open a revision of its company document") + // ErrNotReady is an engine that is up but should not be sent work: it // is draining for a shutdown, has no company yet, or its posture is shed // or stuck. The *NotReadyError behind it names which. @@ -92,6 +104,12 @@ const ( // has its own sentinel. var codesForbidden = []string{"unauthorized", "token_refused", "seat_unavailable"} +// codesRevisionUnreadable are the refusals of a revision the engine's keyring +// does not open, one per cause because the remedies share nothing: a key put +// back in the keyring (unreadable_revision), or the document brought back +// sealed (unsealed_revision). +var codesRevisionUnreadable = []string{"unreadable_revision", "unsealed_revision"} + // codesWithProblems are the refusals of a document: the engine located each // failure in it, so they become a *ValidationError. var codesWithProblems = map[string]struct{}{ @@ -105,9 +123,10 @@ var codesWithProblems = map[string]struct{}{ // engine's (*NotReadyError). // // It matches ErrUnauthorized, ErrForbidden, ErrRevisionAdvanced, -// ErrAlreadyConfigured, ErrNoActiveRevision, ErrConfigManaged and -// ErrUnavailable with errors.Is where its status and code say so, so a caller -// can branch on the sentinel and still read the engine's message. +// ErrAlreadyConfigured, ErrNoActiveRevision, ErrConfigManaged, +// ErrRevisionUnreadable and ErrUnavailable with errors.Is where its status +// and code say so, so a caller can branch on the sentinel and still read the +// engine's message. type APIError struct { // StatusCode is the HTTP status. StatusCode int @@ -135,6 +154,10 @@ type APIError struct { // left inert, when the engine kept one. StoredRevisionID string + // RevisionID is the revision a refusal is about, when the engine named + // one: the revision its keyring does not open (ErrRevisionUnreadable). + RevisionID string + // ManagedBy are the token ids that may change the company document, // when the engine refused this write as config_managed. ManagedBy []string @@ -205,6 +228,10 @@ func (e *APIError) Is(target error) bool { // Matched by code and status together: a 403 without the code is // a proxy's, and says nothing about the document. return e.StatusCode == http.StatusForbidden && e.Code == codeConfigManaged + case ErrRevisionUnreadable: + // By code alone: only the engine writes these codes, so a 409 + // without one (a proxy's) says nothing about its keyring. + return slices.Contains(codesRevisionUnreadable, e.Code) } return false } @@ -422,6 +449,7 @@ type refusal struct { Problems []Problem `json:"problems"` CurrentRevisionID string `json:"current_revision_id"` StoredRevisionID string `json:"stored_revision_id"` + RevisionID string `json:"revision_id"` ManagedBy []string `json:"managed_by"` Held map[string]heldAnswer `json:"held"` } @@ -483,6 +511,7 @@ func errorFromResponse(status int, header http.Header, body []byte) error { Hint: r.Hint, CurrentRevisionID: r.CurrentRevisionID, StoredRevisionID: r.StoredRevisionID, + RevisionID: r.RevisionID, ManagedBy: r.ManagedBy, Reason: r.Reason, Grants: r.Grants, From 5bd464688729be55cf6c1c0b2e454ff5488122fd Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 05:05:12 +0000 Subject: [PATCH 05/36] docs(docs): name the engine dashboard's screens as it now does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crewlet's main has regrouped its dashboard, and crewlet#142 keeps it so: the org chart and the schedules moved under Company (Company › Org chart, Company › Schedules), and Settings › People & access split into Settings › People, Settings › Service accounts and Settings › API tokens. The getting-started guide and the people guide still sent somebody to Agents › Org chart to claim the founder seat and invite the others, and to Settings › People & access to change them, and the company document page to Agents › Schedules for a schedule's history: screens the dashboard no longer has. They now name Company › Org chart, Settings › People and Company › Schedules, as Crewlet's own guides do. The labels on the way there (Sign in with token, Use an API token instead, Open seat, Invite, Add person) are as they were. Signed-off-by: Ali Yousefian --- docs/concepts/company-document.md | 2 +- docs/getting-started.md | 2 +- docs/guides/people.md | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/concepts/company-document.md b/docs/concepts/company-document.md index cbf7cd90..e2e32ac1 100644 --- a/docs/concepts/company-document.md +++ b/docs/concepts/company-document.md @@ -165,7 +165,7 @@ The engine does not report when a schedule last fired in a form a status field could carry: its record of the fires it dispatched is each node's own table, which the answering node serves, while the fleet's claim on each fire is internal to its coordination store. A Schedule therefore has no -`lastScheduleTime`; the engine's dashboard (**Agents › Schedules**) shows +`lastScheduleTime`; the engine's dashboard (**Company › Schedules**) shows each schedule's next fire and its history. ### Placement diff --git a/docs/getting-started.md b/docs/getting-started.md index 35ac0859..91604d3a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -323,7 +323,7 @@ bin/kubectl port-forward -n acme svc/acme-engine 8080:8080 Open exactly : the engine is told it is reached there, and accepts a sign-in from that address and no other. While nobody has joined, the sign-in page puts its token form first: paste the token and -choose **Sign in with token**. Then open **Agents › Org chart**, open the +choose **Sign in with token**. Then open **Company › Org chart**, open the founder seat, choose **Add person** with your email address and **Admin** ticked, and open the password link it shows you once. Choose a password, sign in with your address, and set up the authenticator app the engine asks for. diff --git a/docs/guides/people.md b/docs/guides/people.md index 3a779c07..cf31ea7e 100644 --- a/docs/guides/people.md +++ b/docs/guides/people.md @@ -167,7 +167,7 @@ kubectl get secret acme-api-access -n acme -o jsonpath='{.data.token}' | base64 In the dashboard, the sign-in page puts its token form first while nobody has joined (**Sign in with token**; once somebody has, it is folded under -**Use an API token instead**). Paste the token, open **Agents › Org chart**, +**Use an API token instead**). Paste the token, open **Company › Org chart**, open your seat (drawn **Open seat**) and choose: - **Invite**: your address, and **Admin** ticked. The invitation link is @@ -202,8 +202,8 @@ wait for it: a company nobody has joined runs its agents all the same. ## Everyone after -People with `admin` invite and create the others in the dashboard (**Agents › -Org chart**, on the person's seat; **Settings › People & access** to change, +People with `admin` invite and create the others in the dashboard (**Company › +Org chart**, on the person's seat; **Settings › People** to change, suspend or remove them), or with `crewlet iam`. Each needs a vacant Human seat: add the Seat first. Under `Enforce` the company document is read-only in the dashboard, so a seat is added as a `Seat` object, not there From 7ef90797bfc573ec51e27b95d6d94c89c47aac44 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 08:52:01 +0000 Subject: [PATCH 06/36] docs(docs): give the backup command the token the engine asks for The runtime page's backup command ran crewlet backup in the data pod with no token. Every command that reaches a running node takes its token from CREWLET_API_TOKEN and nothing else, never a flag, and on the main image, which resolves no request without a credential (only an unstamped development build run with -dev-principal does, and only on loopback), the node answered the backup 401, so none was taken (crewlet's main and crewlet#142 alike). The command now pipes the access token the Company publishes into CREWLET_API_TOKEN inside the pod, as the people guide's invitation does, and passes the backup's directory as an argument expanded on the operator's machine, so the timestamp's quoting survives the single-quoted shell. The node is named with -url, the flag crewlet's main gives it. Checked: crewlet's main binary takes -url and refuses -api, and under dash, which the engine image's Debian base runs as sh, the token reaches crewlet's environment and the directory its -dir. The full command was not run against a cluster. Signed-off-by: Ali Yousefian --- docs/concepts/runtime.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index 88d83aee..22155f2d 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -899,11 +899,17 @@ the start, which `EngineHealthy=False Alarms` shows. Take backups through the engine, which is the only process that can read its estate. `crewlet backup` writes a verified copy into a directory on the data -pod's volume, without stopping the engine: +pod's volume, without stopping the engine. Like every command that reaches a +running node it takes its token from `CREWLET_API_TOKEN` alone, never a flag, +so pipe in the access token the Company publishes +([Reaching the engine](#reaching-the-engine)); the directory is named on your +machine, so the timestamp is its clock's, in UTC: ```bash -kubectl exec -n acme acme-data-0 -- crewlet backup -config /etc/crewlet/crewlet.yaml \ - -url http://127.0.0.1:8080 -dir /var/lib/crewlet/backups/$(date -u +%Y-%m-%dT%H-%M) +dir=/var/lib/crewlet/backups/$(date -u +%Y-%m-%dT%H-%M) +kubectl get secret acme-api-access -n acme -o jsonpath='{.data.token}' | base64 -d | + kubectl exec -i -n acme acme-data-0 -- sh -c 'CREWLET_API_TOKEN=$(cat) exec crewlet backup \ + -config /etc/crewlet/crewlet.yaml -url http://127.0.0.1:8080 -dir "$1"' sh "$dir" ``` Run it at least daily. Copy each backup off the volume (`kubectl cp`), since From a4cb1dc38ddb146c854c16b8b6b9a8d324747b49 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Mon, 26 Oct 2026 03:37:26 +0100 Subject: [PATCH 07/36] test(engine): word the client's doubles as crewlet's main does Two of the engine client's doubles answered with words crewlet's main never writes: - The 503 every route but the probes answers while a node starts said "This node is starting." with a hint of its own. The engine says "This node is still starting and is not serving yet. Try again in a moment.", with what it is recovering as the detail and the hint that names /health's status and /ready's reason, each capitalised and ended with a full stop, as httpjson.Sentence ends every refusal's detail and hint. - An unreadable lease table on GET /fleet carried "the lease table could not be read" as its message and no Retry-After. The engine answers a coordination store that did not answer with the unavailable code's own sentence and a Retry-After of five seconds, the health tick's. No operator code reads those words, so what each test decides is as it was: ErrUnavailable with the engine's Retry-After and not ErrNotReady, and an error rather than an empty fleet. They now describe an engine somebody runs. Signed-off-by: Ali Yousefian --- internal/engine/client_test.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 55b1166f..e0e61c0a 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -745,8 +745,12 @@ func TestStartingWriteIsAskedAgainWhenTheEngineSays(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Retry-After", "10") - answer(http.StatusServiceUnavailable, `{"error":"starting","message":"This node is starting.",`+ - `"hint":"retry after the Retry-After; /health stops answering status \"starting\" once the node serves"}`)(w, r) + answer(http.StatusServiceUnavailable, `{"error":"starting",`+ + `"message":"This node is still starting and is not serving yet. Try again in a moment.",`+ + `"detail":"This node is recovering what it holds — its store, its broker and its copy of the estate — `+ + `and serves nothing until it has started.",`+ + `"hint":"Retry after the Retry-After: once the node serves, /health stops answering status \"starting\" `+ + `and /ready stops naming reason \"starting\"."}`)(w, r) }) _, err := apply(f, write) var apiErr *engine.APIError @@ -1374,7 +1378,13 @@ func TestFleetLastApplyOfAnUnlistedNodeIsNone(t *testing.T) { // empty fleet: an empty one would read as every seat unplaced. func TestFleetRefusals(t *testing.T) { unavailable := newFakeEngine(t) - unavailable.handle("GET /fleet", answer(http.StatusServiceUnavailable, `{"error":"unavailable","message":"the lease table could not be read"}`)) + // The lease table is the coordination store's, and one that did not + // answer clears by the health tick's five seconds. + unavailable.handle("GET /fleet", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "5") + answer(http.StatusServiceUnavailable, + `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`)(w, r) + }) if got, err := unavailable.client().Fleet(context.Background()); err == nil { t.Errorf("Fleet() of an engine answering 503 = %+v, want an error", got) } else if apiErr, ok := errors.AsType[*engine.APIError](err); !ok || apiErr.StatusCode != http.StatusServiceUnavailable { From 603b286a9550ec2c4325ea41d37dea0eeea6425b Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 21:07:35 +0000 Subject: [PATCH 08/36] test(controller): answer the withdraw test's engine double faithfully The double that stands in for the engine in the secret withdrawal tests said two things the engine never says. Its /health posture was "serving", where the engine's postures are serve, wait, shed, isolated and stuck, and it refused an invalid document with 422, where the engine writes 400 for validation_error. Neither changed what the tests decide, since the client reads the posture through Diverged and a refusal by its code, but a double that disagrees with the engine certifies a client against an engine nobody runs. Answer the posture as engine.PostureServe and the refusal as a 400, as crewlet's main and crewlet#142 alike answer them. Signed-off-by: Ali Yousefian --- internal/controller/config_withdraw_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 9709abfb..3fcf16d0 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -169,11 +169,12 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { } switch { case r.Method == http.MethodGet && r.URL.Path == "/health": - answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": "serving", "applied_epoch": len(e.revisions)}) + answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": engine.PostureServe, + "applied_epoch": len(e.revisions)}) case r.Method == http.MethodGet && r.URL.Path == configRoute: e.readActive(w, active) case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: - answer(http.StatusUnprocessableEntity, map[string]any{"error": "validation_error", "detail": "refused", + answer(http.StatusBadRequest, map[string]any{"error": "validation_error", "detail": "refused", "problems": []map[string]string{{"path": "roles", "kind": "invalid", "message": "refused"}}}) case r.Method == http.MethodPut && r.URL.Path == configRoute && r.URL.Query().Get("dry_run") == "true": answer(http.StatusOK, map[string]any{"base_revision_id": active.id}) From 73af263d46bff9429e5aeef0837333f83de89d1f Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 15:00:32 +0000 Subject: [PATCH 09/36] fix(controller): count a kept data claim at the size it has A Company deleted under spec.deletionPolicy Retain keeps its data volume claim, and a Company created again under its name runs on that claim as it is: its StatefulSet names the same claim, so the pod binds it whatever the new template asks for. dataVolumeSize counted every claim at no less than the template it took for the claim's maker, so where the class had been raised in between (a 12Gi claim kept, the class now 16Gi), the engine's stream was sized for the 16Gi the template asks for, on the 12Gi volume the pod has, which the stream could outgrow. A StatefulSet makes each claim at its template's size, and a claim's request only ever grows, so one asking for less than the template is not the template's: it now counts at what it asks for, or at what it was given if less. Every claim the template made is counted as before. TestDataVolumeSize gains a kept claim under the template, before there is a StatefulSet, rounded up, given less than it asks, not bound yet and expanded past the template; the first five fail with the rule removed. The runtime page says how a kept claim is counted, and that a Company created again over one has its stream sized from it. Signed-off-by: Ali Yousefian --- docs/concepts/runtime.md | 15 ++++++++++----- internal/controller/company_workload.go | 17 ++++++++++++++++- internal/controller/company_workload_test.go | 16 +++++++++++++--- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index 22155f2d..99f4ddbf 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -542,11 +542,14 @@ claim, not the class: The operator reads the size the claim has, the smaller of what it asks for and what it was given, and never less than the size it was made with, so an expansion counts once it has landed and a provisioner that rounded a claim up -moves nothing. The class's `data.storage.size` is not involved, and the -StatefulSet keeps the template it was created with. A StorageClass that -cannot expand leaves only a new volume: take a backup and keep the keyring -Secret with it ([Backups](#backups)), then restore it onto a large enough -volume as Crewlet's backup guide describes. +moves nothing. A claim that asks for less than the StatefulSet's template was +not made from it: it is one a `Retain` deletion kept, which a Company created +again under the same name runs on as it is, so it counts at what it asks for, +or what it was given if less. The class's `data.storage.size` is not +involved, and the StatefulSet keeps the template it was created with. A +StorageClass that cannot expand leaves only a new volume: take a backup and +keep the keyring Secret with it ([Backups](#backups)), then restore it onto a +large enough volume as Crewlet's backup guide describes. The engine migrates its data forward only. The operator records the image the data pod last ran *and was ready on* in `status.engine.image`, and its tag in @@ -824,6 +827,8 @@ operator: A Company created again with the same name and namespace after a `Retain` deletion adopts the kept keyring and finds its volume, and with it its estate. +Its event stream is sized from that volume as the claim has it, whatever its +class gives a new volume now. To discard a retained company for good, delete the claims first and the keyring Secret after them: a Company that finds its claims without its keyring holds its pod back rather than generate a keyring that cannot open diff --git a/internal/controller/company_workload.go b/internal/controller/company_workload.go index 30c6a492..33c45588 100644 --- a/internal/controller/company_workload.go +++ b/internal/controller/company_workload.go @@ -975,6 +975,14 @@ func (r *CompanyReconciler) statefulSetNamed(ctx context.Context, namespace, nam // what its engine reserves (made under an older floor, or before an engine // that keeps more on its stream) had no way out: the template cannot change, // and an expanded claim left the stream as small as it was. +// +// A claim that asks for less than the template is the exception, since the +// template did not make it: a StatefulSet makes each claim at its template's +// size, and a claim's request only ever grows. It is one a Company deleted +// under spec.deletionPolicy Retain kept, which a Company created again under +// its name runs on as it is, whatever its class gives a new volume now, so +// it counts at what it asks for, or at what it was given if less. Counted at +// the template, the stream was sized for a volume the pod does not have. func dataVolumeSize( existing *appsv1.StatefulSet, class *crewletaiv1alpha1.EngineClass, claims []corev1.PersistentVolumeClaim, ) resource.Quantity { @@ -997,7 +1005,14 @@ func dataVolumeSize( size := template requested, asked := claim.Spec.Resources.Requests[corev1.ResourceStorage] granted, bound := claim.Status.Capacity[corev1.ResourceStorage] - if asked && bound { + switch { + case asked && requested.Cmp(template) < 0: + // A kept claim, which the template did not make. + size = requested + if bound && granted.Cmp(size) < 0 { + size = granted + } + case asked && bound: grown := requested if granted.Cmp(grown) < 0 { grown = granted diff --git a/internal/controller/company_workload_test.go b/internal/controller/company_workload_test.go index 21632c45..d74fe289 100644 --- a/internal/controller/company_workload_test.go +++ b/internal/controller/company_workload_test.go @@ -134,11 +134,15 @@ func TestSeatPodStartupSeconds(t *testing.T) { // to the volumes the data pods actually have: the claim template's size (the // existing StatefulSet's, or the class's before there is one), grown by an // expansion of every claim once the expansion has landed, never shrunk by a -// claim given less than it asked for, and the smallest across the pods. +// claim given less than it asked for, and the smallest across the pods; and a +// claim the template did not make, which asks for less than it, at the size +// that claim has. func TestDataVolumeSize(t *testing.T) { t.Parallel() - // kept is the size the existing StatefulSet's claim template asks for. - const kept = "16Gi" + // kept is the size the existing StatefulSet's claim template asks for, + // and older the size of a claim a deleted Company's StatefulSet made from + // a smaller template, which a Company created again under its name runs on. + const kept, older = "16Gi", "12Gi" class := &crewletaiv1alpha1.EngineClass{} class.Spec.Data.Storage.Size = new(resource.MustParse("32Gi")) existing := &appsv1.StatefulSet{} @@ -174,6 +178,12 @@ func TestDataVolumeSize(t *testing.T) { []corev1.PersistentVolumeClaim{bound("64Gi", "64Gi"), bound(kept, kept)}, kept, }, {"a claim being deleted", existing, []corev1.PersistentVolumeClaim{deleting}, kept}, + {"a kept claim under the template", existing, []corev1.PersistentVolumeClaim{bound(older, older)}, older}, + {"a kept claim before there is a StatefulSet", nil, []corev1.PersistentVolumeClaim{bound(older, older)}, older}, + {"a kept claim rounded up", existing, []corev1.PersistentVolumeClaim{bound(older, "14Gi")}, older}, + {"a kept claim given less than it asks", existing, []corev1.PersistentVolumeClaim{bound(older, "10Gi")}, "10Gi"}, + {"a kept claim not bound yet", existing, []corev1.PersistentVolumeClaim{claimRequesting(older)}, older}, + {"a kept claim expanded past the template", existing, []corev1.PersistentVolumeClaim{bound("20Gi", "20Gi")}, "20Gi"}, } { t.Run(c.name, func(t *testing.T) { t.Parallel() From 1f0ab5540ceea3303aa25ca21b2d590272186228 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 00:37:45 +0000 Subject: [PATCH 10/36] fix(controller): come back when a failed withdrawal says it will When the activation a withdrawn value owes fails, SecretWithdrawalFailed says when the operator tries again: engineRetry of the error, which is the engine's own Retry-After (two seconds for an engine that cannot say who is asking), ten seconds for anything without one, and the resync interval for a revision the engine cannot open. The reconcile came back at another time. After a document the engine refused, settleRefused requeued ten seconds on whatever the event said, so a revision the engine cannot open, which no wait clears, was asked about again every ten seconds under an event saying five minutes; and a reconcile with nothing to write came back at the smaller of ten seconds and the resync interval. settle now returns the retry withdrawalFailed reported, zero where nothing failed, and both callers requeue at it: settleRefused as it is, and the reconcile with nothing to write no later than the resync interval; both still look within the engine's pointer poll while a value is kept for a revision other than the fleet's (heldResync). The reconcile after a write keeps its own two-second poll, which comes sooner. The storeEngine double can refuse a reload (reloadDown) as an engine that cannot say who is asking refuses one, with a Retry-After of two seconds. TestARefusedDocumentComesBackWhenItsWithdrawalReportSays holds a refused document whose activation then failed to the two seconds the event promises; it went red with settleRefused's ten seconds put back. Signed-off-by: Ali Yousefian --- internal/controller/config_run.go | 9 +-- internal/controller/config_withdraw.go | 26 ++++---- internal/controller/config_withdraw_test.go | 67 +++++++++++++++++---- 3 files changed, 73 insertions(+), 29 deletions(-) diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index b83cf57b..6ebaa679 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -508,14 +508,15 @@ func (run *configRun) enforce( // below. if active != nil && len(diffs) == 0 && hash == recorded.AppliedHash && len(pushed.Put) == 0 { run.inSync(active) - failed, err := run.settle(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) + again, err := run.settle(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) if err != nil { return ctrl.Result{}, err } - if failed { - return ctrl.Result{RequeueAfter: min(engineRetryInterval, run.resyncInterval())}, nil + next := run.resyncInterval() + if again > 0 { + next = min(again, next) } - return run.heldResync(ctrl.Result{RequeueAfter: run.resyncInterval()}), nil + return run.heldResync(ctrl.Result{RequeueAfter: next}), nil } reloaded, err := run.reloadedBySync(ctx, eng, active, diffs, pushed.Reload) if err != nil { diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index 8c10e2d5..ba9a4d5f 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -8,6 +8,7 @@ import ( "maps" "slices" "strings" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -359,37 +360,34 @@ func (run *configRun) retainedSentence() string { // sync, nil to read it; reload is the engine's own re-activation at the // sync's removals (secretsync.Result.Reload), which reactivate takes for the // activation where it landed. Its error is the API server's; an activation -// that failed is reported (withdrawalFailed), and failed says so, for the -// reconcile to come back engineRetryInterval later. +// that failed is reported (withdrawalFailed), and retry is when that report +// says the operator tries again, zero where nothing failed. func (run *configRun) settle( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, -) (failed bool, err error) { +) (retry time.Duration, err error) { if len(state.Withdrawn()) == 0 { - return false, nil + return 0, nil } next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state) if err := secretsync.SaveState(ctx, run.Client, ref, run.company, next); err != nil { - return false, err + return 0, err } if reactivateErr != nil { - run.withdrawalFailed(ctx, reactivateErr) - return true, nil + return run.withdrawalFailed(ctx, reactivateErr).RequeueAfter, nil } - return false, nil + return 0, nil } // settleRefused is settle for a document the engine refused, as the -// reconcile ends there. +// reconcile ends there: it comes back when the report of an activation that +// failed says it does, and otherwise when an object changes. func (run *configRun) settleRefused( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, ) (ctrl.Result, error) { - failed, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) - if failed { - return ctrl.Result{RequeueAfter: engineRetryInterval}, err - } - return run.heldResync(ctrl.Result{}), err + retry, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) + return run.heldResync(ctrl.Result{RequeueAfter: retry}), err } // withdrawalFailed reports err, which kept the engine from reading its store diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 3fcf16d0..568a4224 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -14,6 +14,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,9 +63,10 @@ import ( // said); runsNone names nodes beside those that refused the fleet's revision // and run no company, as a seat pod that started on nothing and refused it // does; fleetDown makes GET /fleet unanswerable, and missing names the -// revisions GET /config/revisions does not find. configDown and secretsDown -// make it answer a read of the fleet's document, and a write to the store, as -// an engine that cannot decide who is asking does; unreadable, when set, gives +// revisions GET /config/revisions does not find. configDown, secretsDown and +// reloadDown make it answer a read of the fleet's document, a write to the +// store, and a reload of the fleet's revision, as an engine that cannot decide +// who is asking does; unreadable, when set, gives // the body it answers a read of the fleet's document with, as an engine whose // keyring does not open that revision does (config_unreadable_test.go). // @@ -79,6 +81,7 @@ type storeEngine struct { store map[string]string refuse bool configDown, secretsDown bool + reloadDown bool beforeReload, afterReload func(e *storeEngine) afterApply func(e *storeEngine) reloadAnswer *map[string]any @@ -101,13 +104,16 @@ func dataNode(node string) bool { // otherwise. const storeNode = "acme-data-0" -// down reports whether r is a request configDown or secretsDown makes the -// engine refuse: a read of the fleet's document, or a write to the store, one -// name's or a batch's. +// down reports whether r is a request configDown, secretsDown or reloadDown +// makes the engine refuse: a read of the fleet's document, a write to the +// store, one name's or a batch's, or a reload of the fleet's revision. func (e *storeEngine) down(r *http.Request) bool { if r.Method == http.MethodGet { return r.URL.Path == configRoute && e.configDown } + if r.URL.Path == reloadRoute { + return e.reloadDown + } return (r.URL.Path == secretsRoute || strings.HasPrefix(r.URL.Path, secretsRoute+"/")) && e.secretsDown } @@ -143,7 +149,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { defer e.mu.Unlock() body, _ := io.ReadAll(r.Body) answer := func(status int, v any) { answerJSON(w, status, v) } - if r.Method == http.MethodPost && r.URL.Path == "/config/reload" && e.beforeReload != nil { + if r.Method == http.MethodPost && r.URL.Path == reloadRoute && e.beforeReload != nil { hook := e.beforeReload e.beforeReload = nil hook(e) @@ -186,7 +192,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { hook(e) } activated(applied) - case r.Method == http.MethodPost && r.URL.Path == "/config/reload": + case r.Method == http.MethodPost && r.URL.Path == reloadRoute: reloaded := activate(active.document) if e.afterReload != nil { hook := e.afterReload @@ -423,11 +429,13 @@ func (e *storeEngine) activateOther(document []byte) { }) } -// configRoute is the engine's route of the company document, fleetRoute that -// of its fleet, secretsRoute that of its secret store, and dataRole and -// seatsRole the roles of a data pod's node and a seat pod's there. +// configRoute is the engine's route of the company document, reloadRoute the +// one that activates it again, fleetRoute that of its fleet, secretsRoute +// that of its secret store, and dataRole and seatsRole the roles of a data +// pod's node and a seat pod's there. const ( configRoute = "/config" + reloadRoute = "/config/reload" fleetRoute = "/fleet" secretsRoute = "/secrets" dataRole = "data" @@ -1253,3 +1261,40 @@ func TestAReactivationByTheEngineNamesTheRequestThatMadeIt(t *testing.T) { }) } } + +// TestARefusedDocumentComesBackWhenItsWithdrawalReportSays holds a document +// the engine refused, after which the activation owed could not be made, to +// the retry the SecretWithdrawalFailed event promises: here the engine's own +// two seconds. It came back ten seconds on whatever the event said, and a +// revision the engine cannot open, which the event says is looked at again +// at the resync interval, ten seconds on as well. +func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 lets the registry go, and an earlier sync withdrew it: an + // activation without it is owed. + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + // The dry run is refused, and the activation owed then meets an engine + // that cannot say who is asking, for two seconds. + w.engine.refuse = true + w.engine.reloadDown = true + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Warning SecretWithdrawalFailed ") }) + if i < 0 || !strings.Contains(recorded[i], "tries again in 2s") || result.RequeueAfter != 2*time.Second { + t.Errorf("the next look is %s on and events %q; want SecretWithdrawalFailed promising 2s, and 2s", + result.RequeueAfter, recorded) + } +} From c092c44b97c0be0f3eae178f721989e0f90db588 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 02:10:01 +0000 Subject: [PATCH 11/36] fix(controller): record an adopted revision at its own epoch When the engine runs the rendered document as a revision the operator did not activate itself (a reload after the engine minted a credential, or a person's crewlet config reload), the operator adopts it into status.config without a write. It moved revisionID onto the new revision and kept epoch as it was: the epoch of the revision it replaced. verifyApplied takes that epoch for the one to wait for, so a Company read the reload as applied as soon as the engine had applied the revision before it, and its Applied event paired the new revision with the old one's epoch ("applied revision r3 (epoch 2)"). The field says it is the activation counter of the revision it names. GET /config names the revision and no epoch, and /health the epoch a node has applied, which may be another revision's; the fleet's activation pointer, which GET /fleet answers beside its nodes, names both. So the engine client reads it (Fleet.Activation), and the in-sync path records the epoch it names for the adopted revision, from the fleet the reconcile's secret sync has just read (keepServing). The in-sync branch of enforce moves into a helper of its own (keepInSync), which keeps enforce under the cyclomatic bound with the error it now handles. Waiting for that epoch makes the wait's deadline matter: verifyApplied reports ApplyFailed once the engine has not applied the epoch 72 seconds after status.config.appliedAt, and the adoption kept appliedAt as it was, the operator's last write, another revision's and possibly days old. A reload its node had not applied yet read as an engine that does not poll its activation pointer the moment the operator saw it. So an adoption records appliedAt as when the operator found it, the wait counts from there, the field's doc says so, and ApplyFailed says the operator activated the revision or found it activated. A fleet that cannot be read records nothing and fails the reconcile, which reads both again. So does one that answers without its activation (the engine could not read its pointer for the answer), with an error that claims no race, since nothing said another revision activated. GET /config and the fleet's activation both read the fleet's pointer, so an activation naming another revision is a write that activated one between the two reads: a lost race (activationAhead, an ErrRevisionAdvanced), which converge reads again from the start once, as it reads a write that lost. The errors say the operator has not recorded the revision rather than that it did not activate it, which it cannot know. TestARevisionAdoptedInSyncIsRecordedAtItsEpoch fails with the epoch left as it was, with appliedAt left as the last write's, with the activation's revision unchecked, and with a fleet that cannot be read, a missing activation or one naming another revision read as the other; the envtest reload case holds status.config to the reload's own revision and epoch, and the engine suite holds Fleet.Activation to the fleet's revision. TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline fails with appliedAt left as the last write's and with ApplyFailed saying only that the operator activated the revision, and TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument with the second attempt judged against the fleet the first one read. The storeEngine double gains hooks before a read of the document or of the fleet, the activation pointer in its fleet's answer, an activation-less fleet and a node behind on its applies. The applying concept page and the troubleshooting guide say which epoch is recorded and when its wait starts; and the guide's section on ApplyFailed now says it covers a revision the operator adopted as well as one it wrote, which it already did before this commit through the engine's own verdict of a diverged apply. Signed-off-by: Ali Yousefian --- api/v1alpha1/company_types.go | 5 +- config/crd/bases/crewlet.ai_companies.yaml | 6 +- .../templates/crd/companies.crewlet.ai.yaml | 6 +- docs/concepts/applying.md | 49 ++-- docs/guides/troubleshooting.md | 33 +-- docs/reference/api.md | 2 +- internal/controller/config_apply.go | 12 +- internal/controller/config_run.go | 90 ++++++- internal/controller/config_test.go | 9 +- internal/controller/config_withdraw_test.go | 237 +++++++++++++++++- internal/engine/client.go | 7 + internal/engine/client_test.go | 25 ++ internal/engine/engine_test.go | 15 +- internal/engine/types.go | 21 ++ 14 files changed, 450 insertions(+), 67 deletions(-) diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index f65492ac..59b12bf8 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -1460,7 +1460,10 @@ type ConfigStatus struct { // +kubebuilder:validation:MaxLength=128 AppliedHash string `json:"appliedHash,omitempty"` - // appliedAt is when the operator last applied a revision. + // appliedAt is when the operator activated the revision this records, + // or found the engine running it where it adopted a revision of the + // rendered document it did not write (a reload): what the operator's + // wait for the engine to apply its epoch counts from. // +optional AppliedAt *metav1.Time `json:"appliedAt,omitempty"` diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index 047a7951..e8dbb944 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -1730,7 +1730,11 @@ spec: minProperties: 1 properties: appliedAt: - description: appliedAt is when the operator last applied a revision. + description: |- + appliedAt is when the operator activated the revision this records, + or found the engine running it where it adopted a revision of the + rendered document it did not write (a reload): what the operator's + wait for the engine to apply its epoch counts from. format: date-time type: string appliedHash: diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index 5fd22561..833cb662 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -1733,7 +1733,11 @@ spec: minProperties: 1 properties: appliedAt: - description: appliedAt is when the operator last applied a revision. + description: |- + appliedAt is when the operator activated the revision this records, + or found the engine running it where it adopted a revision of the + rendered document it did not write (a reload): what the operator's + wait for the engine to apply its epoch counts from. format: date-time type: string appliedHash: diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index 2ea9186b..ffb06c92 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -310,7 +310,8 @@ sequenceDiagram > `stuck` only once it has concluded that it did not: an attempt ended in > failure, or the apply has run past the engine's 45-second grace while a peer > applied the epoch. On such a posture, or on `wait` still 72 seconds after the -> write (four of its 18-second polls), the Company reports +> write (four of its 18-second polls), or after the operator found a revision it +> adopts (below), the Company reports > `ConfigApplied=False ApplyFailed` with the engine's status, posture and > account of the failure (from its record of the attempt in `/fleet`), and an > `ApplyFailed` event. The fleet's revision stays the one the write activated @@ -398,7 +399,15 @@ write. A revision the engine activates on its own with the same content — a reload after a credential was minted — is not drift: the operator adopts its id in -`status.config.revisionID` and writes nothing. +`status.config.revisionID`, and in `status.config.epoch` the epoch the +fleet's activation names for it (`GET /fleet`), and writes nothing. It +reports `ConfigApplied=True` once the engine has applied that epoch, not the +epoch of the revision it replaced, and `ApplyFailed` when the engine has not +72 seconds after the operator found it, which `status.config.appliedAt` +records: the operator's last write is another revision's, and may be days +old. A fleet whose activation names another revision by then had another +write activate one between the two reads, and the operator records nothing and +starts over once from a fresh read, as it does a write that lost its race. The comparison is over what the engine serves back, which is its own encoding of the document rather than the bytes it was given. It disregards the @@ -533,24 +542,24 @@ takes no work for a person that a pause could hold. Besides the conditions: - **Company** `status.config`: the revision the engine runs the rendered - document as, its activation epoch, `appliedHash`, when the operator last - applied, and how many objects of each kind it was rendered from (platform - model providers included). `status.config.placement` is where the document - the engine runs pins its agent seats: every value of `crewlet.ai/pool` - (`data` and the Shared pools that run a seat) and of `crewlet.ai/seat` (each - seat of a PerSeat pool). It is read from the engine's own document under - every policy, so under Observe and Seed too. A new revision's pins join - the record before the operator's write that activates it, and while the - engine applies it the record is the previous revision's pins and the new - one's together, because the engine keeps running the previous revision - until it has applied the new one, and for good if that apply fails. Once - the engine reports the new revision's epoch applied, the record is its - pins alone; a document the engine owns (Observe, Seed) is narrowed to only - once the engine's own record of its last apply names that revision and - says it ended well. - The runtime reconciler reads it to keep a seat pool's pods running for as - long as the running revision pins a seat to them ([A pool removed while - seats still run in it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). + document as, its activation epoch, `appliedHash`, when the operator + activated it or found it running (`appliedAt`), and how many objects of each + kind it was rendered from (platform model providers included). + `status.config.placement` is where the document the engine runs pins its + agent seats: every value of `crewlet.ai/pool` (`data` and the Shared pools + that run a seat) and of `crewlet.ai/seat` (each seat of a PerSeat pool). It + is read from the engine's own document under every policy, so under Observe + and Seed too. A new revision's pins join the record before the operator's + write that activates it, and while the engine applies it the record is the + previous revision's pins and the new one's together, because the engine + keeps running the previous revision until it has applied the new one, and + for good if that apply fails. Once the engine reports the new revision's + epoch applied, the record is its pins alone; a document the engine owns + (Observe, Seed) is narrowed to only once the engine's own record of its last + apply names that revision and says it ended well. The runtime reconciler + reads it to keep a seat pool's pods running for as long as the running + revision pins a seat to them ([A pool removed while seats still run in + it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). - **Seat**: its handle, its identity in the engine (`seatID`, derived from the company name and the handle), the pool that runs it and the StatefulSet whose pods do (`status.placement.workload`; unset while the diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 75c7a099..1d7c9987 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -117,7 +117,7 @@ with your last change yet. | | False | `PoolCapacityExceeded` | The company's seat pods would number more than its class's `seatPools.maxPods`: its Shared pools' replicas alone (the message says so; lower them), or with the pods of its PerSeat seats, each of which reports it too. | | | False | `Invalid` | The engine's dry run refused the document, in its own words, every problem in this message. A problem at a place a part rendered (a Seat, Unit, LLMProvider, MCPServer, Integration or Schedule) is also reported on that part, as `Applied=False Invalid`; one at a place the Company renders itself (its own fields, `spec.embeddings`, `spec.sandbox` and its setup steps) is in this message alone. | | | False | `Drifted` | Under `Observe`, the engine runs another document; the message names the places that differ. | -| | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write. | +| | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write, or after the operator found it running where it adopts a revision it did not write. | | | False | `ApplyFailed` | The engine's first data pod, the one the operator reaches, did not apply the revision: it keeps running the one it ran before, and starts on the one its volume names if it restarts, the same one unless the engine is still moving it (the message says which), and the engine's secret store keeps every value of either meanwhile. The message carries the engine's status, its posture (`isolated` on a single node, `shed` or `stuck` beside peers that applied it) and its own account of the failure; see [The engine did not apply a revision](#the-engine-did-not-apply-a-revision). | | | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine asked to be asked again later (`503`: it is starting, or cannot read the coordination store that holds the fleet's revision). Retried every ten seconds, or sooner when the engine says when; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | @@ -474,18 +474,21 @@ kubectl get events -n acme --field-selector involvedObject.name=acme,reason=Revi ## The engine did not apply a revision -`ConfigApplied=False ApplyFailed` means the engine stored and activated the -revision the operator wrote and then could not apply it, or never did. The -revision stays the fleet's: `GET /config` answers it, and every pod tries it -again. A pod that did not apply it keeps serving the revision it ran before, -and a data pod starts on the revision its volume names if it restarts, that -same one unless the engine is still moving it (a seat pod starts on nothing -and tries the fleet's revision again); the dry run accepted the new one, -so what failed is something only applying it meets on this engine: a -subsystem it could not start (Worklet, the code sandbox, the inbound -edge), or a rule that needs the data pod's own configuration. The -condition's message carries the engine's account of the failure, its -`status` and its `posture`: +`ConfigApplied=False ApplyFailed` means the engine activated a revision of +the rendered document and then could not apply it, or never did: the revision +the operator wrote, or one with the same document it found running and +adopted without a write (a person's `crewlet config reload`, or the engine's +own reload after it minted a credential). The revision stays the fleet's: +`GET /config` answers it, and every pod tries it again. A pod that did not +apply it keeps serving the revision it ran before, and a data pod starts on +the revision its volume names if it restarts, that same one unless the +engine is still moving it (a seat pod starts on nothing and tries the +fleet's revision again); the dry run accepted the document, so what failed +is something only applying it meets on this engine: a subsystem it could not +start (Worklet, the code sandbox, the inbound edge), a value it resolves +when the revision activates, or a rule that needs the data pod's own +configuration. The condition's message carries the engine's account of the +failure, its `status` and its `posture`: - `isolated`: the only node, or every node, failed the revision. It keeps serving and stays ready, and retries the revision three times at most. @@ -544,7 +547,9 @@ has neither applied the revision nor concluded that it failed within 72 seconds (its posture still reads `wait`), the engine is not polling its activation pointer at all, or an apply hangs with no peer to compare it against; its log says which, and a restart of the data pod is the usual -remedy. +remedy. The 72 seconds run from `status.config.appliedAt`: when the operator +activated the revision, or found one it adopted running, never from its last +write of another revision, which may be days old. ```bash kubectl logs acme-data-0 -n acme | grep -E 'config_apply_failed|reconcile_tick_failed' | tail diff --git a/docs/reference/api.md b/docs/reference/api.md index ec0aa653..0e6fc291 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -761,7 +761,7 @@ _Appears in:_ | `revisionID` _string_ | revisionID is the engine's id of the last revision the operator saw
run the rendered document: the one it applied, one the engine
activated with the same content (a reload), or, under Observe, the
one it found in sync. Under Seed it is the revision the document was
seeded as. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| | `epoch` _integer_ | epoch is the engine's activation counter for that revision. The
engine applies a revision asynchronously; it runs it once the applied
epoch its /health reports has reached this. | | Minimum: 0
Optional: \{\}
| | `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as last applied: a change to
either is applied as a new revision, since the engine reads a
rotated value only when a revision activates. The values enter it
only as keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| -| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator last applied a revision. | | Optional: \{\}
| +| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator activated the revision this records,
or found the engine running it where it adopted a revision of the
rendered document it did not write (a reload): what the operator's
wait for the engine to apply its epoch counts from. | | Optional: \{\}
| | `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from. | | MinProperties: 1
Optional: \{\}
| | `placement` _[RevisionPlacement](#revisionplacement)_ | placement is where the revision the engine runs pins its agent seats,
read from the engine's document under every management policy. The
operator keeps a seat pool's pods running while the running revision
pins a seat to them, even after the pool or the seat's placement is
gone from the spec: a pod stopped while its seat is still pinned to
it would leave the seat held by nobody, and a blocked document keeps
the last good revision running. Once a revision that pins nothing to
them runs, the pods are deleted. | | MinProperties: 1
Optional: \{\}
| diff --git a/internal/controller/config_apply.go b/internal/controller/config_apply.go index 5d786bcf..a58f408f 100644 --- a/internal/controller/config_apply.go +++ b/internal/controller/config_apply.go @@ -42,7 +42,9 @@ const enginePollInterval = 18 * time.Second const engineLagGrace = 45 * time.Second // applyDeadline is how long after a write the operator waits for an engine -// whose posture still reads wait. A node reports wait for the whole of an +// whose posture still reads wait, or after it first finds the engine running +// a revision of the rendered document it did not write, which it adopts +// (inSync). A node reports wait for the whole of an // apply, and turns serve when it succeeds or a diverged posture when an // attempt fails, so wait past this is one of two things the engine itself // never concludes: it is not polling its activation pointer at all, or an @@ -115,7 +117,8 @@ func (r *CompanyConfigReconciler) clock() time.Time { // reports about it: True once the engine has applied the epoch status.config // records and has not diverged since, Applying while its posture reads wait, // and ApplyFailed when its posture reads diverged, or still wait -// applyDeadline after the write. It runs on every look at the engine, the +// applyDeadline after the write, or after the operator found the revision it +// adopted (status.config.appliedAt). It runs on every look at the engine, the // periodic resync included, so an engine that diverges after applying the // operator's revision, on one someone activates later, is reported too. // health and healthErr are what readHealth read; result is what the @@ -156,8 +159,9 @@ func (run *configRun) verifyApplied( return result case config.AppliedAt != nil && run.clock().Sub(config.AppliedAt.Time) >= applyDeadline: run.applyFailed(config.RevisionID, fmt.Sprintf("The engine has not applied revision %s (epoch %d) %s after the "+ - "operator activated it, and has not concluded that it failed: it reports status %q and posture %q at applied "+ - "epoch %d, and keeps running that. It is not polling its activation pointer, or an apply hangs; its log says which.", + "operator activated it or found it activated, and has not concluded that it failed: it reports status %q and "+ + "posture %q at applied epoch %d, and keeps running that. It is not polling its activation pointer, or an apply "+ + "hangs; its log says which.", config.RevisionID, target, applyDeadline, health.Status, health.Posture, health.AppliedEpoch)) return ctrl.Result{RequeueAfter: engineRetryInterval} } diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 6ebaa679..10ffedd5 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -412,6 +412,26 @@ func (run *configRun) converge(ctx context.Context, eng *engine.Client) (ctrl.Re } } +// activationAhead is inSync's answer where the fleet's activation names a +// revision other than the one GET /config answered with: both read the +// fleet's activation pointer, so another write activated a revision between +// the two reads. It is a lost race (engine.ErrRevisionAdvanced), which +// converge reads again from the start once, as it reads a write that lost. +type activationAhead struct { + // read is the revision GET /config answered with. + read string + // activation is the fleet's. + activation engine.Activation +} + +func (e *activationAhead) Error() string { + return fmt.Sprintf("the fleet runs the rendered document as revision %s, which the operator has not recorded, "+ + "and its activation named revision %s (epoch %d) by the time the operator read the fleet: another write "+ + "activated a revision between the two reads", e.read, e.activation.RevisionID, e.activation.Epoch) +} + +func (e *activationAhead) Unwrap() error { return engine.ErrRevisionAdvanced } + // activeConfig is the fleet's active configuration, the revision the // engine's activation pointer names, which every node answers whatever it // runs itself (keepServing reads that), or nil when it has none yet. @@ -507,16 +527,7 @@ func (run *configRun) enforce( // written or removes (reloadedBySync), or else the operator's apply // below. if active != nil && len(diffs) == 0 && hash == recorded.AppliedHash && len(pushed.Put) == 0 { - run.inSync(active) - again, err := run.settle(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) - if err != nil { - return ctrl.Result{}, err - } - next := run.resyncInterval() - if again > 0 { - next = min(again, next) - } - return run.heldResync(ctrl.Result{RequeueAfter: next}), nil + return run.keepInSync(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) } reloaded, err := run.reloadedBySync(ctx, eng, active, diffs, pushed.Reload) if err != nil { @@ -804,15 +815,67 @@ func diffWith(doc *render.Document, active *engine.ActiveConfig) ([]render.Path, return render.Diff(doc.YAML, active.Document) } +// keepInSync is enforce for an engine that runs the rendered document with +// every value it references: it records the revision (inSync), makes the +// activation the secret sync's state owes (settle), and comes back at the +// resync, or sooner where that activation failed and its report says so, or +// a value is kept for a revision other than the fleet's (heldResync). reload +// and reloadAction are the sync's, as settle takes them. +func (run *configRun) keepInSync( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, + state secretsync.State, ref types.NamespacedName, +) (ctrl.Result, error) { + if err := run.inSync(ctx, eng, active); err != nil { + return ctrl.Result{}, err + } + again, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) + if err != nil { + return ctrl.Result{}, err + } + next := run.resyncInterval() + if again > 0 { + next = min(again, next) + } + return run.heldResync(ctrl.Result{RequeueAfter: next}), nil +} + // inSync records an engine that runs the rendered document, with every // value it references, without a write: possibly as a revision the operator -// did not write itself, such as a reload of the same document, which it -// adopts. -func (run *configRun) inSync(active *engine.ActiveConfig) { +// has not recorded, such as a reload of the same document, which it adopts at +// the epoch the fleet's activation names for it, and from now: the engine is +// held to applying that epoch applyDeadline after the operator found it, not +// after the operator's last write, another revision's, which may be days old. +// It fails where the fleet cannot be read or answers without its activation, +// and with an *activationAhead, a lost race (engine.ErrRevisionAdvanced), +// where the activation names another revision: converge reads again from the +// start. +func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) error { recorded := run.company.Status.Config if recorded.RevisionID != active.RevisionID { + // AT THE EPOCH IT ACTIVATED AS, which only the fleet's activation + // says: the one recorded is the revision it replaces, so the + // engine read as having applied it once it applied that one, + // before it had applied this, and the Applied event paired this + // revision with the other's epoch. + fleet, err := run.fleetOf(ctx, eng) + if err != nil { + return fmt.Errorf("the engine runs the rendered document as revision %s, which the operator has not recorded, "+ + "and its fleet, which says as what epoch it activated, could not be read: %w", active.RevisionID, err) + } + switch { + case fleet.Activation == nil: + // The read of the document named a revision, so one is active: + // the engine answered its fleet without reading the pointer. + return fmt.Errorf("the engine runs the rendered document as revision %s, which the operator has not recorded, "+ + "and its fleet answered without its activation, which says as what epoch it activated", active.RevisionID) + case fleet.Activation.RevisionID != active.RevisionID: + return &activationAhead{read: active.RevisionID, activation: *fleet.Activation} + } config := recorded.DeepCopy() config.RevisionID = active.RevisionID + config.Epoch = new(fleet.Activation.Epoch) + found := metav1.NewTime(run.clock()) + config.AppliedAt = &found config.Objects = run.objectCounts() run.config = config } @@ -830,6 +893,7 @@ func (run *configRun) inSync(active *engine.ActiveConfig) { } } run.applied = appliedVerdict{kind: appliedInRevision, revision: active.RevisionID} + return nil } // recordApply records a revision the operator wrote. diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go index 4888ab82..46d8b713 100644 --- a/internal/controller/config_test.go +++ b/internal/controller/config_test.go @@ -1097,10 +1097,17 @@ var _ = Describe("Company config", Ordered, func() { applied(c, crewletaiv1alpha1.ReasonApplied) By("a person activating the running document again") - _, err := engine.New(e.URL, e.AccessToken).ReloadConfig(ctx, "reloaded by a person") + reloaded, err := engine.New(e.URL, e.AccessToken).ReloadConfig(ctx, "reloaded by a person") Expect(err).NotTo(HaveOccurred()) Eventually(count).WithTimeout(settle).Should(Equal(3)) Consistently(count).WithTimeout(3 * quietPeriod).Should(Equal(3)) + Eventually(func(g Gomega) crewletaiv1alpha1.ConfigStatus { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(c), c)).To(Succeed()) + return c.Status.Config + }).WithTimeout(settle).Should(SatisfyAll( + HaveField("RevisionID", reloaded.RevisionID), + HaveField("Epoch", HaveValue(Equal(reloaded.Epoch))), + ), "the reload is adopted at its own epoch, not at the epoch of the revision it replaced") By("deleting the key's Secret") Expect(k8sClient.Delete(ctx, &corev1.Secret{Namespace: namespace, Name: name + "-npm"})).To(Succeed()) diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 568a4224..5dae94d8 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -47,7 +48,9 @@ import ( // it is to refuse is refused as the engine refuses an invalid one. // beforeReload, afterReload and afterApply, when set, run once, under its // lock, just before a reload is answered and just after a reload or an apply -// is: somebody else's write landing between two of the operator's. +// is: somebody else's write landing between two of the operator's. Its fleet +// answers the activation pointer beside its nodes: the fleet's revision at +// its place in the history, which is its epoch. // // A write into its store (PUT /secrets, PUT /secrets/{name}), and a removal // of a value (DELETE /secrets/{name}), re-activates the fleet's revision when its @@ -92,6 +95,15 @@ type storeEngine struct { fleetDown bool missing map[string]bool unreadable func(revision string) string + // beforeConfig and beforeFleet run once, before the engine answers the + // next read of the fleet's document or of its fleet: an activation + // somebody else makes in that moment. + beforeConfig, beforeFleet func(e *storeEngine) + // noActivation makes /fleet answer without its activation pointer, as + // an engine that could not read it for the answer does; appliedLag is + // how many epochs /health reports the node behind the latest. + noActivation bool + appliedLag int } // dataNode reports whether node is a pod of the Company's data StatefulSet, @@ -149,11 +161,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { defer e.mu.Unlock() body, _ := io.ReadAll(r.Body) answer := func(status int, v any) { answerJSON(w, status, v) } - if r.Method == http.MethodPost && r.URL.Path == reloadRoute && e.beforeReload != nil { - hook := e.beforeReload - e.beforeReload = nil - hook(e) - } + e.before(r) active := e.revisions[len(e.revisions)-1] activate := func(document []byte) storeRevision { revision := storeRevision{id: fmt.Sprintf("r%d", len(e.revisions)+1), document: document, summary: r.Header.Get("X-Summary")} @@ -175,8 +183,12 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { } switch { case r.Method == http.MethodGet && r.URL.Path == "/health": - answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": engine.PostureServe, - "applied_epoch": len(e.revisions)}) + posture := engine.PostureServe + if e.appliedLag > 0 { + posture = engine.PostureWait + } + answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": posture, + "applied_epoch": len(e.revisions) - e.appliedLag}) case r.Method == http.MethodGet && r.URL.Path == configRoute: e.readActive(w, active) case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: @@ -350,7 +362,8 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st for _, node := range slices.Sorted(maps.Keys(e.runsNone)) { nodes = append(nodes, row(node, len(e.revisions), active.id, engine.ApplyError, "")) } - answerJSON(w, http.StatusOK, map[string]any{"this_node": storeNode, "nodes": nodes, "target_epoch": len(e.revisions)}) + answerJSON(w, http.StatusOK, map[string]any{"this_node": storeNode, "nodes": nodes, "target_epoch": len(e.revisions), + "activation": e.activation()}) case isRevision: i := slices.IndexFunc(e.revisions, func(rev storeRevision) bool { return rev.id == revisionID }) if i < 0 || e.missing[revisionID] { @@ -370,6 +383,43 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st return true } +// before runs the hook, if one is set, that r's route takes before the engine +// answers it, once: beforeReload, beforeConfig or beforeFleet. +func (e *storeEngine) before(r *http.Request) { + var hook *func(e *storeEngine) + switch { + case r.Method == http.MethodPost && r.URL.Path == reloadRoute: + hook = &e.beforeReload + case r.Method == http.MethodGet && r.URL.Path == configRoute: + hook = &e.beforeConfig + case r.Method == http.MethodGet && r.URL.Path == fleetRoute: + hook = &e.beforeFleet + default: + return + } + if run := *hook; run != nil { + *hook = nil + run(e) + } +} + +// activationAnswer is the activation pointer as GET /fleet answers it. +type activationAnswer struct { + RevisionID string `json:"revision_id"` + Epoch int `json:"epoch"` +} + +// activation is the activation pointer the engine answers beside its nodes: +// the fleet's revision at the epoch it activated as, which is its place in +// the history, nil where noActivation says the engine could not read it. The +// caller holds e's lock. +func (e *storeEngine) activation() *activationAnswer { + if e.noActivation { + return nil + } + return &activationAnswer{RevisionID: e.revisions[len(e.revisions)-1].id, Epoch: len(e.revisions)} +} + // answerJSON writes v as the JSON answer, with status. func answerJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json") @@ -1298,3 +1348,172 @@ func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { result.RequeueAfter, recorded) } } + +// TestARevisionAdoptedInSyncIsRecordedAtItsEpoch pins that a revision of the +// rendered document the operator did not activate, a reload of its document +// here, is recorded at the epoch the fleet's activation names for it, and as +// found now (appliedAt), which the wait for the engine to apply it counts +// from. Recorded at the epoch of the revision it replaced, the engine read as +// having applied it once it applied that one, and the Applied event paired it +// with that epoch. A fleet that cannot be read, or that answers without its +// activation, records nothing and claims no race: the reconcile fails and the +// next one reads both again. A fleet whose activation names another revision +// records nothing either, and is a lost race (engine.ErrRevisionAdvanced), +// which converge reads again from the start. +func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { + ctx := context.Background() + found := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + w := newWithdrawal(t, runningLetsGo) + company := w.run.company + company.Status.Config.Epoch = new(int64(1)) + company.Status.Config.AppliedAt = &metav1.Time{Time: found.Add(-6 * time.Hour)} + w.again(t, company) + w.run.now = func() time.Time { return found } + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + active, err := activeConfig(ctx, w.client) + if err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active); err != nil { + t.Fatal(err) + } + config := w.statusWritten().Status.Config + if config.RevisionID != "r2" || config.Epoch == nil || *config.Epoch != 2 { + epoch := "none" + if config.Epoch != nil { + epoch = strconv.FormatInt(*config.Epoch, 10) + } + t.Errorf("status.config = revision %s at epoch %s; want r2 at its own epoch, 2", config.RevisionID, epoch) + } + if config.AppliedAt == nil || !config.AppliedAt.Equal(&metav1.Time{Time: found}) { + t.Errorf("status.config.appliedAt = %v, want when the operator found r2, %s, not its last write", config.AppliedAt, found) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.fleetDown = true + w.engine.mu.Unlock() + if err := w.run.inSync(ctx, w.client, active); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + w.run.config != nil { + t.Errorf("inSync over a fleet that cannot be read = %v, recording %+v; want an error that claims no race and "+ + "nothing recorded", err, w.run.config) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.fleetDown, w.engine.noActivation = false, true + w.engine.mu.Unlock() + if err := w.run.inSync(ctx, w.client, active); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + strings.Contains(err.Error(), "instead") || w.run.config != nil { + t.Errorf("inSync over a fleet answering without its activation = %v, recording %+v; want an error that claims no "+ + "race and nothing recorded", err, w.run.config) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.noActivation = false + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded again"); err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active); !errors.Is(err, engine.ErrRevisionAdvanced) || w.run.config != nil { + t.Errorf("inSync of r2 while the fleet's activation names r3 = %v, recording %+v; want a lost race "+ + "(ErrRevisionAdvanced) and nothing recorded", err, w.run.config) + } +} + +// TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline pins that the wait for the +// engine to apply a revision the operator adopts counts from when it found +// it, not from the operator's last write, another revision's: a reload the +// engine has not applied yet is Applying however old that write is, with no +// ApplyFailed event, and ApplyFailed only applyDeadline after the operator +// found it, in words that do not say the operator activated it. +func TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline(t *testing.T) { + ctx := context.Background() + found := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + w := newWithdrawal(t, runningLetsGo) + company := w.run.company + company.Status.Config.Epoch = new(int64(1)) + company.Status.Config.AppliedAt = &metav1.Time{Time: found.Add(-6 * time.Hour)} + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.appliedLag = 1 + w.engine.mu.Unlock() + look := func(at time.Time) { + t.Helper() + w.run.now = func() time.Time { return at } + active, err := activeConfig(ctx, w.client) + if err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active); err != nil { + t.Fatal(err) + } + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + } + + w.again(t, company) + look(found) + if got := w.run.configApplied; got.Reason != crewletaiv1alpha1.ReasonApplying { + t.Fatalf("ConfigApplied of a reload the engine has not applied, six hours after the operator's last write = %+v, "+ + "want Applying", got) + } + if warned(drained(w.events), crewletaiv1alpha1.ReasonApplyFailed) { + t.Error("an ApplyFailed event for a reload the operator found a moment ago") + } + + company = w.statusWritten() + meta.SetStatusCondition(&company.Status.Conditions, *w.run.configApplied) + w.again(t, company) + look(found.Add(applyDeadline)) + if got := w.run.configApplied; got.Reason != crewletaiv1alpha1.ReasonApplyFailed || + !strings.Contains(got.Message, "found it activated") { + t.Errorf("ConfigApplied of that reload %s after the operator found it = %+v, want ApplyFailed saying the operator "+ + "found it activated", applyDeadline, got) + } +} + +// TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument pins that inSync +// judges the revision an attempt of converge read against a fleet read after +// that read, in the same attempt (keepServing reads it afresh). An attempt +// whose fleet names a later revision than its document read starts over (a +// lost race); judged against the answer the first attempt read, the second +// attempt's read, later again, would look like a second lost race, and the +// reconcile would fail over an engine that was fine. +func TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Fatalf("revisions after the write = %v, want [r1 r2]", got) + } + w.again(t, w.statusWritten()) + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + reloadOnce := func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { + reloadOnce(e) + e.beforeConfig = reloadOnce + } + w.engine.mu.Unlock() + + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the second attempt to adopt the revision it read", err) + } + config := w.statusWritten().Status.Config + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3", "r4", "r5"}) { + t.Errorf("revisions = %v, want [r1 r2 r3 r4 r5]: nothing written, only the reloads", got) + } + if config.RevisionID != "r5" || config.Epoch == nil || *config.Epoch != 5 { + t.Errorf("status.config = %+v, want r5 at epoch 5, the revision the second attempt read", config) + } +} diff --git a/internal/engine/client.go b/internal/engine/client.go index dcf28522..6e7e396c 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -230,11 +230,18 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { } `json:"unplaceable"` UnmannedRoles []string `json:"unmanned_roles"` TargetEpoch int64 `json:"target_epoch"` + Activation *struct { + RevisionID string `json:"revision_id"` + Epoch int64 `json:"epoch"` + } `json:"activation"` } if err := c.call(ctx, request{method: http.MethodGet, path: "/fleet"}, http.StatusOK, &answer); err != nil { return Fleet{}, err } f := Fleet{ThisNode: answer.ThisNode, UnmannedRoles: answer.UnmannedRoles, TargetEpoch: answer.TargetEpoch} + if a := answer.Activation; a != nil && a.RevisionID != "" { + f.Activation = &Activation{RevisionID: a.RevisionID, Epoch: a.Epoch} + } for i := range answer.Nodes { n := &answer.Nodes[i] f.Nodes = append(f.Nodes, FleetNode{ diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index e0e61c0a..c373ae3e 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -1309,6 +1309,31 @@ func TestFleetReadsNodesAndSeatLeases(t *testing.T) { }); record != want || !record.Status.Failed() { t.Errorf("LastApply() = %+v, want %+v, failed", record, want) } + want := engine.Activation{RevisionID: "r7", Epoch: 7} + if a := got.Activation; a == nil || *a != want { + t.Errorf("Activation = %+v, want %+v", a, want) + } +} + +// TestFleetWithNoActivationNamesNone pins that a fleet whose activation +// pointer names nothing, which the engine answers as null both when nothing +// has been activated and when it could not read the pointer, has no +// Activation rather than one naming the empty revision. +func TestFleetWithNoActivationNamesNone(t *testing.T) { + for _, body := range []string{ + `{"this_node":"node-0","nodes":[],"target_epoch":0,"activation":null}`, + `{"this_node":"node-0","nodes":[],"target_epoch":0}`, + } { + f := newFakeEngine(t) + f.handle("GET /fleet", answer(http.StatusOK, body)) + got, err := f.client().Fleet(context.Background()) + if err != nil { + t.Fatal(err) + } + if got.Activation != nil { + t.Errorf("Fleet() of %s has Activation %+v, want none", body, got.Activation) + } + } } // TestRevisionReadsTheStoredDocument pins the read of one stored revision, diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index ca097ea9..aef218fa 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -234,8 +234,9 @@ func TestARecordOfTheApplyIsWaitedForAndAWrongOneFails(t *testing.T) { // TestEngineFleetNamesTheSeatsHolder pins the read of /fleet against the // engine: the answering node, its roles, broker and labels, the seat it holds -// because its labels match the seat's placement, and a seat whose placement -// no node matches, which no node holds and the engine calls unplaceable. +// because its labels match the seat's placement, a seat whose placement no +// node matches, which no node holds and the engine calls unplaceable, and the +// fleet's activation, which names the fleet's revision at its epoch. func TestEngineFleetNamesTheSeatsHolder(t *testing.T) { t.Parallel() ctx := context.Background() @@ -271,6 +272,16 @@ func TestEngineFleetNamesTheSeatsHolder(t *testing.T) { t.Errorf("bo: held %t, unplaceable %v; want held by nobody and unplaceable, as no node is labelled for its pool", held, fleet.Unplaceable) } + // The fleet's activation names the revision GET /config answers, at its + // epoch: the one read that says which epoch a revision the operator did + // not activate itself activated as. + active, err := client.ActiveConfig(ctx) + if err != nil { + t.Fatal(err) + } + if a := fleet.Activation; a == nil || a.RevisionID != active.RevisionID || a.Epoch < 1 { + t.Errorf("Fleet().Activation = %+v, want the fleet's revision %s at its epoch", a, active.RevisionID) + } } // TestEngineApplyActivatesTheRevision pins the write's answer to what the engine diff --git a/internal/engine/types.go b/internal/engine/types.go index ae42a4c6..bfd0b65c 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -481,6 +481,27 @@ type Fleet struct { // TargetEpoch is the activation epoch the fleet converges on, zero when // nothing has been activated or the engine could not read it. TargetEpoch int64 + + // Activation is the fleet's activation pointer as the engine read it: + // the revision it names and that revision's activation epoch, read in + // one read with TargetEpoch. Nil when nothing has been activated or the + // engine could not read the pointer, which it does not tell apart. + // + // It is the one answer to which epoch a revision the operator did not + // activate itself, such as a reload of its document, activated as: + // GET /config names the revision and not its epoch, and /health the + // epoch a node has applied, which may be another revision's. + Activation *Activation +} + +// Activation is one activation of the fleet's company document. +type Activation struct { + // RevisionID is the revision the activation made active. + RevisionID string + + // Epoch is the activation's epoch: every engine has applied the + // revision once its applied epoch reaches it. + Epoch int64 } // Holder returns the lease of the seat with handle and the id the engine From 5e32db0052fc0cc69d6ec339c7a4cb199f9ea984 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 02:48:32 +0000 Subject: [PATCH 12/36] test(engine): pin a reload that loses its race as ErrRevisionAdvanced ReloadConfig's doc says a reload is a compare-and-set against the fleet's activation pointer, so a write landing inside the request fails it with ErrRevisionAdvanced, as any write that lost a race does (crewlet answers 409 revision_advanced, naming the revision it stored and the one that won). Nothing pinned that mapping for a reload: a reactivation that fails leaves its withdrawal owed and is reported, and the next reconcile decides again by the revision then active, so a reload read as another failure would go unnoticed. TestAReloadThatLosesItsRaceIsRevisionAdvanced pins it. Signed-off-by: Ali Yousefian --- internal/engine/client_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index c373ae3e..2e1e4d54 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -1384,6 +1384,19 @@ func TestActiveConfigWhileTheFleetCannotBeRead(t *testing.T) { } } +// TestAReloadThatLosesItsRaceIsRevisionAdvanced pins that a reload, which +// names no revision, is a write that races others: one that loses its +// activation to another write is ErrRevisionAdvanced, as the engine answers it +// (409 revision_advanced, naming the revision it stored and the one that won). +func TestAReloadThatLosesItsRaceIsRevisionAdvanced(t *testing.T) { + f := newFakeEngine(t) + f.handle("POST /config/reload", answer(http.StatusConflict, + `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`)) + if _, err := f.client().ReloadConfig(context.Background(), "a reload that loses"); !errors.Is(err, engine.ErrRevisionAdvanced) { + t.Errorf("ReloadConfig() = %v, want ErrRevisionAdvanced", err) + } +} + // TestFleetLastApplyOfAnUnlistedNodeIsNone pins that a node the fleet does // not list yet has recorded nothing, rather than another node's record. func TestFleetLastApplyOfAnUnlistedNodeIsNone(t *testing.T) { From dcf729c551a4d0f77abbff6a02c1ecbe2f8a6388 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 30 Oct 2026 16:42:25 +0100 Subject: [PATCH 13/36] fix(controller): owe a store change until a revision activates after it The engine reads its secret store only when a revision activates. When the sync writes back a value somebody overwrote, or one the store lost, the fingerprints stay as they were, and so does the hash of what the operator applied, but a revision activated since the overwrite may run the other value. The in-sync check caught that only through the sync of the same reconcile: it required that this reconcile's sync wrote nothing. A reconcile that wrote a value back and then did not get its write of the document through (a dry run the engine could not answer, a write that failed, a write that lost its race and the retry whose sync then had nothing left to write) left nothing behind that said so. The next look found the document, its values and the hash all matching, and took the engine for in sync while it went on running a revision that read the store before the write-back, until the next Secret change. So the state records it, as it records a withdrawal: every name Sync writes a value under stays in State.Written, saved in the state Secret under written-names, until a revision activates after it. And what settles either is no longer only an activation the operator makes and sees answered, which a save lost after it, or a restart, left owed: the next look wrote the same document again over a write that ran (an epoch every node applied for nothing), or reloaded the revision again for a withdrawal it had read. Every change is marked with the epoch of the fleet's activation pointer read first after it (State.Observed): any revision the pointer names at a greater epoch activated after the change, and every node reads the store at its attempt to apply it, a node that refuses the revision included, whose running revision resolves its references from what that attempt read. The operator reads the pointer, and saves the marks, before it asks for the write or the reload that is to read them (markChanges), and a change is settled once the pointer is read naming a revision above its mark (State.Ran, settleRead), whoever activated it: the operator's own write or reload, whose saved record was lost, a person's reload, or the engine's own reload after it mints a credential. settleRead takes the fleet that keepServing reads afresh after the reconcile's read of the document, and counts only the revision that read returned, as one it saw opened: GET /config and the activation both read the pointer, and a revision a write activated between the two reads may be one no node can open. An activation the operator makes and sees answered settles every change still (State.Activated, State.Reloaded), and a pointer it cannot read leaves the changes for one. The marks the read lowers are saved before any activation is asked for too: the sync saves them in a reconcile that applies, and a reconcile whose document is held saves them itself (State.SameMarks says the read moved one), since a save lost after its reload left a change loaded unread again, which no epoch is above. A value Sync or Withdraw deletes owes nothing, and nor does one the store no longer holds that nothing writes again. And a value is marked before it is written. Sync hands its caller the State with every name it is about to write recorded as written, and the fingerprints as they were, and writes nothing unless the caller saved it: recorded only after the write, a save lost between the two left a value no revision read and nothing that said so, and the next sync, finding the store holding the operator's own value under the fingerprint it had, had nothing left to write. A name saved as written whose value never reached the store costs an activation for nothing; saved under its new fingerprint, a rotated value that never reached the store would never be written at all. The engine re-activates the revision itself at a sync's last write or removal of a value it reads (secretsync.Result.Reload). That activation lands inside the request, after every change the sync made, so no read of the pointer can mark them first: it settles them from its answer once read back, as the operator's apply where the sync wrote a value (reloadedBySync, State.Activated) and as the withdrawal it owes where it only removed one (reactivate). A write whose re-activation failed leaves its values owed, for the operator's apply. With that the in-sync check no longer compares the hash it recorded: a revision of the rendered document the engine runs above every mark has read every value, whatever the operator recorded, a write of it whose record the reconcile that made it never saved included, and it is recorded with the hash of what it runs. And a reload that fails, or loses its race to another writer, is read back (reloadFailed): a revision above the withdrawal's mark settles it, recorded as the reactivation where it is the running document again (the request landed although its answer did not, or somebody reloaded it), and reported with a Warning Reactivated that the operator activated nothing again where it is another document. It was taken for a failed withdrawal: a Warning that the engine may still start a tool server with a value that no longer exists, and an epoch every node applied for nothing at the next look. The write's summary names the Secrets owed rather than only what this reconcile wrote, since an earlier one may have written them; and a reactivation the operator records is dated by its clock, as every other record is, not the process's. Sync's write-ahead goes into helpers of its own (toWrite, writeAhead), which keep it under the cyclomatic bound; State, with its new map, is 72 bytes, under the 80 gocritic holds a value parameter to. storeEngine can fail a dry run as an engine that cannot decide one, and runs a hook once a value is written into its store. TestAValueWrittenBackStaysOwedThroughAFailedWrite fails with the in-sync check back on this reconcile's writes (the reload is adopted and nothing written) and with the summary built from them. TestAWrittenValueIsOwedUntilARevisionActivatesAfterIt fails with Sync not recording a write, with Activated or Reloaded not settling one, with Reactivated settling one, with a deleted value kept owed, with one the store lost kept owed, and with the record not carried from one sync to the next. TestAChangeIsSettledByAnActivationAboveTheEpochReadAfterIt fails with a later read raising a mark, with an activation at the epoch read settling it, with a name written again keeping its mark, and with Unobserved blind to a change. TestSyncSavesAWriteBeforeItWritesTheValue fails with the save skipped and with the new fingerprint saved ahead of the write. TestSaveStateKeepsTheWrittenNames fails with the names or their epochs not saved or not loaded, and with a negative epoch read as one. In the controller, TestASaveLostAfterAConfirmedWriteWritesNothingAgain fails with the read settling nothing; TestAWriteBackWhoseSaveIsLostStaysOwed with the save before the write skipped; TestARevisionThatReadEveryValueIsInSyncWhateverHashWasRecorded with the hash compared again or not recorded; TestAFailedReloadIsReadBack with the read back settling nothing, with the running document again taken for another, and with the reactivation dated by the process's clock; TestAnActivationAfterAWithdrawalSettlesIt with the read settling nothing; TestAnActivationNoNodeCanOpenSettlesNothing with an activation other than the revision GET /config returned settling; TestAReloadIsRecordedOnlyForTheDocumentItRead with the unconfirmed reload settling withdrawals alone; and TestAHeldReconcileSavesTheMarksItsReadSetBeforeItReloads, whose fleet answers the read that says what every pod runs and not the one after the withdrawal, with the held reconcile's save of what its read marked skipped. The envtest of a deleted value let go while the engine runs a reload of the rendered document expects that one activation to be a reload of the running revision (Reactivated) now: with the recorded hash no part of the question, the engine is in sync, and the withdrawal is all it owes, where the operator used to write the same document again; so does TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired, where the engine activates the running revision again at the removal. The tests that pin the operator's own activation after a write-back or a withdrawal make the engine's own re-activation fail, and TestAFailedReloadIsReadBack owes the values pushed after r1 explicitly (owePushed). newWithdrawal records r1 as activated after both values were pushed, as its comment says: it saved them as written and unread, which every reconcile would have written the document again for. The applying concept page says what the engine has read, and when a change stays a reason to write or to activate again, what a change no read of the pointer followed waits for, and what the state Secret holds; status.config.appliedHash's doc says what it records now. Signed-off-by: Ali Yousefian --- api/v1alpha1/company_types.go | 8 +- config/crd/bases/crewlet.ai_companies.yaml | 8 +- .../templates/crd/companies.crewlet.ai.yaml | 8 +- docs/concepts/applying.md | 147 +++-- docs/guides/troubleshooting.md | 2 +- docs/reference/api.md | 2 +- internal/controller/config_reload_test.go | 3 + internal/controller/config_run.go | 152 ++++- internal/controller/config_test.go | 5 +- internal/controller/config_withdraw.go | 130 ++++- internal/controller/config_withdraw_test.go | 549 +++++++++++++++++- internal/controller/sentence_test.go | 4 +- internal/secretsync/doc.go | 4 +- internal/secretsync/engine_test.go | 16 +- internal/secretsync/state.go | 253 +++++++- internal/secretsync/state_test.go | 45 +- internal/secretsync/sync.go | 96 ++- internal/secretsync/sync_test.go | 218 ++++++- 18 files changed, 1436 insertions(+), 214 deletions(-) diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index 59b12bf8..3554c9d3 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -1451,10 +1451,10 @@ type ConfigStatus struct { Epoch *int64 `json:"epoch,omitempty"` // appliedHash is the hash of the rendered document and of the values - // of the Secret keys it references, as last applied: a change to - // either is applied as a new revision, since the engine reads a - // rotated value only when a revision activates. The values enter it - // only as keyed fingerprints, so it reveals nothing about them. + // of the Secret keys it references, as the revision recorded runs + // them: the operator's last write, or a revision it found running that + // document after every value it wrote. The values enter it only as + // keyed fingerprints, so it reveals nothing about them. // +optional // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=128 diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index e8dbb944..f4067fa3 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -1740,10 +1740,10 @@ spec: appliedHash: description: |- appliedHash is the hash of the rendered document and of the values - of the Secret keys it references, as last applied: a change to - either is applied as a new revision, since the engine reads a - rotated value only when a revision activates. The values enter it - only as keyed fingerprints, so it reveals nothing about them. + of the Secret keys it references, as the revision recorded runs + them: the operator's last write, or a revision it found running that + document after every value it wrote. The values enter it only as + keyed fingerprints, so it reveals nothing about them. maxLength: 128 minLength: 1 type: string diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index 833cb662..da736a6e 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -1743,10 +1743,10 @@ spec: appliedHash: description: |- appliedHash is the hash of the rendered document and of the values - of the Secret keys it references, as last applied: a change to - either is applied as a new revision, since the engine reads a - rotated value only when a revision activates. The values enter it - only as keyed fingerprints, so it reveals nothing about them. + of the Secret keys it references, as the revision recorded runs + them: the operator's last write, or a revision it found running that + document after every value it wrote. The values enter it only as + keyed fingerprints, so it reveals nothing about them. maxLength: 128 minLength: 1 type: string diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index ffb06c92..ebdfa0e1 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -116,11 +116,11 @@ flowchart TD values are one activation: every write but the last asks the engine to activate nothing (`defer_reload`), and the last activates the revision once, with every value in the store. A sync whose last write fails, or - that stops before it, records the values it wrote deferred as unwritten, - so the next sync writes them again and its last write activates them; - where the last write activates nothing (the revision reads none of its - values) or its activation fails, the operator applies the document - itself. + that stops before it, records no fingerprint for the values it wrote + deferred, so the next sync writes them again and its last write + activates them; where the last write activates nothing (the revision + reads none of its values) or its activation fails, the operator applies + the document itself. The entries the running document still references are kept until the next document replaces it, and so are those of every other revision an engine pod runs or would start on: each pod applies the fleet's revision on its @@ -201,40 +201,55 @@ flowchart TD without, and is told by the event alone. Step 7 deletes the value once the next document runs. Whenever the reconcile ends without activating a revision after it deleted a value, because the document is held at any - step, the operator activates the revision the engine runs again, - unchanged, so the engine reads its store without the deleted value - rather than starting a tool server or an executor run with it (a - `Reactivated` event): where that revision read the value, the engine did - so itself as the operator removed it, and the operator takes that - activation for its own; otherwise, or where the engine's failed, it asks - for one (`POST /config/reload`). It keeps what it owes in its record of - the store until a revision has activated. It activates again only a - revision that goes without the value, never one that keeps it, and - records the revision that activates it again as its own only once it has - read back that it is that document: the engine activates again whichever - revision is active when the request lands, so one somebody else activated - in between is reported with a Warning `Reactivated`, and read as a - revision whose reading the operator does not know. Nothing makes the - engine's reload or its deletion conditional on the revision the operator - read, so a revision somebody activates in that moment goes without a - value it may require until the Secret exists again. + step, or because the engine already runs it with every value it + references (step 4), the revision the engine runs is activated again, + unchanged, so the engine reads its store without the deleted value rather + than starting a tool server or an executor run with it (a `Reactivated` + event): where that revision read the value, the engine did so itself as + the operator removed it, and the operator takes that activation for its + own; otherwise, or where the engine's failed, it asks for one + (`POST /config/reload`). It keeps what it owes in its record of the store + until the engine runs a revision activated after the deletion, whoever + activated it ([What the engine has read](#what-the-engine-has-read)). + It activates again only a revision that goes without the value, never one + that keeps it, and records the revision that activates it again as its + own only once it has read back that it is that document: the engine + activates again whichever revision is active when the request lands, so + one somebody else activated in between is reported with a Warning + `Reactivated`, and read as a revision whose reading the operator does not + know. A request of the operator's that fails is read back too: a revision + the engine runs that activated after the deletion read the store without + the value, so nothing is owed and nothing is activated again; where it is + the running document again (the request landed although its answer did + not, or somebody reloaded the document), it is recorded as the + reactivation, and where it is another document, a Warning `Reactivated` + says the operator activated nothing again. Nothing makes the engine's + reload or its deletion conditional on the revision the operator read, so + a revision somebody activates in that moment goes without a value it may + require until the Secret exists again. 4. **Compare.** The operator compares the engine's active document with the - rendered one, and compares a hash of the rendered document and - of the values it references with the one it recorded when it last applied - (`status.config.appliedHash`). If both match, there is nothing to write. - A changed value alone needs a new revision: the engine reads a rotated - value only when a revision activates. Where the engine runs the rendered - document, the write in step 3 made that revision, since the engine - activates the revision that reads a value it is written, and the - operator records it as its own apply (`status.config`), once it has read - back that the engine runs it, rather than activating the same document a - second time; its summary is the engine's, `reload: secret - written`. Where the engine did not activate one (nothing it runs reads - the value, or its activation failed or was overtaken by another write), - the operator applies the same document itself, with a summary that names - the Secret. So with a value written back in step 3, for the same reason: - since it was replaced, a reload may have handed the engine the other - value. + rendered one, and checks that the revision the engine runs activated + after every value it wrote into the store. If both hold, there is nothing + to write, and the revision is recorded with a hash of the rendered + document and of the values it references (`status.config.appliedHash`) + whoever activated it. A changed value alone needs a new revision: the + engine reads a rotated value only when a revision activates. Where the + engine runs the rendered document, the write in step 3 made that + revision, since the engine activates the revision that reads a value it + is written, and the operator records it as its own apply + (`status.config`), once it has read back that the engine runs it, rather + than activating the same document a second time; its summary is the + engine's, `reload: secret written`. Where the engine did not + activate one (nothing it runs reads the value, or its activation failed + or was overtaken by another write), the operator applies the same + document itself, with a summary that names the Secret. So with a value + written back in step 3, for the same reason: since it was replaced, a + reload may have handed the engine the other value. A value stays a reason + to write until the engine runs a revision activated after it is written + ([What the engine has read](#what-the-engine-has-read)), not just in the + reconcile that wrote it: a write that fails, or loses its race to another + writer, leaves the next sync nothing to write while the engine may still + run a revision that read the store before it. 5. **Dry-run** the document. The engine checks everything the API server cannot: rules that span objects, such as a Seat that asks for a sandbox the Company does not offer. Each problem the engine reports is traced back, @@ -338,12 +353,14 @@ changed. A fingerprint without the key that Secret also holds says nothing about a value. The same Secret holds the operator's record of the store's other facts: the revision whose reading of the values it knows, with the Secret keys that revision lets go without (`optional-keys`), the values it -deleted that a revision has yet to activate without, and the ones it said it -keeps. It records nothing of the engine's pods: what each runs and would -start on is what `GET /fleet` says of it now, since a pod applies revisions, -and a data pod moves the revision its volume starts it on, whether or not -the operator is looking, and a pod the fleet does not list keeps every value -it may need instead (step 3). +deleted that a revision has yet to activate without (`withdrawn-names`), the +ones it wrote that a revision has yet to activate with (`written-names`), +each of those two with the activation epoch it waits past, and the ones it +said it keeps. It records nothing of the engine's pods: what each runs and +would start on is what `GET /fleet` says of it now, since a pod applies +revisions, and a data pod moves the revision its volume starts it on, whether +or not the operator is looking, and a pod the fleet does not list keeps every +value it may need instead (step 3). The engine is reached at its first data pod, `http://-data-0.-data..svc:8080`, with the @@ -362,6 +379,43 @@ its keyring does not open the fleet's revision, the Company reports again when the data pods restart and every five minutes besides ([The engine cannot open its company document](../guides/troubleshooting.md#the-engine-cannot-open-its-company-document)). +### What the engine has read + +The engine reads its secret store only when a revision activates: every node +reads it at its attempt to apply the activation, a node that refuses the +revision included, whose running revision resolves its references from what +that attempt read. So a value written or deleted reaches nothing the engine +runs until a revision activates after it. The fleet's activation pointer +(`GET /fleet`, `activation`) names the fleet's revision at its epoch, and the +epoch only grows. So the operator reads the pointer after every change it +makes to the store, before it asks for the activation that is to read it, and +marks each change with the epoch read (`withdrawn-names` and `written-names` +keep it after the name): any revision the pointer names at a greater epoch +activated after the change. A change is settled once the pointer is read +naming such a revision, whoever activated it: the operator's own write or +reload, a person's `crewlet config reload`, or the engine's own reload after +it mints a credential. Only a revision the operator's read of `GET /config` +returned counts, as one it saw opened: both read the pointer, and a revision +a write activated between the two reads may be one no node can open. A value +is marked as written before it is +written, and the marks are saved before the activation is asked for, so a +save the operator loses after either, or a restart between them, leaves the +change owed, or settled by the next read of the engine, never settled by a +revision that activated before it. A write of the document that the engine +runs is therefore never sent again because the record of its activation was +lost: the next reconcile finds the revision running above the epoch every +value was marked by, and takes it for in sync. That needs a read of the +pointer between the change and the activation. A change the operator could +not follow with one, because the fleet could not be read or the engine had no +document yet, is settled by the activation the operator sees answered, and +where the record of that answer is lost as well, the next reconcile +activates a revision again for it. So is a change whose activation the +engine made itself: the last write or removal of a sync activates the +revision inside the same request, before any read of the pointer can follow the change, so the +operator settles it from that request's answer once it has read back that +the engine runs the revision it names, and a save lost after that costs one +activation more. + --- ## Management policies @@ -587,8 +641,9 @@ deleted Secret it keeps in the store for the revision the engine runs, which requires it or whose reading the operator does not know, `Reactivated` when it activated the running revision again so the engine reads its store without the value of a deleted Secret, a Warning `Reactivated` when the -revision activated again is not the document it read, a Warning -`SecretWithdrawalFailed` when it could not, +revision activated again is not the document it read, or when a request that +failed finds another document running that activated after the deletion, a +Warning `SecretWithdrawalFailed` when it could not, `Drifted`, `Invalid` with the engine's problems, and `EngineWarnings` when the engine applied a revision it has something to say about. Events, conditions and logs name Secrets, keys and store entries, diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 1d7c9987..36968690 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -110,7 +110,7 @@ with your last change yet. | | True | `DriftCorrected` | Under `Enforce`, someone changed the document in the engine and the operator applied the rendered one over it. A `Drifted` event names what differed. | | | True | `InSync` | Under `Observe`, the engine runs exactly the rendered document. | | | True | `Seeded` | Under `Seed`, the engine has a document and owns it from now on. | -| | False | `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `SecretNotFound`, `SecretTooLarge`, `PerSeatNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `OptionalKeyUnused` | An object of the company keeps the document from rendering. The object at fault reports the same reason on `Applied`; fix it there ([What blocks a document](../concepts/company-document.md#what-blocks-a-document)). Meanwhile, under `Enforce`, the operator still takes out of the engine's secret store the value of a deleted Secret the document lets go without, and activates the running revision again so the engine reads the store without it, with a `Reactivated` event, where the running revision lets the value go too, or a Warning `Reactivated` when the revision activated again is not the document it read; one that reads it as required, or another document somebody else activated, keeps it until the next document runs, with a `SecretKept` event, the only report of it here, since the Secrets are not read ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). | +| | False | `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `SecretNotFound`, `SecretTooLarge`, `PerSeatNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `OptionalKeyUnused` | An object of the company keeps the document from rendering. The object at fault reports the same reason on `Applied`; fix it there ([What blocks a document](../concepts/company-document.md#what-blocks-a-document)). Meanwhile, under `Enforce`, the operator still takes out of the engine's secret store the value of a deleted Secret the document lets go without, and activates the running revision again so the engine reads the store without it, with a `Reactivated` event, where the running revision lets the value go too, or a Warning `Reactivated` when the revision activated again is not the document it read, or when its request failed and another document, activated after the deletion, runs instead, and it activated nothing again; one that reads it as required, or another document somebody else activated, keeps it until the next document runs, with a `SecretKept` event, the only report of it here, since the Secrets are not read ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). | | | False | `NameConflict` | The Secret `-secret-sync`, where the operator keeps what it pushed to the engine's store, exists and is someone else's; `SecretsSynced` says the same. Rename or delete it; nothing is applied meanwhile. | | | False | `SeatPoolsNotAllowed` | The Company declares `spec.topology.seatPools`, and its class's `seatPools.policy` is not `Allowed`. Remove the pools, or run the company with a class that allows them. | | | False | `SandboxRequired` | The Company has an agent seat and a model provider, and no `spec.sandbox`: every agent seat's executor runs in a box, and Crewlet refuses a company that places none on every write. Set `spec.sandbox.defaultRunIn` to `Direct`, or to `E2B` with `spec.sandbox.e2b` ([Where a seat's executor runs](cli-agents.md#where-a-seats-executor-runs)). | diff --git a/docs/reference/api.md b/docs/reference/api.md index 0e6fc291..74582eff 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -760,7 +760,7 @@ _Appears in:_ | --- | --- | --- | --- | | `revisionID` _string_ | revisionID is the engine's id of the last revision the operator saw
run the rendered document: the one it applied, one the engine
activated with the same content (a reload), or, under Observe, the
one it found in sync. Under Seed it is the revision the document was
seeded as. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| | `epoch` _integer_ | epoch is the engine's activation counter for that revision. The
engine applies a revision asynchronously; it runs it once the applied
epoch its /health reports has reached this. | | Minimum: 0
Optional: \{\}
| -| `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as last applied: a change to
either is applied as a new revision, since the engine reads a
rotated value only when a revision activates. The values enter it
only as keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| +| `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as the revision recorded runs
them: the operator's last write, or a revision it found running that
document after every value it wrote. The values enter it only as
keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| | `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator activated the revision this records,
or found the engine running it where it adopted a revision of the
rendered document it did not write (a reload): what the operator's
wait for the engine to apply its epoch counts from. | | Optional: \{\}
| | `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from. | | MinProperties: 1
Optional: \{\}
| | `placement` _[RevisionPlacement](#revisionplacement)_ | placement is where the revision the engine runs pins its agent seats,
read from the engine's document under every management policy. The
operator keeps a seat pool's pods running while the running revision
pins a seat to them, even after the pool or the seat's placement is
gone from the spec: a pod stopped while its seat is still pinned to
it would leave the seat held by nobody, and a blocked document keeps
the last good revision running. Once a revision that pins nothing to
them runs, the pods are deleted. | | MinProperties: 1
Optional: \{\}
| diff --git a/internal/controller/config_reload_test.go b/internal/controller/config_reload_test.go index c1fac7c0..96f168d6 100644 --- a/internal/controller/config_reload_test.go +++ b/internal/controller/config_reload_test.go @@ -75,6 +75,9 @@ func TestARotationIsTheEnginesOneActivation(t *testing.T) { if state := w.state(t); state.Recorded() != "r2" || !slices.Equal(state.LetGo("r2"), []string{w.registry}) { t.Fatalf("the state records %q letting %v go, want r2 read as the rendered document", state.Recorded(), state.LetGo("r2")) } + if owed := w.state(t).Written(); len(owed) != 0 { + t.Errorf("the state owes %v, want nothing: r2 activated after the sync wrote the rotated value", owed) + } if !slices.ContainsFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated Activated revision r2 ") }) { t.Errorf("events %q, want r2's activation", recorded) } diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 10ffedd5..58f3f7ce 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -507,6 +507,9 @@ func (run *configRun) enforce( if err := run.keepServing(ctx, eng, &pushing); err != nil { return ctrl.Result{}, err } + // What the fleet keepServing read settles or marks, the sync below + // saves, whatever it writes, before any activation is asked for. + state, _ = run.settleRead(ctx, eng, active, state) // SecretKept is said of what the reconcile ends up keeping: a value the // sync below keeps for the running revision goes after the apply. defer run.tellKept() @@ -517,17 +520,27 @@ func (run *configRun) enforce( hash := appliedHash(run.doc, state, desired.Names()) recorded := c.Status.Config - // A value written back although no Secret changed (one somebody else - // overwrote, or one the store lost) leaves the hash where it was, but the - // engine reads its store only when a revision activates: since the - // overwrite, a reload may have handed it the other value, which it would - // keep running. So a change of a value is followed by an activation, - // which is how a rotated credential reaches a running engine: the - // engine's own, which re-activates the revision that reads a value it is - // written or removes (reloadedBySync), or else the operator's apply - // below. - if active != nil && len(diffs) == 0 && hash == recorded.AppliedHash && len(pushed.Put) == 0 { - return run.keepInSync(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) + // The engine runs the rendered document with every value only where the + // revision it runs read the store after the last value the sync wrote: + // the engine reads its store only when a revision activates. The state + // says which values no activation is known to have read yet + // (State.Written), settled above by the activation the engine runs where + // that came after them, and every value is in it from before it was + // written: a value rotated, or one written back although no Secret + // changed (one somebody else overwrote, or one the store lost), which + // leaves the hash where it was, and one a write of the document that + // failed or lost its race left the next sync nothing to write. So the + // recorded hash is no part of the question: a revision of the rendered + // document that read every value is in sync whatever the operator + // recorded, a write of it whose record the reconcile that made it never + // saved included, and a withdrawal is all it owes (settle), activated by + // the engine at the removal or else by the operator. One that did not is + // activated again, which is how a rotated credential reaches a running + // engine: by the engine itself, which re-activates the revision that + // reads a value it is written (reloadedBySync), or else by the + // operator's apply below. + if active != nil && len(diffs) == 0 && len(state.Written()) == 0 { + return run.keepInSync(ctx, eng, active, hash, pushed.Reload, pushed.ReloadAction, state, stateRef) } reloaded, err := run.reloadedBySync(ctx, eng, active, diffs, pushed.Reload) if err != nil { @@ -535,11 +548,13 @@ func (run *configRun) enforce( } if reloaded != nil { // The engine runs the rendered document with every value the sync - // left in its store: applying it again would be a second activation - // of the same thing. It is recorded as the apply would have been, - // and it read the store without every name withdrawn. + // left in its store, activated by the sync's own last request after + // every write and removal: applying it again would be a second + // activation of the same thing. It is recorded as the apply would + // have been, and it read the store without every name withdrawn and + // with every value written. run.recordApply(*reloaded, hash, false, active, diffs, - strings.TrimPrefix(run.summary(active, diffs, pushed.Put), "crewlet-operator: ")+", re-activated by the engine") + strings.TrimPrefix(run.summary(active, diffs, state.Written()), "crewlet-operator: ")+", re-activated by the engine") state = state.Activated(reloaded.RevisionID, run.doc.Optional) if err := secretsync.SaveState(ctx, run.Client, stateRef, c, state); err != nil { return ctrl.Result{}, err @@ -568,7 +583,13 @@ func (run *configRun) enforce( if err := run.widenPlacement(ctx, *run.rendered); err != nil { return ctrl.Result{}, err } - req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, pushed.Put)} + // What the write's activation reads is marked before it is asked for, + // so that a save lost after it still leaves what it read settled by the + // next read of the engine. + if state, err = run.markChanges(ctx, eng, state, stateRef); err != nil { + return ctrl.Result{}, err + } + req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, state.Written())} if checked.BaseRevisionID == "" { req.CreateOnly = true } else { @@ -703,15 +724,80 @@ func (run *configRun) reloadedBySync( return &engine.ApplyResult{RevisionID: reload.RevisionID, Epoch: reload.Epoch, Warnings: []string{}}, nil } +// settleRead is state with every change of the store settled that the +// fleet's activation came after (State.Ran), and every change no read of the +// fleet's activation pointer has followed marked by the epoch it names now +// (State.Observed), and whether that moved a mark, which the caller saves +// before it asks for an activation (markChanges' rule: a save lost after the +// activation must leave a change the activation settles). It reads the fleet +// the reconcile last read (fleetOf), which keepServing reads afresh after +// this attempt's read of the document. active is the revision that read +// returned, which GET /config opened: the activation settles only where it is +// that revision. Both read the fleet's activation pointer, so another +// revision there is one a write activated between the two reads, perhaps one +// no node opens. state holds only what earlier reconciles changed, so any +// read of the fleet this attempt makes followed it. A fleet that cannot be +// read, or answers without its activation, settles and marks nothing. +func (run *configRun) settleRead( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, +) (secretsync.State, bool) { + if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { + return state, false + } + fleet, err := run.fleetOf(ctx, eng) + if err != nil || fleet.Activation == nil { + return state, false + } + next := state + if active != nil && fleet.Activation.RevisionID == active.RevisionID { + next = next.Ran(fleet.Activation.Epoch) + } + next = next.Observed(fleet.Activation.Epoch) + return next, !next.SameMarks(state) +} + +// markChanges reads the fleet's activation pointer after the changes of the +// store state records that no read has followed yet, marks each with the +// epoch it names (State.Observed), and saves state, before the caller asks +// the engine for an activation: one the engine is later read running above +// that epoch came after every change, whoever made it, so a save lost after +// the activation the caller asks for leaves what that activation read settled +// by the next read of the engine (settleRead) rather than owed for good. A +// fleet that cannot be read, or answers without its activation, leaves them +// unread, for an activation the operator sees confirmed to settle; a save +// that fails is the caller's error, and no activation is asked for. +func (run *configRun) markChanges( + ctx context.Context, eng *engine.Client, state secretsync.State, ref types.NamespacedName, +) (secretsync.State, error) { + if !state.Unobserved() { + return state, nil + } + fleet, err := eng.Fleet(ctx) + if err != nil || fleet.Activation == nil { + logf.FromContext(ctx).V(1).Info("Failed to read the fleet's activation pointer after a change of the engine's "+ + "secret store; the change waits for an activation the operator sees confirmed", "error", fmt.Sprint(err)) + return state, nil + } + state = state.Observed(fleet.Activation.Epoch) + if err := secretsync.SaveState(ctx, run.Client, ref, run.company, state); err != nil { + return state, err + } + return state, nil +} + // sync pushes the desired values into the engine's secret store, deletes // what nothing references and what the revision the engine runs goes -// without, and saves what it did, whether or not it got through. What it -// keeps for that revision is recorded (keep), for the reconcile to say once -// it knows it ends up keeping it (tellKept). +// without, and saves what it did, whether or not it got through, and what it +// is about to write before it writes it (secretsync.Sync). What it keeps for +// that revision is recorded (keep), for the reconcile to say once it knows it +// ends up keeping it (tellKept). func (run *configRun) sync( ctx context.Context, eng *engine.Client, desired secretsync.Desired, state secretsync.State, ref types.NamespacedName, ) (secretsync.State, secretsync.Result, error) { - next, result, syncErr := secretsync.Sync(ctx, eng, desired, state, engine.TokenLogin(render.TokenOperator)) + next, result, syncErr := secretsync.Sync(ctx, eng, desired, state, engine.TokenLogin(render.TokenOperator), + func(ahead secretsync.State) error { + return secretsync.SaveState(ctx, run.Client, ref, run.company, ahead) + }) if err := secretsync.SaveState(ctx, run.Client, ref, run.company, next); err != nil { return next, result, errors.Join(syncErr, err) } @@ -822,10 +908,10 @@ func diffWith(doc *render.Document, active *engine.ActiveConfig) ([]render.Path, // a value is kept for a revision other than the fleet's (heldResync). reload // and reloadAction are the sync's, as settle takes them. func (run *configRun) keepInSync( - ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, - state secretsync.State, ref types.NamespacedName, + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string, reload *engine.Reload, + reloadAction string, state secretsync.State, ref types.NamespacedName, ) (ctrl.Result, error) { - if err := run.inSync(ctx, eng, active); err != nil { + if err := run.inSync(ctx, eng, active, hash); err != nil { return ctrl.Result{}, err } again, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) @@ -848,9 +934,13 @@ func (run *configRun) keepInSync( // It fails where the fleet cannot be read or answers without its activation, // and with an *activationAhead, a lost race (engine.ErrRevisionAdvanced), // where the activation names another revision: converge reads again from the -// start. -func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) error { +// start. hash is what it runs (appliedHash), which the record takes: the one +// recorded may be another, the operator's write of the same document whose +// record the reconcile that made it never saved being taken for in sync too. +func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string) error { recorded := run.company.Status.Config + config := recorded.DeepCopy() + config.AppliedHash = hash if recorded.RevisionID != active.RevisionID { // AT THE EPOCH IT ACTIVATED AS, which only the fleet's activation // says: the one recorded is the revision it replaces, so the @@ -871,12 +961,13 @@ func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *en case fleet.Activation.RevisionID != active.RevisionID: return &activationAhead{read: active.RevisionID, activation: *fleet.Activation} } - config := recorded.DeepCopy() config.RevisionID = active.RevisionID config.Epoch = new(fleet.Activation.Epoch) found := metav1.NewTime(run.clock()) config.AppliedAt = &found config.Objects = run.objectCounts() + } + if !equality.Semantic.DeepEqual(*config, recorded) { run.config = config } run.recordPlacement(*run.rendered, false) @@ -1557,9 +1648,10 @@ func placesOf(diffs []render.Path) string { } // summary is the X-Summary of a write: "crewlet-operator: " and the objects -// whose part of the document changed, or the Secrets whose rotated values -// the write makes the engine read. -func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, put []string) string { +// whose part of the document changed, or the Secrets whose values, written +// since a revision last activated (written, secretsync.State.Written), the +// write makes the engine read. +func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, written []string) string { var objects []string add := func(s string) { if !slices.Contains(objects, s) { @@ -1582,7 +1674,7 @@ func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, } } for _, k := range run.doc.Refs { - if slices.Contains(put, secretsync.StoreName(k)) && active != nil && len(diffs) == 0 { + if slices.Contains(written, secretsync.StoreName(k)) && active != nil && len(diffs) == 0 { add("Secret " + k.Namespace + "/" + k.Name) } } diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go index 46d8b713..ac57db9c 100644 --- a/internal/controller/config_test.go +++ b/internal/controller/config_test.go @@ -1116,7 +1116,10 @@ var _ = Describe("Company config", Ordered, func() { Consistently(count).WithTimeout(6*time.Second).Should(Equal(4), "a revision was activated with the deleted Secret's value in the store, and activated again") Eventually(startedWith).WithTimeout(settle).ShouldNot(HaveKey(optionalRegistry)) - Expect(log.of(name)).NotTo(ContainElement(HavePrefix("Normal Reactivated "))) + // The engine runs the rendered document, which read every value the + // store still holds, so the one activation owed is the reload of it + // without the deleted value, not a write of the same document again. + Expect(log.of(name)).To(ContainElement(HavePrefix("Normal Reactivated "))) Expect(log.of(name)).NotTo(ContainElement(HavePrefix("Normal SecretKept "))) applied(c, crewletaiv1alpha1.ReasonApplied) runsRendered(e, c) diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index ba9a4d5f..db8328dd 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -102,7 +102,11 @@ import ( // later reconcile, of this operator or one restarted; the activation an // apply makes is recorded before anything is deleted after it, so a deletion // whose own record a failed write lost is withdrawn again by a later -// reconcile, which knows the revision lets the key go. +// reconcile, which knows the revision lets the key go. And it is settled by +// any activation the engine is read running after the deletion, whoever made +// it, by the fleet's activation epoch the deletion was marked with +// (settleRead, markChanges): a reload that failed, or lost its race to +// another writer, is read back rather than taken for a failed withdrawal. // // The engine offers no conditional reload and no conditional delete: // whichever revision is active when the request lands is the one activated @@ -186,6 +190,17 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err := run.keepServing(ctx, eng, &read); err != nil { return ctrl.Result{}, err } + // What the fleet keepServing read settles or marks is saved before + // anything is asked of the engine: the reload below may land and the save + // after it be lost, and a mark the read lowered that only this reconcile + // knows of would be loaded unread again, which no activation's epoch is + // above (markChanges). + state, moved := run.settleRead(ctx, eng, active, state) + if moved { + if err := secretsync.SaveState(ctx, run.Client, stateRef, c, state); err != nil { + return ctrl.Result{}, err + } + } next, res, withdrawErr := secretsync.Withdraw(ctx, eng, read, state) if len(res.Deleted) > 0 { run.event(corev1.EventTypeNormal, "SecretsSynced", "SyncSecrets", @@ -201,7 +216,7 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) } var reactivateErr error if withdrawErr == nil { - next, reactivateErr = run.reactivate(ctx, eng, active, res.Reload, res.ReloadAction, next) + next, reactivateErr = run.reactivate(ctx, eng, active, res.Reload, res.ReloadAction, next, stateRef) } if err := secretsync.SaveState(ctx, run.Client, stateRef, c, next); err != nil { return ctrl.Result{}, err @@ -369,7 +384,7 @@ func (run *configRun) settle( if len(state.Withdrawn()) == 0 { return 0, nil } - next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state) + next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state, ref) if err := secretsync.SaveState(ctx, run.Client, ref, run.company, next); err != nil { return 0, err } @@ -414,9 +429,9 @@ func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Resu // The engine activates its revision again itself when a write or a removal // changes a value that revision reads (Crewlet's ADR-0070), and reload is // what the last such request answered (secretsync.Result.Reload), made by -// the engine call reloadAction (secretsync.Result.ReloadAction): where it landed, -// the engine read its store without every name removed, and it is the -// activation; asking for another would activate the same document twice. +// the engine call reloadAction (secretsync.Result.ReloadAction): where it +// landed, the engine read its store without every name removed, and it is +// the activation; asking for another would activate the same document twice. // Where it is nil or failed, the operator asks for one (POST /config/reload). // // Neither names a revision: the engine stores again whichever revision the @@ -426,9 +441,14 @@ func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Resu // one read lets go only where it is that document; otherwise the state goes // on recording the one read, which is no longer the fleet's, and a Warning // says so. +// Before the operator's request the withdrawals are marked by the fleet's +// activation epoch, and state saved in ref (markChanges): a request that +// fails, or loses its race to another writer, is read back, and a revision +// the fleet runs above that epoch came after every withdrawal, whoever +// activated it, and settles them (reloadFailed). func (run *configRun) reactivate( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, - state secretsync.State, + state secretsync.State, ref types.NamespacedName, ) (secretsync.State, error) { withdrawn := state.Withdrawn() if len(withdrawn) == 0 { @@ -454,15 +474,18 @@ func (run *configRun) reactivate( if reload.Landed() { by = reloadAction } - byEngine := by != "" reloaded := engine.ApplyResult{} - if byEngine { + if by != "" { reloaded.RevisionID, reloaded.Epoch = reload.RevisionID, reload.Epoch } else { var err error + if state, err = run.markChanges(ctx, eng, state, ref); err != nil { + return state, err + } if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(held)); err != nil { - return state, fmt.Errorf("activating revision %s again, so that the engine reads its secret store without %s: %w", - active.RevisionID, strings.Join(held, ", "), err) + return run.reloadFailed(ctx, eng, active, state, held, + fmt.Errorf("activating revision %s again, so that the engine reads its secret store without %s: %w", + active.RevisionID, strings.Join(held, ", "), err)) } } now, err := activeConfig(ctx, eng) @@ -475,10 +498,56 @@ func (run *configRun) reactivate( run.reactivatedUnconfirmed(active, reloaded, by, held, "its document is not that one: somebody activated another revision before the request landed") default: - run.recordReactivation(active, reloaded, by, held) + run.recordReactivation(active, reloaded, by, held, nil) return state.Activated(reloaded.RevisionID, state.LetGoKeys(active.RevisionID)), nil } - return state.Reactivated(), nil + // Whatever its document, the revision the request activated read the + // store after every write and withdrawal the state records. + return state.Reloaded(), nil +} + +// reloadFailed is state once a request to activate active again, so that the +// engine reads its store without held, failed with failed: settled as far as +// the fleet's revision, read afresh, activated after the withdrawals, since +// nothing read before the request saw what activated since, and failed where +// any of held is still owed. A revision the fleet's activation names above +// the epoch the withdrawals were marked by (markChanges) came after every one +// of them, whoever activated it: the request itself, where it landed and its +// answer did not, or another writer, who won the race with it. Where its +// document is active's, the fleet runs that document again without held, as +// a reactivation the operator confirmed does, and it is recorded as one +// (recordReactivation, whose event says the request failed); otherwise it is +// a revision whose reading the operator does not know, which a Warning says, +// and the state goes on recording the one read, as reactivatedUnconfirmed +// leaves it. A request that failed outright leaves the fleet on active, which +// settles nothing. +func (run *configRun) reloadFailed( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, held []string, + failed error, +) (secretsync.State, error) { + now, err := activeConfig(ctx, eng) + if err != nil || now == nil || now.RevisionID == active.RevisionID { + return state, failed + } + fleet, err := eng.Fleet(ctx) + if err != nil || fleet.Activation == nil || fleet.Activation.RevisionID != now.RevisionID { + return state, failed + } + settled := state.Ran(fleet.Activation.Epoch) + if owed := settled.Withdrawn(); slices.ContainsFunc(held, func(name string) bool { return slices.Contains(owed, name) }) { + return state, failed + } + activation := engine.ApplyResult{RevisionID: now.RevisionID, Epoch: fleet.Activation.Epoch} + if bytes.Equal(now.Document, active.Document) { + run.recordReactivation(active, activation, "", held, failed) + return settled.Adopted(now.RevisionID, state.LetGoKeys(active.RevisionID)), nil + } + run.event(corev1.EventTypeWarning, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( + "Did not activate revision %s again: revision %s, another document, activated as epoch %d after the operator "+ + "removed %s from the engine's secret store, and read the store without them; the operator takes it to read "+ + "every value as required, as it does any revision it has no record of activating, and the request failed: %s", + active.RevisionID, now.RevisionID, fleet.Activation.Epoch, strings.Join(held, ", "), failed))) + return settled, nil } // reactivationSummary is the summary of the revision that activates the @@ -495,16 +564,29 @@ func reactivationSummary(held []string) string { // recordReactivation records the running document activated again as // reloaded, without held, by the engine at the request of the sync's engine // call by (WriteSecret, PutSecrets or DeleteSecret: a write or a removal of -// a value the revision reads), or at the operator's request, by empty. A revision the operator recorded -// as its own is replaced by the one that activates it again, with the same -// content, so that the next one it applies does not take it for an edit -// somebody else made; one it did not write stays somebody else's. -func (run *configRun) recordReactivation(active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string) { - who, action := reactivatedBy(by) - run.event(corev1.EventTypeNormal, "Reactivated", action, - "%s revision %s again, unchanged, as revision %s (epoch %d), so that the engine reads its secret store "+ - "without %s: the Secret or key each stands for no longer exists, and the revision lets it go without its value.", - who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", ")) +// a value the revision reads), or at the operator's request, by empty. A +// revision the operator recorded as its own is replaced by the one that +// activates it again, with the same content, so that the next one it applies +// does not take it for an edit somebody else made; one it did not write stays +// somebody else's. failed is the error the operator's request answered where +// the engine was read back running the document again anyway (reloadFailed), +// nil where it answered the activation. +func (run *configRun) recordReactivation( + active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string, failed error, +) { + if failed == nil { + who, action := reactivatedBy(by) + run.event(corev1.EventTypeNormal, "Reactivated", action, + "%s revision %s again, unchanged, as revision %s (epoch %d), so that the engine reads its secret store "+ + "without %s: the Secret or key each stands for no longer exists, and the revision lets it go without its "+ + "value.", who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", ")) + } else { + run.event(corev1.EventTypeNormal, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( + "The engine runs revision %s again, unchanged, as revision %s (epoch %d), activated after the operator removed "+ + "%s from its secret store, whose Secret or key no longer exists and which the revision lets go without its "+ + "value, so it reads the store without them; the operator's request to activate it again failed: %s", + active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", "), failed))) + } recorded := run.company.Status.Config if run.config != nil { recorded = *run.config @@ -513,7 +595,7 @@ func (run *configRun) recordReactivation(active *engine.ActiveConfig, reloaded e return } config := recorded.DeepCopy() - now := metav1.Now() + now := metav1.NewTime(run.clock()) config.RevisionID, config.Epoch, config.AppliedAt = reloaded.RevisionID, new(reloaded.Epoch), &now run.config = config if run.applied.kind == appliedInRevision && run.applied.revision == active.RevisionID { diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 5dae94d8..13145cf0 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -104,6 +104,12 @@ type storeEngine struct { // how many epochs /health reports the node behind the latest. noActivation bool appliedLag int + // dryRunDown makes the engine answer a dry run as one that cannot + // decide it, a failure no wait of the reconcile's own clears. + dryRunDown bool + // afterPut, when set, runs once, under its lock, just after a value is + // written into the store. + afterPut func(e *storeEngine) } // dataNode reports whether node is a pod of the Company's data StatefulSet, @@ -116,9 +122,10 @@ func dataNode(node string) bool { // otherwise. const storeNode = "acme-data-0" -// down reports whether r is a request configDown, secretsDown or reloadDown -// makes the engine refuse: a read of the fleet's document, a write to the -// store, one name's or a batch's, or a reload of the fleet's revision. +// down reports whether r is a request configDown, secretsDown, reloadDown or +// dryRunDown makes the engine refuse: a read of the fleet's document, a write +// to the store, one name's or a batch's, a reload of the fleet's revision, or +// a dry run. func (e *storeEngine) down(r *http.Request) bool { if r.Method == http.MethodGet { return r.URL.Path == configRoute && e.configDown @@ -126,6 +133,9 @@ func (e *storeEngine) down(r *http.Request) bool { if r.URL.Path == reloadRoute { return e.reloadDown } + if r.URL.Path == configRoute && dryRun(r) { + return e.dryRunDown + } return (r.URL.Path == secretsRoute || strings.HasPrefix(r.URL.Path, secretsRoute+"/")) && e.secretsDown } @@ -194,7 +204,7 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: answer(http.StatusBadRequest, map[string]any{"error": "validation_error", "detail": "refused", "problems": []map[string]string{{"path": "roles", "kind": "invalid", "message": "refused"}}}) - case r.Method == http.MethodPut && r.URL.Path == configRoute && r.URL.Query().Get("dry_run") == "true": + case r.Method == http.MethodPut && r.URL.Path == configRoute && dryRun(r): answer(http.StatusOK, map[string]any{"base_revision_id": active.id}) case r.Method == http.MethodPut && r.URL.Path == configRoute: applied := activate(body) @@ -252,6 +262,11 @@ func (e *storeEngine) serveSecretWrite( read = append(read, name) } } + if e.afterPut != nil { + hook := e.afterPut + e.afterPut = nil + hook(e) + } var reload any if len(read) > 0 { reload = e.reactivate(activate, active, "reload: secrets "+strings.Join(read, ", ")+" written") @@ -259,6 +274,11 @@ func (e *storeEngine) serveSecretWrite( answerJSON(w, http.StatusOK, map[string]any{"names": names, readByField: readBy, reloadField: reload}) case r.Method == http.MethodPut && isSecret: e.store[name] = string(body) + if e.afterPut != nil { + hook := e.afterPut + e.afterPut = nil + hook(e) + } readBy, reload := []string{}, any(nil) if reads(name) { readBy = []string{readPath} @@ -420,6 +440,11 @@ func (e *storeEngine) activation() *activationAnswer { return &activationAnswer{RevisionID: e.revisions[len(e.revisions)-1].id, Epoch: len(e.revisions)} } +// dryRun reports whether r asks only for a check of the document. +func dryRun(r *http.Request) bool { + return r.URL.Query().Get("dry_run") == "true" +} + // answerJSON writes v as the JSON answer, with status. func answerJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json") @@ -547,6 +572,11 @@ type withdrawal struct { registry string // requiring is a document whose server requires the registry. requiring []byte + // pushed is what the operator pushed for r1, both values, and owe makes + // the state record them pushed after it (owePushed). + pushed secretsync.Desired + recorded string + optional []secretref.Key // stateWrites counts the writes of the secret sync's state since the // reconcile began, and failWrite, when it is not 0, is the one that // fails, the API server not answering. @@ -618,18 +648,18 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { if err != nil { t.Fatal(err) } - var state secretsync.State - if r.recorded != "" { - state = state.Activated(r.recorded, runningDoc.Optional) - } // Pushed before r1 activated, so that r1 read them: nothing is - // re-activated for them. + // re-activated for them, and nothing is owed for them after it. fe.reloadAnswer = new(map[string]any(nil)) - state, _, err = secretsync.Sync(ctx, eng, present, state, operatorLogin) + state, _, err := secretsync.Sync(ctx, eng, present, secretsync.State{}, operatorLogin, discardState) if err != nil { t.Fatal(err) } fe.reloadAnswer = nil + state = state.Reloaded() + if r.recorded != "" { + state = state.Activated(r.recorded, runningDoc.Optional) + } stateRef := types.NamespacedName{Namespace: company.Namespace, Name: secretSyncStateName(company)} if err := secretsync.SaveState(ctx, cluster, stateRef, company, state); err != nil { t.Fatal(err) @@ -640,6 +670,7 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { w.stateWrites.Store(0) w.engine, w.client, w.cluster = fe, eng, cluster + w.pushed, w.recorded, w.optional = present, r.recorded, runningDoc.Optional w.registry = secretsync.StoreName(secretref.Key{Namespace: "ns", Name: optionalSecret, Key: optionalKey}) w.again(t, company) if !fe.holds(w.registry) { @@ -648,6 +679,30 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { return w } +// owePushed makes the state the one a sync leaves that pushed both values +// again after r1 activated, whose writes the engine re-activated nothing at: +// the revision recorded as before, and both values owed until a revision +// activates after them. +func (w *withdrawal) owePushed(t *testing.T) { + t.Helper() + ctx := context.Background() + w.engine.reloadAnswer = new(map[string]any(nil)) + state, _, err := secretsync.Sync(ctx, w.client, w.pushed, secretsync.State{}, operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + w.engine.reloadAnswer = nil + if w.recorded != "" { + state = state.Adopted(w.recorded, w.optional) + } + if len(state.Written()) == 0 { + t.Fatal("the sync pushed nothing again") + } + if err := secretsync.SaveState(ctx, w.cluster, w.ref(), w.run.company, state); err != nil { + t.Fatal(err) + } +} + // again makes w.run a new reconcile of company, as a reconcile after the last // one sees it, rendering optionalRun's document. func (w *withdrawal) again(t *testing.T, company *crewletaiv1alpha1.Company) { @@ -676,6 +731,15 @@ func (w *withdrawal) statusWritten() *crewletaiv1alpha1.Company { return company } +// ref is where the operator keeps the secret sync's state of w's Company. +func (w *withdrawal) ref() types.NamespacedName { + return types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} +} + +// discardState is a save that takes every State secretsync.Sync hands it +// ahead of a write, and keeps none: the test saves what Sync returns itself. +func discardState(secretsync.State) error { return nil } + // state is the secret sync's state as the reconcile saved it. func (w *withdrawal) state(t *testing.T) secretsync.State { t.Helper() @@ -951,8 +1015,10 @@ func TestARevisionThatLetsAValueGoGoesWithoutItWhateverTheNextDocumentReads(t *t // the running revision lets it go too, the operator's own or the document it // renders, it goes before the apply, and the engine activates the running // revision again at its removal, which reads it: that is the document the -// operator renders, so it is the one activation it takes, recorded as the -// operator's apply, and nothing is dry-run that the engine could refuse. +// operator renders, which the engine runs with every value, so it is the one +// activation it takes, the withdrawal it owes (Reactivated), recorded in place +// of the revision before it, and nothing is dry-run that the engine could +// refuse. func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { for _, tc := range []struct { running @@ -962,8 +1028,8 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { revisions []string kept bool // byEngine is the engine's re-activation at the removal, the - // document being the one the engine runs, taken for the apply - // before anything is dry-run. + // document being the one the engine runs, taken for the withdrawal + // it owes before anything is dry-run. byEngine bool }{ {running: runningRequires, revisions: []string{"r1", "r2", "r3"}}, @@ -1012,11 +1078,13 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { t.Errorf("the state lets %v go for %s, owes %v and retains %v, want the registry, nothing and nothing", state.LetGo(last), last, state.Withdrawn(), state.Retained()) } - reactivated := len(tc.revisions) == 3 || tc.refused && !tc.byEngine - if told(recorded, "Reactivated") != reactivated { - t.Errorf("events %q: Reactivated told %t, want %t", recorded, !reactivated, reactivated) + // Every revision that lets the registry go is activated again + // without it: by the operator after the next document, or by the + // engine at the removal where the running one lets it go. + if !told(recorded, "Reactivated") { + t.Errorf("events %q, want Reactivated", recorded) } - if reactivated && !strings.Contains(w.engine.summaryOf(), w.registry) { + if !strings.Contains(w.engine.summaryOf(), w.registry) { t.Errorf("revision %s is summarised %q, which does not name the registry", last, w.engine.summaryOf()) } if (!tc.refused || tc.byEngine) && @@ -1166,10 +1234,14 @@ func TestAReloadIsRecordedOnlyForTheDocumentItRead(t *testing.T) { if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3"}) { t.Fatalf("the revisions are %v, want r1, r2 and r3, the reload %s", got, reload) } + // The reload, whatever its document, activated after every + // value the operator wrote too, the one the fixture pushed. state := w.state(t) - if state.Records(reload) || len(state.LetGo(reload)) != 0 || len(state.Withdrawn()) != 0 { - t.Errorf("the state records %s: %t, letting %v go, and owes %v; want %s unknown and nothing owed", - reload, state.Records(reload), state.LetGo(reload), state.Withdrawn(), reload) + if state.Records(reload) || len(state.LetGo(reload)) != 0 || len(state.Withdrawn()) != 0 || + len(state.Written()) != 0 { + t.Errorf("the state records %s: %t, letting %v go, and owes %v withdrawn and %v written; "+ + "want %s unknown and nothing owed", + reload, state.Records(reload), state.LetGo(reload), state.Withdrawn(), state.Written(), reload) } if !warned(recorded, "Reactivated") || told(recorded, "Reactivated") { t.Errorf("events %q, want a Warning Reactivated", recorded) @@ -1237,14 +1309,14 @@ func TestReactivateRepeatsOnlyARevisionThatLetsTheValueGo(t *testing.T) { t.Fatal(err) } // Removed by a sync that kept nothing for the running revision. - state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) if err != nil || !slices.Equal(state.Withdrawn(), []string{w.registry}) { t.Fatalf("the sync withdrew %v (%v), want the registry", state.Withdrawn(), err) } if res.Reload.Landed() != byEngine { t.Fatalf("the sync reports the engine's re-activation %+v", res.Reload) } - next, err := w.run.reactivate(ctx, w.client, active, res.Reload, res.ReloadAction, state) + next, err := w.run.reactivate(ctx, w.client, active, res.Reload, res.ReloadAction, state, w.ref()) if err != nil { t.Fatal(err) } @@ -1296,13 +1368,13 @@ func TestAReactivationByTheEngineNamesTheRequestThatMadeIt(t *testing.T) { if err != nil { t.Fatal(err) } - state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) if err != nil || !res.Reload.Landed() { t.Fatalf("the sync = (%+v, %v), want the engine's re-activation", res.Reload, err) } recorder := &actionRecorder{} w.run.Recorder = recorder - if _, err := w.run.reactivate(ctx, w.client, active, res.Reload, action, state); err != nil { + if _, err := w.run.reactivate(ctx, w.client, active, res.Reload, action, state, w.ref()); err != nil { t.Fatal(err) } if !slices.Contains(recorder.actions, "Reactivated "+action) { @@ -1326,7 +1398,7 @@ func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { read := w.resolve(t) read.Keep("r1", w.registry) ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} - owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin) + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) if err != nil { t.Fatal(err) } @@ -1376,7 +1448,7 @@ func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { if err != nil { t.Fatal(err) } - if err := w.run.inSync(ctx, w.client, active); err != nil { + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err != nil { t.Fatal(err) } config := w.statusWritten().Status.Config @@ -1395,7 +1467,7 @@ func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { w.engine.mu.Lock() w.engine.fleetDown = true w.engine.mu.Unlock() - if err := w.run.inSync(ctx, w.client, active); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || w.run.config != nil { t.Errorf("inSync over a fleet that cannot be read = %v, recording %+v; want an error that claims no race and "+ "nothing recorded", err, w.run.config) @@ -1405,7 +1477,7 @@ func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { w.engine.mu.Lock() w.engine.fleetDown, w.engine.noActivation = false, true w.engine.mu.Unlock() - if err := w.run.inSync(ctx, w.client, active); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || strings.Contains(err.Error(), "instead") || w.run.config != nil { t.Errorf("inSync over a fleet answering without its activation = %v, recording %+v; want an error that claims no "+ "race and nothing recorded", err, w.run.config) @@ -1418,7 +1490,7 @@ func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { if _, err := w.client.ReloadConfig(ctx, "reloaded again"); err != nil { t.Fatal(err) } - if err := w.run.inSync(ctx, w.client, active); !errors.Is(err, engine.ErrRevisionAdvanced) || w.run.config != nil { + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); !errors.Is(err, engine.ErrRevisionAdvanced) || w.run.config != nil { t.Errorf("inSync of r2 while the fleet's activation names r3 = %v, recording %+v; want a lost race "+ "(ErrRevisionAdvanced) and nothing recorded", err, w.run.config) } @@ -1450,7 +1522,7 @@ func TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline(t *testing.T) { if err != nil { t.Fatal(err) } - if err := w.run.inSync(ctx, w.client, active); err != nil { + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err != nil { t.Fatal(err) } h, err := w.client.Health(ctx) @@ -1478,6 +1550,421 @@ func TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline(t *testing.T) { } } +// TestAValueWrittenBackStaysOwedThroughAFailedWrite pins that a value the sync +// writes back, which leaves the hash as it was, stays a reason to write the +// document until a revision activates after it, however the reconcile that +// wrote it back ends. The store lost the operator's value; a person's reload +// activated a revision reading the store without it; the reconcile that wrote +// it back, whose own re-activation by the engine failed, could not get its +// write past the dry run. The next reconcile's sync has nothing to write, and +// the document, its values and the engine's revision all match: judged by what +// that sync wrote, the engine was in sync, and it went on running the revision +// that read the store without the value. +func TestAValueWrittenBackStaysOwedThroughAFailedWrite(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.dryRunDown = true + w.engine.mu.Unlock() + if err := w.enforce(t); err == nil { + t.Fatal("enforce() = nil over a dry run the engine could not answer") + } + if !w.engine.holds(tavily) || !slices.Equal(w.state(t).Written(), []string{tavily}) { + t.Fatalf("after the write-back: store holds %s = %t, state owes %v; want it held and owed", + tavily, w.engine.holds(tavily), w.state(t).Written()) + } + + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.dryRunDown = false + w.engine.mu.Unlock() + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3", "r4"}) { + t.Fatalf("revisions = %v, want r4 written over the reload, which read the store without %s", got, tavily) + } + if summary := w.engine.summaryOf(); !strings.Contains(summary, "Secret ns/"+requiredSecret) { + t.Errorf("r4's summary = %q, want it to name the Secret written back", summary) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("state owes %v after r4 activated, want nothing", got) + } +} + +// settled runs reconciles of w's Company, each from the status the one before +// it wrote, until one activates nothing, and returns the engine's revisions. +func (w *withdrawal) settled(t *testing.T) []string { + t.Helper() + ids := w.engine.ids() + for range 4 { + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + if now := w.engine.ids(); !slices.Equal(now, ids) { + ids = now + continue + } + return ids + } + t.Fatalf("the reconciles go on activating revisions: %v", ids) + return nil +} + +// TestASaveLostAfterAConfirmedWriteWritesNothingAgain pins that a write of the +// document the engine confirmed settles every value it read, whether or not +// the reconcile that made it got to save so. The store lost a value, the sync +// wrote it back, the engine's own re-activation at the write failed, and the +// operator's write activated a revision reading it; +// the save of the state after that write is lost, and the next reconcile +// starts from the state saved before the write, with the status that recorded +// the write or, where the reconcile went no further, the status from before +// it. Settled only by the save the reconcile lost, the value stayed owed, and +// the next reconcile wrote the same document again, an epoch every node +// applied for nothing. +func TestASaveLostAfterAConfirmedWriteWritesNothingAgain(t *testing.T) { + for _, tc := range []struct { + name string + statusLost bool + }{ + {name: "the status records the write"}, + {name: "the status is the one from before the write", statusLost: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + ids := w.settled(t) + before := w.run.company.DeepCopy() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.afterApply = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + if err := w.enforce(t); !errors.Is(err, errStateWrite) { + t.Fatalf("the reconcile that wrote the value back ends with %v, want the save after its write", err) + } + written := w.engine.ids() + if len(written) != len(ids)+1 || !w.engine.holds(tavily) { + t.Fatalf("revisions %v and the store holds %s: %t, want one write after %v and the value back", + written, tavily, w.engine.holds(tavily), ids) + } + + w.failWrite.Store(0) + next := w.statusWritten() + if tc.statusLost { + next = before + } + w.again(t, next) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, written) { + t.Fatalf("revisions = %v, want %v: the write that read the value runs", got, written) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: the revision that runs read every value", got) + } + if config := w.statusWritten().Status.Config; config.RevisionID != written[len(written)-1] { + t.Errorf("status.config records %s, want the write that runs, %s", config.RevisionID, written[len(written)-1]) + } + }) + } +} + +// TestAWriteBackWhoseSaveIsLostStaysOwed pins that a value the sync writes +// back stays owed when the save after the write is lost. The store lost the +// operator's value and a person's reload activated a revision reading the +// store without it; the sync wrote it back, the engine's own re-activation at +// the write failed, and the sync could not save that it had written it. +// Recorded only after the write, nothing said so: the next sync found the +// store holding the operator's own value under the fingerprint it had, wrote +// nothing, and the operator took the reload for in sync while the engine went +// on running it without the value. +func TestAWriteBackWhoseSaveIsLostStaysOwed(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.settled(t) + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + reloaded := w.engine.ids() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + // The save after the value is written is lost. + w.engine.mu.Lock() + w.engine.afterPut = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + if err := w.enforce(t); !errors.Is(err, errStateWrite) { + t.Fatalf("the reconcile that wrote the value back ends with %v, want the save after the write", err) + } + if !w.engine.holds(tavily) || !slices.Equal(w.engine.ids(), reloaded) { + t.Fatalf("the store holds %s: %t, revisions %v; want it written back and nothing activated after %v", + tavily, w.engine.holds(tavily), w.engine.ids(), reloaded) + } + + w.failWrite.Store(0) + w.again(t, w.statusWritten()) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); len(got) != len(reloaded)+1 { + t.Fatalf("revisions = %v, want the document written over the reload, which read the store without %s", got, tavily) + } + if summary := w.engine.summaryOf(); !strings.Contains(summary, "Secret ns/"+requiredSecret) { + t.Errorf("the write's summary = %q, want it to name the Secret written back", summary) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v after the write activated, want nothing", got) + } +} + +// TestARevisionThatReadEveryValueIsInSyncWhateverHashWasRecorded pins that the +// hash the operator recorded is no part of whether the engine is in sync: a +// revision of the rendered document that activated after every value the sync +// wrote runs the document with every value, and is recorded with the hash of +// what it runs. Held to the recorded hash, a write whose record was never +// saved, of a document whose own hash moved, was written again. +func TestARevisionThatReadEveryValueIsInSyncWhateverHashWasRecorded(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + ids := w.settled(t) + stale := w.run.company.DeepCopy() + stale.Status.Config.AppliedHash = "the hash of another document" + w.again(t, stale) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, ids) { + t.Fatalf("revisions = %v, want %v: nothing written over a revision that runs the document", got, ids) + } + if w.run.config == nil || w.run.config.AppliedHash == stale.Status.Config.AppliedHash || w.run.config.AppliedHash == "" { + t.Errorf("status.config = %+v, want the hash of what runs recorded", w.run.config) + } +} + +// TestAFailedReloadIsReadBack pins that a request to activate the running +// revision again that fails is read back: a revision the engine runs above the +// epoch the withdrawal was marked by came after the withdrawal, whoever +// activated it, and settles it. Where its document is the one read, the +// engine runs it again without the value and it is recorded as the operator's +// reactivation, its event saying the request failed; where it is another +// document, a Warning says the operator activated nothing again and takes it +// to read every value as required. A request that fails with nothing +// activated since leaves the withdrawal owed and reported. Taken for a failed +// withdrawal whatever activated, the operator reported a value the engine no +// longer read and activated a revision again for nothing. +func TestAFailedReloadIsReadBack(t *testing.T) { + // What somebody activates before the request is refused. + const ( + nothing = iota + same + another + ) + for _, tc := range []struct { + name string + other int + }{ + {name: "nothing activated", other: nothing}, + {name: "the running document activated again", other: same}, + {name: "another document activated", other: another}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.owePushed(t) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + switch tc.other { + case same: + w.engine.beforeReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + case another: + w.engine.beforeReload = func(e *storeEngine) { e.activateOther(w.requiring) } + } + w.engine.mu.Unlock() + found := time.Date(2026, 10, 25, 9, 0, 0, 0, time.UTC) + w.run.now = func() time.Time { return found } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + state := w.state(t) + if tc.other == nothing { + if !slices.Equal(state.Withdrawn(), []string{w.registry}) || len(state.Written()) == 0 || + !warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want the registry and the "+ + "values pushed before it owed, and the failure reported", state.Withdrawn(), state.Written(), recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Fatalf("the revisions are %v, want r1 and the r2 somebody activated", got) + } + if len(state.Withdrawn()) != 0 || len(state.Written()) != 0 || warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want both settled by r2, which "+ + "activated after them, and no failure reported", state.Withdrawn(), state.Written(), recorded) + } + if again := tc.other == same; state.Records("r2") != again || told(recorded, "Reactivated") != again || + warned(recorded, "Reactivated") == again { + t.Errorf("the state records r2: %t, events %q; want r2 recorded and a Normal Reactivated: %t, "+ + "a Warning otherwise", state.Records("r2"), recorded, again) + } + if tc.other != same { + return + } + if !slices.Equal(state.LetGo("r2"), []string{w.registry}) { + t.Errorf("the state lets %v go for r2, want the registry, as r1 does", state.LetGo("r2")) + } + // r1 was the operator's, so r2, its document again, is. + if config := w.run.config; config == nil || config.RevisionID != "r2" || config.Epoch == nil || + *config.Epoch != 2 || config.AppliedAt == nil || !config.AppliedAt.Time.Equal(found) { + t.Errorf("status.config = %+v, want r2 at epoch 2, applied when the operator found it, %s", config, found) + } + }) + } +} + +// TestAnActivationNoNodeCanOpenSettlesNothing pins that the fleet's +// activation settles a change only where it names the revision the read of +// the document returned, which GET /config opened: one a write activated +// between the two reads may be one no node can open, which reads the store +// for nobody. +func TestAnActivationNoNodeCanOpenSettlesNothing(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { + e.activateOther(e.revisions[len(e.revisions)-1].document) + e.unreadable = keyringLost + } + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.state(t).Withdrawn(); !slices.Equal(got, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry: r2, which the pointer names, is one no node opens", got) + } +} + +// TestAnActivationAfterAWithdrawalSettlesIt pins that a revision the engine +// runs above the epoch a withdrawal was marked by settles it, whoever +// activated it: a person's reload after a request of the operator's failed +// read the store without the value, and nothing is activated again for it. +func TestAnActivationAfterAWithdrawalSettlesIt(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.state(t).Withdrawn(); !slices.Equal(got, []string{w.registry}) { + t.Fatalf("the state owes %v after a request that failed, want the registry", got) + } + w.engine.mu.Lock() + w.engine.reloadDown = false + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Errorf("the revisions are %v, want r1 and the person's r2: nothing activated again", got) + } + if got := w.state(t).Withdrawn(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: r2 read the store without the registry", got) + } +} + +// TestAHeldReconcileSavesTheMarksItsReadSetBeforeItReloads pins that a +// held reconcile saves the epoch its read of the fleet gave a change before +// it asks for the reload that is to settle it. The withdrawal was made, the +// fleet's activation pointer could not be read after it, and the request that +// was to settle it failed, so it was saved unread. The next held reconcile +// reads the pointer, which marks it, and reloads, and the save after the +// reload is lost. Marked in memory alone, the withdrawal was loaded unread +// again, which no activation's epoch is above, and the reconcile after that +// activated the running revision a third time for a withdrawal the second had +// settled. +func TestAHeldReconcileSavesTheMarksItsReadSetBeforeItReloads(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + // The fleet answers the read that says what every pod runs, before the + // withdrawal, and not the one after it that is to mark it. + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) || !state.Unobserved() { + t.Fatalf("the state owes %v, unread: %t; want the registry, which no read of the fleet followed", + state.Withdrawn(), state.Unobserved()) + } + + w.engine.mu.Lock() + w.engine.fleetDown, w.engine.reloadDown = false, false + w.engine.afterReload = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); !errors.Is(err, errStateWrite) { + t.Fatalf("withdrawHeld() = %v, want the save after the reload lost", err) + } + reloaded := w.engine.ids() + if !slices.Equal(reloaded, []string{"r1", "r2"}) { + t.Fatalf("the revisions are %v, want r1 activated again as r2", reloaded) + } + + w.failWrite.Store(0) + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, reloaded) { + t.Errorf("the revisions are %v, want %v: r2 activated after the withdrawal, and nothing again", got, reloaded) + } + if got := w.state(t).Withdrawn(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: r2 read the store without the registry", got) + } +} + // TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument pins that inSync // judges the revision an attempt of converge read against a fleet read after // that read, in the same attempt (keepServing reads it afresh). An attempt diff --git a/internal/controller/sentence_test.go b/internal/controller/sentence_test.go index 03ce4e4b..c6035294 100644 --- a/internal/controller/sentence_test.go +++ b/internal/controller/sentence_test.go @@ -152,12 +152,12 @@ func TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce(t *testing.T) { w := newWithdrawal(t, runningLetsGo) // The operator asks for the activation: the engine did not at the removal. w.engine.reloadAnswer = new(map[string]any(nil)) - state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) if err != nil { t.Fatal(err) } w.engine.afterReload = func(e *storeEngine) { e.configDown = true } - if _, err := w.run.reactivate(ctx, w.client, nil, res.Reload, res.ReloadAction, state); err != nil { + if _, err := w.run.reactivate(ctx, w.client, nil, res.Reload, res.ReloadAction, state, w.ref()); err != nil { t.Fatal(err) } recorded := drained(w.events) diff --git a/internal/secretsync/doc.go b/internal/secretsync/doc.go index 37977159..2c133f51 100644 --- a/internal/secretsync/doc.go +++ b/internal/secretsync/doc.go @@ -73,7 +73,9 @@ // reads its store only when a revision activates, so each name removed // is recorded (State.Withdrawn) until one has, for the caller to make // one where none is applied, or the one it applied started with the -// value still there. +// value still there. Which activation came after a removal, or after a +// value written (State.Written), is told by the fleet's activation +// epoch read after it (State.Observed, State.Ran), whoever activated it. // // # What the operator owns in the store // diff --git a/internal/secretsync/engine_test.go b/internal/secretsync/engine_test.go index 8df9ca03..dd63d5aa 100644 --- a/internal/secretsync/engine_test.go +++ b/internal/secretsync/engine_test.go @@ -47,7 +47,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { } desired := desiredOf(map[string]string{slack: "xoxb", signing: "sig"}) - state, res, err := Sync(ctx, eng, desired, State{}, self) + state, res, err := Sync(ctx, eng, desired, State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -76,7 +76,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { // The engine's listing is all a second Sync has to go on besides the // state, so this is where a mismatch in how the client reports the // writer would show up as a rewrite on every reconcile. - state, res, err = Sync(ctx, eng, desired, state, self) + state, res, err = Sync(ctx, eng, desired, state, self, discard) if err != nil { t.Fatalf("second Sync: %v", err) } @@ -91,7 +91,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { if err := other.PutSecret(ctx, slack, []byte("typed-by-hand")); err != nil { t.Fatalf("overwriting %s: %v", slack, err) } - _, res, err = Sync(ctx, eng, desired, state, self) + _, res, err = Sync(ctx, eng, desired, state, self, discard) if err != nil { t.Fatalf("third Sync: %v", err) } @@ -124,7 +124,7 @@ func TestContractASyncIsOneActivation(t *testing.T) { if err != nil { t.Fatal(err) } - state, res, err := Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb", signing: "signing"}), State{}, self) + state, res, err := Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb", signing: "signing"}), State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -140,7 +140,7 @@ func TestContractASyncIsOneActivation(t *testing.T) { t.Fatalf("the engine runs revision %s, want the re-activation %s", active.RevisionID, res.Reload.RevisionID) } - _, res, err = Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb-rotated", signing: "signing"}), state, self) + _, res, err = Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb-rotated", signing: "signing"}), state, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -182,7 +182,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { self := engine.TokenLogin(enginetest.OperatorTokenID) served := StoreName(secretref.Key{Namespace: nimbusNS, Name: "nimbus-e2b", Key: apiKeyKey}) - state, _, err := Sync(ctx, eng, desiredOf(map[string]string{served: enginetest.SandboxKeyValue}), State{}, self) + state, _, err := Sync(ctx, eng, desiredOf(map[string]string{served: enginetest.SandboxKeyValue}), State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -226,7 +226,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { desired.Keep(r.RevisionID) keeping := desired keeping.KeepServed(p.RevisionID, served) - state, res, err := Sync(ctx, eng, keeping, state, self) + state, res, err := Sync(ctx, eng, keeping, state, self, discard) if err != nil { t.Fatalf("Sync keeping the served revision: %v", err) } @@ -243,7 +243,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { return servingP() }) - if _, res, err = Sync(ctx, eng, desired, state, self); err != nil || !slices.Equal(res.Deleted, []string{served}) { + if _, res, err = Sync(ctx, eng, desired, state, self, discard); err != nil || !slices.Equal(res.Deleted, []string{served}) { t.Fatalf("Sync for the fleet's revision alone = %+v, %v; want %s deleted", res, err, served) } if err := e.Restart(t); err == nil { diff --git a/internal/secretsync/state.go b/internal/secretsync/state.go index f725c478..69d817c8 100644 --- a/internal/secretsync/state.go +++ b/internal/secretsync/state.go @@ -9,7 +9,9 @@ import ( "fmt" "io" "maps" + "math" "slices" + "strconv" "strings" corev1 "k8s.io/api/core/v1" @@ -38,10 +40,14 @@ const ( fingerprintKeyLen = sha256.Size // withdrawnField is the data key of the state Secret that lists the - // withdrawn names (State.Withdrawn), one per line. Like - // fingerprintKeyField it has a dash, so Owns never accepts it, and so - // have the three after it. + // withdrawn names (State.Withdrawn), and writtenField the one that lists + // the written names (State.Written): one per line, each followed, after + // a space, by the activation epoch it waits for an activation above, + // where a read of the fleet's activation pointer has followed it + // (State.Observed). Like fingerprintKeyField each has a dash, so Owns + // never accepts it, and so have the three after them. withdrawnField = "withdrawn-names" + writtenField = "written-names" // revisionField and letGoField are the data keys of the state Secret // that hold the revision whose reading of the store the State records @@ -55,7 +61,9 @@ const ( // State is what the operator last pushed into one engine's secret store: a // fingerprint of the value of every name it pushed, and the key the -// fingerprints are computed under; the names it withdrew (Withdrawn); one +// fingerprints are computed under; the names it withdrew (Withdrawn), and +// the ones it wrote (Written), that no activation is known to have read the +// store after, with the activation epoch each waits past (Observed, Ran); one // revision of the engine's document whose reading of the store it knows, with // the Secret keys that revision lets go without (Activated, Adopted, LetGo); // and the names it last kept for the running revision although their Secret @@ -80,7 +88,12 @@ type State struct { // value its methods return changed copies of, and each copy shares it. key string fingerprints map[string][]byte - withdrawn map[string]struct{} + // withdrawn and written are the store changes no activation is known to + // have read, by name: each the epoch of the fleet's activation pointer + // read first after it, which an activation the engine runs at a greater + // epoch came after, and unread where no read has followed it yet. + withdrawn map[string]int64 + written map[string]int64 // revision is the revision whose reading the State records, empty for // none, and letGo the Secret keys its document lets go without, by // store name. @@ -91,7 +104,7 @@ type State struct { // Withdrawn are the names Sync or Withdraw removed from the store because the // optional reference each stands for has no Secret or key any more, since a -// revision last activated, sorted. +// revision activated after the removal, sorted. // // The engine reads its store when a revision activates, and starts every tool // server and executor run with what it read then: removing a name changes @@ -101,12 +114,42 @@ type State struct { // holds. So the caller activates a revision once a name is withdrawn, the // next document or the running one again, and then records it with // Activated, or with Reactivated where it activated none because the -// revision the engine runs needs no activation for it. Kept in the saved -// State, a withdrawal survives a failed activation and a restarted operator. +// revision the engine runs needs no activation for it; and any activation the +// engine is read running above the epoch the withdrawal waits past settles it +// too (Ran), whoever made it. Kept in the saved State, a withdrawal survives a +// failed activation, a lost save after it and a restarted operator. func (s State) Withdrawn() []string { return slices.Sorted(maps.Keys(s.withdrawn)) } +// Written are the names Sync wrote a value under that no activation is known +// to have read the store after, sorted: every value it pushed, whether new or +// written back over one somebody else wrote or the store lost. +// +// A written value changes nothing the engine runs until a revision activates, +// as a withdrawal does not, and a value written back leaves the fingerprints, +// and so the caller's hash of what it applied, where they were: since the +// value was replaced, a revision may have activated reading the other one, +// which it goes on running. So a revision of the caller's document is one it +// runs with every value only once a revision has activated after the last +// write (Activated, Reloaded, Ran). Sync saves the mark before the value is +// written, so a write survives a write of the document that fails or loses +// its race, a save lost after the value was written, and a restarted +// operator, which the caller's own record of what one reconcile wrote did not. +func (s State) Written() []string { + return slices.Sorted(maps.Keys(s.written)) +} + +// Reloaded is s once a revision has activated after every write and +// withdrawal it records, a revision whose document the caller does not know +// included: the engine read the store with each value written and without +// each withdrawn, so both are settled. The revision it records stays +// recorded. +func (s State) Reloaded() State { + s.withdrawn, s.written = nil, nil + return s +} + // Reactivated is s with every withdrawal it records settled: a revision has // activated since, so the engine has read the store without them, or the // revision the engine runs is one no activation is owed for (see LetGo). @@ -116,10 +159,99 @@ func (s State) Reactivated() State { return s } +// unread is the mark of a change of the store no read of the fleet's +// activation pointer has followed yet: no activation is above it, so none +// settles it but one the caller makes, and the first read after it lowers it +// to the epoch read (Observed). +const unread = math.MaxInt64 + +// Observed is s once the caller has read the fleet's activation pointer at +// epoch after every change of the store s records: each change no read has +// followed yet waits for an activation above epoch, which the pointer can +// only name once a write after this read moved it, so after the change; one +// a read followed already keeps the earlier epoch. Zero is a pointer that +// names nothing. +// +// The epoch is read after the changes, never before: one read before a change +// would settle it by an activation the pointer named before it, which read +// the store without it. And it is read before the activation the caller asks +// for, and saved with s before that is sent, so that a save lost after the +// activation leaves a change that activation settles (Ran) rather than one it +// waits past. +func (s State) Observed(epoch int64) State { + s.withdrawn = observed(s.withdrawn, epoch) + s.written = observed(s.written, epoch) + return s +} + +// Ran is s once the caller has read the fleet's activation pointer at epoch, +// naming a revision it has seen opened (the one GET /config answered): every +// change that waits for an activation above a lower epoch is settled, whoever +// made that activation and whatever its document, since it came after the +// change, and every node reads the store at its attempt to apply it, a node +// that refuses it included, whose running revision resolves its references +// from what that attempt read. A revision no node can open (one sealed under +// a key the keyring lost, or stored without a seal) is not one they read the +// store for, however the pointer names it, and settles nothing. +func (s State) Ran(epoch int64) State { + s.withdrawn = ran(s.withdrawn, epoch) + s.written = ran(s.written, epoch) + return s +} + +// SameMarks reports whether s and o record the same changes of the store, +// each with the same mark: whether a save of one leaves what a save of the +// other did. +func (s State) SameMarks(o State) bool { + return maps.Equal(s.withdrawn, o.withdrawn) && maps.Equal(s.written, o.written) +} + +// Unobserved reports whether s records a change of the store no read of the +// fleet's activation pointer has followed yet (Observed). +func (s State) Unobserved() bool { + for _, marks := range []map[string]int64{s.withdrawn, s.written} { + for _, epoch := range marks { + if epoch == unread { + return true + } + } + } + return false +} + +// observed is marks with every one above epoch lowered to it, in a map of its +// own: the State's maps are values other States share. +func observed(marks map[string]int64, epoch int64) map[string]int64 { + if marks == nil { + return nil + } + next := make(map[string]int64, len(marks)) + for name, mark := range marks { + next[name] = min(mark, epoch) + } + return next +} + +// ran is marks less every one an activation at epoch is above, in a map of +// its own, nil for none. +func ran(marks map[string]int64, epoch int64) map[string]int64 { + var left map[string]int64 + for name, mark := range marks { + if epoch > mark { + continue + } + if left == nil { + left = map[string]int64{} + } + left[name] = mark + } + return left +} + // Activated is s once the operator has activated revision, whose document -// lets go without the Secret keys optional: every withdrawal is settled, as -// Reactivated, and LetGo answers optional for revision from now on, and -// nothing for any other. +// lets go without the Secret keys optional: every withdrawal and every write +// is settled, as Reloaded, and LetGo answers optional for revision from now +// on, and nothing for any other. // // The caller records every activation it makes and knows the document of: an // apply, with render.Document.Optional of the document it applied, and an @@ -127,7 +259,7 @@ func (s State) Reactivated() State { // revision it read, once it has confirmed that the revision activated again // is that one. func (s State) Activated(revision string, optional []secretref.Key) State { - return s.Reactivated().Adopted(revision, optional) + return s.Reloaded().Adopted(revision, optional) } // Adopted is s recording revision, whose document lets go without the Secret @@ -196,27 +328,47 @@ func (s State) letsGo(revision, name string) bool { // Retained are the names the last Sync or Withdraw left in the store, // although the optional reference each stands for has no Secret or key any // more, because the revision the engine runs reads them as required, or is -// one whose reading the State does not record (Result.Retained), sorted. The caller compares them with the Retained of -// the State it started from, to tell a name it newly keeps from one it has -// already said it keeps. +// one whose reading the State does not record (Result.Retained), sorted. The +// caller compares them with the Retained of the State it started from, to +// tell a name it newly keeps from one it has already said it keeps. func (s State) Retained() []string { return slices.Sorted(maps.Keys(s.retained)) } -// withdraw records name as withdrawn in s, which it changes in place. +// withdraw records name as withdrawn in s, which it changes in place: a +// change no read has followed, whatever an earlier withdrawal of it waited +// past. func (s *State) withdraw(name string) { if s.withdrawn == nil { - s.withdrawn = map[string]struct{}{} + s.withdrawn = map[string]int64{} + } + s.withdrawn[name] = unread +} + +// write records a value written under name in s, which it changes in place, +// as withdraw records a withdrawal. +func (s *State) write(name string) { + if s.written == nil { + s.written = map[string]int64{} } - s.withdrawn[name] = struct{}{} + s.written[name] = unread +} + +// deleted records name removed from the store in s, which it changes in +// place: what was written under it is no value any revision reads now, and +// a withdrawal, where it is one, is what an activation owes for it. +func (s *State) deleted(name string) { + delete(s.fingerprints, name) + delete(s.written, name) } -// carried is a State that starts from s: its key, its record of withdrawals -// and of the activated revision, and what it last retained, in maps of its -// own where a Sync or a Withdraw changes them in place, and fingerprints. +// carried is a State that starts from s: its key, its record of withdrawals, +// of writes and of the activated revision, and what it last retained, in +// maps of its own where a Sync or a Withdraw changes them in place, and +// fingerprints. func (s State) carried(fingerprints map[string][]byte) State { return State{ - key: s.key, fingerprints: fingerprints, withdrawn: maps.Clone(s.withdrawn), + key: s.key, fingerprints: fingerprints, withdrawn: maps.Clone(s.withdrawn), written: maps.Clone(s.written), revision: s.revision, letGo: s.letGo, retained: s.retained, } } @@ -331,9 +483,10 @@ func LoadState(ctx context.Context, r client.Reader, ref types.NamespacedName, o } // A line that is no name the operator owns, or no Secret key, is // dropped, as a malformed fingerprint is: only a hand edit writes one, - // and no entry of it can be the operator's to have withdrawn, let go or - // retained. - s.withdrawn = ownedLines(secret.Data[withdrawnField]) + // and no entry of it can be the operator's to have withdrawn, written, + // let go or retained. + s.withdrawn = markLines(secret.Data[withdrawnField]) + s.written = markLines(secret.Data[writtenField]) s.revision = string(secret.Data[revisionField]) s.letGo = keyLines(secret.Data[letGoField]) s.retained = ownedLines(secret.Data[retainedField]) @@ -377,6 +530,45 @@ func parseKey(line string) (secretref.Key, bool) { return k, true } +// markLines are the lines of data that are names Owns accepts, each with the +// epoch after it, nil for none. A name with no epoch after it, or one that is +// not a number no lower than zero, is unread: only a hand edit writes the +// second, and reading it as an epoch could settle a change no activation +// read. +func markLines(data []byte) map[string]int64 { + var marks map[string]int64 + for line := range strings.Lines(string(data)) { + name, epoch, read := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + if !Owns(name) { + continue + } + mark := int64(unread) + if n, err := strconv.ParseInt(epoch, 10, 64); read && err == nil && n >= 0 { + mark = n + } + if marks == nil { + marks = map[string]int64{} + } + marks[name] = mark + } + return marks +} + +// marked is marks one per line, sorted by name, each followed by its epoch +// where a read has followed it, never nil. +func marked(marks map[string]int64) []byte { + out := []byte{} + for _, name := range slices.Sorted(maps.Keys(marks)) { + out = append(out, name...) + if mark := marks[name]; mark != unread { + out = append(out, ' ') + out = strconv.AppendInt(out, mark, 10) + } + out = append(out, '\n') + } + return out +} + // ownedLines are the lines of data that are names Owns accepts, as a set, // nil for none. func ownedLines(data []byte) map[string]struct{} { @@ -427,17 +619,18 @@ func SaveState(ctx context.Context, c client.Client, ref types.NamespacedName, o if err != nil { return fmt.Errorf("finding the kind of the owner of the secret sync state %s: %w", ref, err) } - data := make(map[string][]byte, len(s.fingerprints)+5) + data := make(map[string][]byte, len(s.fingerprints)+6) maps.Copy(data, s.fingerprints) if len(s.key) > 0 { data[fingerprintKeyField] = []byte(s.key) } // Written when they are empty too, so this field manager owns them: a // list it left out would stay where a hand edit put it, and every - // reconcile would activate a revision for names it withdrew long ago, - // or take a revision whose reading it does not know for one that lets - // a name go. - data[withdrawnField] = lines(s.Withdrawn()) + // reconcile would activate a revision for names it withdrew or wrote + // long ago, or take a revision whose reading it does not know for one + // that lets a name go. + data[withdrawnField] = marked(s.withdrawn) + data[writtenField] = marked(s.written) data[revisionField] = []byte(s.revision) keys := slices.SortedFunc(maps.Values(s.letGo), secretref.Compare) printed := make([]string, len(keys)) diff --git a/internal/secretsync/state_test.go b/internal/secretsync/state_test.go index 8f17e5eb..f63c510e 100644 --- a/internal/secretsync/state_test.go +++ b/internal/secretsync/state_test.go @@ -153,6 +153,49 @@ func TestSaveStateKeepsTheWithdrawnNames(t *testing.T) { } } +// TestSaveStateKeepsTheWrittenNames: the names written since a revision +// last activated after them survive a save and a load, each with the epoch +// it waits past or none, so a write of the document that fails, or an +// operator that restarts, leaves them owed exactly as they were; a line that +// names no entry the operator owns, which only a hand edit writes, is +// dropped, one whose epoch is no number no lower than zero waits for a read +// as one no read followed does, and an activation empties the list. +func TestSaveStateKeepsTheWrittenNames(t *testing.T) { + ctx := context.Background() + c := newStateClient(t) + eng := newFakeEngine() + state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), State{}) + state, _ = mustSync(t, eng, desiredOf(map[string]string{nameA: "a1", nameB: "b1"}), state.Observed(3)) + if err := SaveState(ctx, c, stateRef, company(), state); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + if want := nameA + " 3\n" + nameB + "\n"; string(secret.Data[writtenField]) != want { + t.Fatalf("written-names = %q, want %q", secret.Data[writtenField], want) + } + secret.Data[writtenField] = append(secret.Data[writtenField], []byte("not-a-store-name 4\n"+nameC+" -1\n")...) + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if !slices.Equal(loaded.Written(), []string{nameA, nameB, nameC}) { + t.Fatalf("Written() after a load = %v, want [%s %s %s]", loaded.Written(), nameA, nameB, nameC) + } + if got := loaded.Ran(4).Written(); !slices.Equal(got, []string{nameB, nameC}) { + t.Fatalf("Written() after a load and an activation at epoch 4 = %v, want [%s %s]", got, nameB, nameC) + } + + if err := SaveState(ctx, c, stateRef, company(), loaded.Activated(revision1, nil)); err != nil { + t.Fatalf("SaveState: %v", err) + } + if got := stateSecret(t, c).Data[writtenField]; len(got) != 0 { + t.Errorf("the state Secret keeps the written names %q after a revision activated", got) + } +} + // TestActivationIsRecordedForItsRevisionAlone: what Activated records of the // operator's last activation answers for that revision and for no other, an // empty one included, the Secret keys it lets go as well as their names, @@ -269,7 +312,7 @@ func TestSaveStateForgetsWhatTheStateDropped(t *testing.T) { t.Fatalf("SaveState: %v", err) } got := slices.Sorted(maps.Keys(stateSecret(t, c).Data)) - want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, revisionField, letGoField, retainedField})) + want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, writtenField, revisionField, letGoField, retainedField})) if !slices.Equal(got, want) { t.Fatalf("state Secret keys = %v, want %v", got, want) } diff --git a/internal/secretsync/sync.go b/internal/secretsync/sync.go index 2481aa27..37eb075d 100644 --- a/internal/secretsync/sync.go +++ b/internal/secretsync/sync.go @@ -143,10 +143,11 @@ type Overwrite struct { // one activation however many requests carry it. Where the last request // re-activates nothing, because the revision reads none of its names, or its // re-activation fails, Result.Reload says so and the caller activates the -// revision itself. A value written deferred is recorded as written (a -// fingerprint in the State) only once the last write request has been -// answered: a sync that fails, or stops, before that leaves it to be written -// again by the next, whose last request activates it. The engine re-activates +// revision itself. A value written deferred gets its fingerprint in the +// State only once the last write request has been answered: a sync that +// fails, or stops, before that leaves it to be written again by the next, +// whose last request activates it. Every value is recorded as owed +// (State.Written) before the first request, as below. The engine re-activates // the revision at the removal of a value it reads too, one removal at a time, // which is what Result.Reload reports where one comes last. // @@ -160,7 +161,20 @@ type Overwrite struct { // state: a retry from it repeats only what is not known to have happened. // A name whose write failed has no fingerprint in it, because the engine may // or may not have stored the value. -func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, self string) (State, Result, error) { +// +// Before it writes the first value, Sync hands save the State with every name +// it is about to write recorded as written (State.Written), and the +// fingerprints as they were, and writes nothing unless save took it. A value +// changes nothing the engine runs until a revision activates after it, and +// one written back leaves its fingerprint where it was: recorded only after +// the write, a save lost between the two left a value no revision read and +// nothing that said so, and the next Sync, finding the store holding its own +// value under the fingerprint it had, had nothing left to write. A name saved +// as written whose value never reached the store owes an activation for +// nothing, which costs one; the other way cost the engine the value. +func Sync( + ctx context.Context, eng EngineSecrets, desired Desired, prev State, self string, save func(State) error, +) (State, Result, error) { var res Result if desired.partial { return prev, res, errors.New("syncing the engine's secret store: the values were not all read, " + @@ -199,7 +213,11 @@ func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, s } } - if err := push(ctx, eng, desired, listed, &next, self, &res); err != nil { + writes := next.toWrite(desired, listed, self) + if err := next.writeAhead(writes, save); err != nil { + return next, res, err + } + if err := push(ctx, eng, desired, writes, listed, &next, self, &res); err != nil { return next, res, err } @@ -233,32 +251,35 @@ func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, s return next, res, fmt.Errorf("deleting %s, %s, from the engine's secret store: %w", name, what, err) } res.removed(removed) - delete(next.fingerprints, name) + next.deleted(name) res.Deleted = append(res.Deleted, name) if absent { next.withdraw(name) } } res.Retained, next.retained = retained, nameSet(retained) + // A value written that the store no longer holds, and that this sync + // did not write again, is none the next activation reads. + for name := range next.written { + if _, ok := next.fingerprints[name]; !ok { + delete(next.written, name) + } + } return next, res, nil } -// push writes every value of desired that the engine does not hold as the -// operator last wrote it, as Sync says, recording each written in next and -// in res, with the engine's last re-activation; a failed write stops it. +// push writes the values of desired named by writes (toWrite), as Sync +// says, recording each written in next and in res, with the engine's last +// re-activation; a failed write stops it. func push( - ctx context.Context, eng EngineSecrets, desired Desired, listed map[string]engine.SecretInfo, next *State, - self string, res *Result, + ctx context.Context, eng EngineSecrets, desired Desired, writes []string, listed map[string]engine.SecretInfo, + next *State, self string, res *Result, ) error { var single, batched []string fingerprints := map[string][]byte{} - for _, name := range desired.Names() { + for _, name := range writes { value := desired.values[name] fp := next.fingerprint(name, value) - info, ok := listed[name] - if ok && info.OperatorID == self && hmac.Equal(next.fingerprints[name], fp) { - continue - } // Until the write is answered the engine may or may not hold it. delete(next.fingerprints, name) fingerprints[name] = fp @@ -354,6 +375,43 @@ func sortWritten(res *Result) { slices.SortFunc(res.Overwritten, func(a, b Overwrite) int { return strings.Compare(a.Name, b.Name) }) } +// toWrite are the names of desired Sync writes, given what the engine lists: +// those it does not list, lists as written with a credential other than self, +// or holds under another fingerprint than s does. +func (s State) toWrite(desired Desired, listed map[string]engine.SecretInfo, self string) []string { + var writes []string + for _, name := range desired.Names() { + info, ok := listed[name] + if !ok || info.OperatorID != self || !hmac.Equal(s.fingerprints[name], s.fingerprint(name, desired.values[name])) { + writes = append(writes, name) + } + } + return writes +} + +// writeAhead records every name of writes as written in s, which it changes +// in place, once save has taken a State of its own recording them so, the +// fingerprints as they were: the writes Sync makes next leave what save took +// as it was. It records and saves nothing for no writes, and records nothing +// where save fails. +func (s *State) writeAhead(writes []string, save func(State) error) error { + if len(writes) == 0 { + return nil + } + ahead := s.carried(maps.Clone(s.fingerprints)) + for _, name := range writes { + ahead.write(name) + } + if err := save(ahead); err != nil { + return fmt.Errorf("recording the values about to be written into the engine's secret store, %s: %w", + strings.Join(writes, ", "), err) + } + for _, name := range writes { + s.write(name) + } + return nil +} + // Withdraw removes from the engine's store the entry of every absent name of // desired (Desired.Missing) that it lists, but one a revision kept reads as // required (Desired.Keep, Desired.KeepServed), and of every name a revision @@ -413,7 +471,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat for _, name := range gone { _, pushed := prev.fingerprints[name] if _, removed := desired.gone(name, prev); pushed && removed && !listed[name] { - delete(next.fingerprints, name) + next.deleted(name) next.withdraw(name) } } @@ -424,7 +482,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat name, err) } res.removed(removed) - delete(next.fingerprints, name) + next.deleted(name) next.withdraw(name) res.Deleted = append(res.Deleted, name) } diff --git a/internal/secretsync/sync_test.go b/internal/secretsync/sync_test.go index 9bb2bece..eb07b15f 100644 --- a/internal/secretsync/sync_test.go +++ b/internal/secretsync/sync_test.go @@ -228,13 +228,17 @@ func desiredOf(values map[string]string) Desired { func mustSync(t *testing.T, eng *fakeEngine, desired Desired, prev State) (State, Result) { t.Helper() - next, res, err := Sync(context.Background(), eng, desired, prev, self) + next, res, err := Sync(context.Background(), eng, desired, prev, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } return next, res } +// discard is a save that takes every State Sync hands it ahead of a write, +// and keeps none. +func discard(State) error { return nil } + func assertStore(t *testing.T, eng *fakeEngine, want map[string]fakeEntry) { t.Helper() if !maps.Equal(eng.entries, want) { @@ -361,7 +365,7 @@ func TestSyncSplitsALargeSetIntoBatches(t *testing.T) { } boom := errors.New("engine unavailable") eng.failPut[second[0]] = boom - partial, res, err := Sync(context.Background(), eng, desiredOf(values), State{}, self) + partial, res, err := Sync(context.Background(), eng, desiredOf(values), State{}, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -403,7 +407,7 @@ func TestASyncWhoseLastWriteFailsActivatesItsValuesNextTime(t *testing.T) { eng.reads[name] = true } eng.failPut[last] = errors.New("engine unavailable") - partial, _, err := Sync(context.Background(), eng, desiredOf(values), State{}, self) + partial, _, err := Sync(context.Background(), eng, desiredOf(values), State{}, self, discard) if err == nil || len(eng.reloaded) != 0 || len(partial.fingerprints) != 0 { t.Fatalf("a failed last write = %v, activated %v, recorded %d names; want the error, no activation "+ "and nothing recorded", err, eng.reloaded, len(partial.fingerprints)) @@ -609,7 +613,7 @@ func TestSyncReturnsWhatItAchieved(t *testing.T) { desired := desiredOf(map[string]string{nameA: "a1", nameB: "b2"}) boom := errors.New("engine unavailable") eng.failPut[nameB] = boom - partial, res, err := Sync(context.Background(), eng, desired, state, self) + partial, res, err := Sync(context.Background(), eng, desired, state, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -626,7 +630,7 @@ func TestSyncReturnsWhatItAchieved(t *testing.T) { delete(eng.failPut, nameB) eng.failDelete[nameC] = boom - partial, res, err = Sync(context.Background(), eng, desired, partial, self) + partial, res, err = Sync(context.Background(), eng, desired, partial, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -656,7 +660,7 @@ func TestSyncChangesNothingWhenTheListingFails(t *testing.T) { eng.takeCalls() eng.failList = errors.New("connection refused") - got, res, err := Sync(context.Background(), eng, desiredOf(map[string]string{nameB: "b1"}), state, self) + got, res, err := Sync(context.Background(), eng, desiredOf(map[string]string{nameB: "b1"}), state, self, discard) if !errors.Is(err, eng.failList) { t.Fatalf("Sync() error = %v", err) } @@ -821,7 +825,7 @@ func TestSyncRefusesAPartialDesired(t *testing.T) { eng := newFakeEngine() state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a"}), State{}) eng.takeCalls() - next, _, err := Sync(context.Background(), eng, Desired{partial: true}, state, self) + next, _, err := Sync(context.Background(), eng, Desired{partial: true}, state, self, discard) if err == nil { t.Fatal("Sync took a partial Desired") } @@ -1177,3 +1181,203 @@ func TestSyncKeepsWhatEveryNodeRuns(t *testing.T) { } }) } + +// TestAWrittenValueIsOwedUntilARevisionActivatesAfterIt: every value Sync +// writes, rotated or written back over one the store lost or somebody else +// wrote, is owed an activation (State.Written) until a revision activates +// after it. A later sync with nothing to write keeps it owed, and so does +// settling the withdrawals alone (Reactivated): the write-back leaves the +// fingerprints as they were, so nothing else would say a revision may have +// read the other value. An activation the operator makes (Activated), or any +// revision activated after it (Reloaded), settles it, and a value Sync +// deletes, or one the store no longer holds and nothing writes again, owes +// nothing. +func TestAWrittenValueIsOwedUntilARevisionActivatesAfterIt(t *testing.T) { + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + if got := state.Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after the first sync = %v, want [%s %s]", got, nameA, nameB) + } + if got := state.Activated(revision1, nil).Written(); len(got) != 0 { + t.Fatalf("Written() after an activation = %v, want none", got) + } + state = state.Activated(revision1, nil) + + delete(eng.entries, nameA) + state, res := mustSync(t, eng, both, state) + if !slices.Equal(res.Put, []string{nameA}) || !slices.Equal(state.Written(), []string{nameA}) { + t.Fatalf("a value the store lost: Put = %v, Written() = %v, want [%s] for both", res.Put, state.Written(), nameA) + } + again, res := mustSync(t, eng, both, state) + if len(res.Put) != 0 || !slices.Equal(again.Written(), []string{nameA}) { + t.Fatalf("a sync with nothing to write: Put = %v, Written() = %v, want nothing put and [%s] still owed", + res.Put, again.Written(), nameA) + } + if got := again.Reactivated().Written(); !slices.Equal(got, []string{nameA}) { + t.Errorf("Written() once the withdrawals alone were settled = %v, want [%s] still owed", got, nameA) + } + if got := again.Reloaded().Written(); len(got) != 0 { + t.Errorf("Written() once a revision activated after the write = %v, want none", got) + } + + eng.entries[nameB] = fakeEntry{value: "theirs", by: access} + state, _ = mustSync(t, eng, both, again.Activated(revision2, nil)) + if got := state.Written(); !slices.Equal(got, []string{nameB}) { + t.Fatalf("Written() after a value somebody else wrote was written back = %v, want [%s]", got, nameB) + } + state, res = mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), state) + if !slices.Equal(res.Deleted, []string{nameB}) || len(state.Written()) != 0 { + t.Fatalf("a written value Sync deleted: Deleted = %v, Written() = %v, want [%s] deleted and nothing owed", + res.Deleted, state.Written(), nameB) + } + + state, _ = mustSync(t, eng, desiredOf(map[string]string{nameA: "a2"}), state) + gone := Desired{absent: map[string]*MissingError{nameA: {Key: keyA}}} + withdrawn, _, err := Withdraw(context.Background(), eng, gone, state) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + if got := withdrawn.Written(); len(got) != 0 { + t.Errorf("Written() of a value Withdraw deleted = %v, want none: its withdrawal is what is owed", got) + } + + delete(eng.entries, nameA) + state, _ = mustSync(t, eng, desiredOf(nil), state) + if got := state.Written(); len(got) != 0 { + t.Errorf("Written() of a value the store lost and nothing writes again = %v, want none", got) + } +} + +// TestSyncSavesAWriteBeforeItWritesTheValue: Sync hands its caller the State +// with every name it is about to write recorded as written, and the +// fingerprints as they were, before it writes the first value, and writes +// nothing the caller did not save. A value written back leaves its +// fingerprint where it was, so a save lost after the write would otherwise +// leave nothing that said a revision has yet to read it: the next Sync finds +// the store holding the operator's own value under that fingerprint and has +// nothing left to write. And a rotated value saved under its new fingerprint +// before it reached the store would never be written, the next Sync taking +// the old value it finds there for the new one. +func TestSyncSavesAWriteBeforeItWritesTheValue(t *testing.T) { + ctx := context.Background() + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + state = state.Activated(revision1, nil) + + // The store lost a1 and somebody wrote over b1: both are written back. + delete(eng.entries, nameA) + eng.entries[nameB] = fakeEntry{value: "theirs", by: access} + var saved []State + _, res, err := Sync(ctx, eng, both, state, self, func(ahead State) error { + if eng.holds(nameA) || eng.entries[nameB].by != access { + t.Error("Sync saved the State after it wrote a value") + } + saved = append(saved, ahead) + return nil + }) + if err != nil || !slices.Equal(res.Put, []string{nameA, nameB}) { + t.Fatalf("Sync() = %+v, %v, want %s and %s written back", res, err, nameA, nameB) + } + if len(saved) != 1 || !slices.Equal(saved[0].Written(), []string{nameA, nameB}) { + t.Fatalf("saved ahead of the writes: %d States, want one recording [%s %s] written", len(saved), nameA, nameB) + } + // The save after the writes is lost: the next sync starts from the one + // ahead of them. b1, written back under the fingerprint it had, has + // nothing left to write, and a1, which the store had lost and so had no + // fingerprint, is written once more; both are still owed. + again, res := mustSync(t, eng, both, saved[0]) + if !slices.Equal(res.Put, []string{nameA}) || !slices.Equal(again.Written(), []string{nameA, nameB}) { + t.Fatalf("a sync from the State saved ahead of the writes: Put = %v, Written() = %v, want [%s] put and "+ + "[%s %s] still owed", res.Put, again.Written(), nameA, nameA, nameB) + } + + // A rotated value whose write fails after the save: the State saved + // ahead of it keeps the old fingerprint, so the next sync writes it. + state = again.Activated(revision2, nil) + rotated := desiredOf(map[string]string{nameA: "a2", nameB: "b1"}) + boom := errors.New("engine unavailable") + eng.failPut[nameA] = boom + saved = nil + if _, _, err := Sync(ctx, eng, rotated, state, self, func(ahead State) error { + saved = append(saved, ahead) + return nil + }); !errors.Is(err, boom) || len(saved) != 1 { + t.Fatalf("Sync() error = %v after %d saves, want the engine's after one", err, len(saved)) + } + delete(eng.failPut, nameA) + if _, res := mustSync(t, eng, rotated, saved[0]); !slices.Equal(res.Put, []string{nameA}) { + t.Fatalf("a sync from the State saved ahead of a rotation that never reached the store: Put = %v, want [%s]", + res.Put, nameA) + } + + // A save that fails writes nothing; a sync with nothing to write saves + // nothing ahead. + delete(eng.entries, nameA) + eng.takeCalls() + refused := errors.New("the API server did not answer") + if _, res, err := Sync(ctx, eng, rotated, state, self, func(State) error { return refused }); !errors.Is(err, refused) || + len(res.Put) != 0 { + t.Fatalf("Sync() = %+v, %v with a save that failed, want the save's error and nothing written", res, err) + } + assertCalls(t, eng, "list") + settled, _ := mustSync(t, eng, rotated, state) + if _, _, err := Sync(ctx, eng, rotated, settled.Activated(revision1, nil), self, func(State) error { + t.Error("Sync saved ahead of a sync that writes nothing") + return nil + }); err != nil { + t.Fatalf("Sync: %v", err) + } +} + +// TestAChangeIsSettledByAnActivationAboveTheEpochReadAfterIt: a write or a +// withdrawal waits for an activation above the epoch of the fleet's +// activation pointer read first after it (Observed), and any activation the +// engine is read running above it settles it (Ran), whoever made it. One no +// read has followed yet is settled by no epoch at all, since the activation a +// read names may predate the change; a later read keeps the earlier epoch; and +// a name changed again after a read waits for a read after that change. +func TestAChangeIsSettledByAnActivationAboveTheEpochReadAfterIt(t *testing.T) { + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + if !state.Unobserved() { + t.Fatal("Unobserved() = false after a sync wrote two values") + } + if got := state.Ran(1 << 40).Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after an activation with no read after the writes = %v, want both still owed", got) + } + state = state.Observed(7) + if state.Unobserved() { + t.Fatal("Unobserved() = true after a read") + } + if got := state.Ran(7).Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after the activation the read named = %v, want both still owed", got) + } + if got := state.Ran(8).Written(); len(got) != 0 { + t.Fatalf("Written() after an activation above the read = %v, want none", got) + } + if got := state.Observed(9).Ran(8).Written(); len(got) != 0 { + t.Fatalf("Written() after a later read and an activation above the first = %v, want none", got) + } + + delete(eng.entries, nameA) + state, _ = mustSync(t, eng, both, state) + if got := state.Ran(8).Written(); !slices.Equal(got, []string{nameA}) { + t.Fatalf("Written() of a value written again after the read = %v, want [%s] still owed", got, nameA) + } + + gone := Desired{absent: map[string]*MissingError{nameB: {Key: keyB}}} + withdrawn, _, err := Withdraw(context.Background(), eng, gone, state.Activated(revision1, []secretref.Key{keyB})) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + withdrawn = withdrawn.Observed(10) + if got := withdrawn.Ran(10).Withdrawn(); !slices.Equal(got, []string{nameB}) { + t.Fatalf("Withdrawn() after the activation the read named = %v, want [%s] still owed", got, nameB) + } + if got := withdrawn.Ran(11).Withdrawn(); len(got) != 0 { + t.Fatalf("Withdrawn() after an activation above the read = %v, want none", got) + } +} From 918c6fa75cfaf9b8b887bf92324d3e418216b801 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 14:00:17 +0000 Subject: [PATCH 14/36] fix(controller): settle the store when the engine refuses a document The secret sync runs before the dry run, so by the time the engine refuses the rendered document the sync may have removed from the store the value of a deleted Secret that the revision the engine runs lets go without. The engine reads its store only when a revision activates, so the operator activates the running revision again for it (settle). It did that only for a document the engine found invalid. A document the engine refused with 409 seat_held (a human seat somebody holds that the document takes away), 403 config_managed (a token the engine does not let change the document) or a 403 the token lacks a grant for went back as an error, which engineFailed reported and nothing settled. For as long as the seat was held the engine went on starting tool servers and executor runs with the value of a Secret that no longer exists, while the applying concept page said that whenever the reconcile ends without an activation after it deleted a value, the operator activates the running revision again. Now every refusal the Company's conditions report as what holds the document is one (refusedByEngine), for the dry run and the write alike: the refusal is recorded as before (invalid, seatHeld, writeRefused, notGranted) and the activation owed is made (settleRefused). Neither a held seat nor a managed document refuses a reload of the running revision, which is admin's; a token without the grant meets the same refusal on the reload, which is reported as a failed withdrawal. A refusal the engine's own users clear is looked at again ten seconds on as before, since nothing in the cluster changes when they do, or sooner where a failed activation's report promises it; a document the engine finds invalid still waits for an object to change, or for the pointer poll while a value is kept for a revision other than the fleet's. storeEngine can refuse every write of the document with seat_held or config_managed (refusal). TestADocumentTheEngineHoldsStillSettlesTheStore fails with those refusals left to engineFailed, with the look ten seconds on lost or made later than a failed activation's report says, and with the activation not made. The applying concept page names the engine's refusals among what holds a document, and the troubleshooting guide's SeatHeld entry says what happens to a value removed while it holds. Signed-off-by: Ali Yousefian --- docs/concepts/applying.md | 22 +-- docs/guides/troubleshooting.md | 10 +- internal/controller/config_run.go | 55 +++++++- internal/controller/config_withdraw_test.go | 145 ++++++++++++++++++-- 4 files changed, 202 insertions(+), 30 deletions(-) diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index ebdfa0e1..564163fd 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -201,16 +201,18 @@ flowchart TD without, and is told by the event alone. Step 7 deletes the value once the next document runs. Whenever the reconcile ends without activating a revision after it deleted a value, because the document is held at any - step, or because the engine already runs it with every value it - references (step 4), the revision the engine runs is activated again, - unchanged, so the engine reads its store without the deleted value rather - than starting a tool server or an executor run with it (a `Reactivated` - event): where that revision read the value, the engine did so itself as - the operator removed it, and the operator takes that activation for its - own; otherwise, or where the engine's failed, it asks for one - (`POST /config/reload`). It keeps what it owes in its record of the store - until the engine runs a revision activated after the deletion, whoever - activated it ([What the engine has read](#what-the-engine-has-read)). + step (the engine's own refusal of it included: a document it finds + invalid, a human seat somebody holds that the document takes away, or + the operator's token refused the change), or because the engine already + runs it with every value it references (step 4), the revision the engine + runs is activated again, unchanged, so the engine reads its store without + the deleted value rather than starting a tool server or an executor run + with it (a `Reactivated` event): where that revision read the value, the + engine did so itself as the operator removed it, and the operator takes + that activation for its own; otherwise, or where the engine's failed, it + asks for one (`POST /config/reload`). It keeps what it owes in its record + of the store until the engine runs a revision activated after the + deletion, whoever activated it ([What the engine has read](#what-the-engine-has-read)). It activates again only a revision that goes without the value, never one that keeps it, and records the revision that activates it again as its own only once it has read back that it is that document: the engine diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 36968690..ce12fd6e 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -432,9 +432,13 @@ while something in the engine's identity directory holds that seat. A person holds a human seat for as long as they are in the company, an open invitation holds the seat it invites to, and a service account may be bound to one. The engine keeps running the revision it runs, and the operator tries again every -ten seconds. The message names each seat and what kind of thing holds it, -never who; a Seat that still exists and is held reports `Applied=False -SeatHeld` too. An admin of the company frees each seat first, in the engine's +ten seconds. When it removes a value from the store meanwhile, the value of a +deleted Secret that revision lets go without, it activates that revision +again so that the engine reads the store without it, as for any document the +engine refuses (step 3 of +[What one reconcile does](../concepts/applying.md#what-one-reconcile-does)). +The message names each seat and what kind of thing holds it, never who; a +Seat that still exists and is held reports `Applied=False SeatHeld` too. An admin of the company frees each seat first, in the engine's dashboard or with [`crewlet iam`](https://github.com/crewlet/crewlet/blob/main/docs/concepts/identity-and-access.md): moves the person to another human seat or removes them, unbinds the service account, or cancels the invitation. `crewlet iam seats` lists what holds each diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 58f3f7ce..5b75f031 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -568,10 +568,8 @@ func (run *configRun) enforce( checked, err := eng.ApplyConfig(ctx, engine.ApplyRequest{Document: run.doc.YAML, DryRun: true}) if err != nil { - if run.invalid(err) { - // The sync above may have withdrawn a value the running - // revision lets go: it is activated again instead. - return run.settleRefused(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) + if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef, err); refused { + return result, saveErr } return ctrl.Result{}, err } @@ -602,8 +600,8 @@ func (run *configRun) enforce( } applied, err := eng.ApplyConfig(ctx, req) if err != nil { - if run.invalid(err) { - return run.settleRefused(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) + if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef, err); refused { + return result, saveErr } return ctrl.Result{}, err } @@ -658,6 +656,51 @@ func (run *configRun) cleanUp( return run.heldResync(ctrl.Result{RequeueAfter: applyPollInterval}), nil } +// refusedByEngine ends the reconcile where the engine refused the rendered +// document, its dry run or its write, with err, for what no retry of the +// write changes and the Company's conditions now say: problems with the +// document (invalid), a human seat somebody holds that it takes away +// (seatHeld), or the operator's token refused the change (writeRefused, +// notGranted). It reports false for any other error, which goes back as one +// (engineFailed). +// +// Whatever the refusal, the sync before it may have changed the store under +// the revision the engine runs, and no write is coming to activate one that +// reads the change, so the activation owed is made now (settleRefused, which +// takes the sync's reload and reloadAction, as settle does), +// before the reconcile ends: a refused document went back as an error, which +// nothing settled, and for as long as somebody held the seat the engine went +// on starting tool servers and executor runs with the value of a Secret that no +// longer exists. Neither a held seat nor a managed document stops a reload of +// the running revision, which is admin's. A refusal the engine's own users +// clear is looked at again as engineRetry says, since nothing in the cluster +// changes when they do; one the Company's objects clear, a document the +// engine finds invalid, when one of them changes. +func (run *configRun) refusedByEngine( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, + state secretsync.State, ref types.NamespacedName, err error, +) (ctrl.Result, bool, error) { + switch { + case run.invalid(err): + result, saveErr := run.settleRefused(ctx, eng, active, reload, reloadAction, state, ref) + return result, true, saveErr + case run.seatHeld(err), run.writeRefused(err), run.notGranted(err): + result, saveErr := run.settleRefused(ctx, eng, active, reload, reloadAction, state, ref) + if saveErr != nil { + return result, true, saveErr + } + wait := run.engineRetry(err) + if result.RequeueAfter > 0 { + // The look a failed activation's report promises, or the + // pointer poll a value kept for another revision asks for + // (heldResync), where that is sooner. + wait = min(wait, result.RequeueAfter) + } + return ctrl.Result{RequeueAfter: wait}, true, nil + } + return ctrl.Result{}, false, nil +} + // adopt is state recording the reading of active, the revision the engine // runs, where its document is the one this reconcile rendered: diffs, where // the two differ, is empty, and the engine's document references every store diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 13145cf0..070973a5 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,7 +46,7 @@ import ( // storeEngine answers the engine's /health, /config, /config/revisions, // /fleet and /secrets routes from a revision history, the last of which is // the fleet's, and a store, as the operator's client reads them. A document -// it is to refuse is refused as the engine refuses an invalid one. +// it is to refuse is refused as refusal says. // beforeReload, afterReload and afterApply, when set, run once, under its // lock, just before a reload is answered and just after a reload or an apply // is: somebody else's write landing between two of the operator's. Its fleet @@ -79,10 +80,16 @@ import ( // in place of that, as a data node whose record of it the engine has not // moved yet to the revision it runs, "" for none. type storeEngine struct { - mu sync.Mutex - revisions []storeRevision - store map[string]string - refuse bool + mu sync.Mutex + revisions []storeRevision + store map[string]string + // refusal, when set, is how the engine refuses every PUT of the document, + // the dry run included: invalidDocument, a document it finds invalid, or + // for what only its own users can clear, heldSeat, a human seat somebody + // holds that the document takes away, or managedDocument, a token the + // engine does not let change the document. A reload it answers as ever: + // none of them refuses one. + refusal *engineRefusal configDown, secretsDown bool reloadDown bool beforeReload, afterReload func(e *storeEngine) @@ -166,6 +173,35 @@ func newStoreEngine(t *testing.T, document []byte) (*storeEngine, *engine.Client return e, engine.New(server.URL, "token") } +// engineRefusal is an answer the engine refuses a document with, as it +// writes it. +type engineRefusal struct { + status int + body string +} + +func (refusal *engineRefusal) answer(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(refusal.status) + _, _ = io.WriteString(w, refusal.body) +} + +var ( + // invalidDocument is the engine's refusal of a document it finds + // invalid. + invalidDocument = &engineRefusal{status: http.StatusBadRequest, body: `{"error":"validation_error",` + + `"detail":"refused","problems":[{"path":"roles","kind":"invalid","message":"refused"}]}`} + // heldSeat is the engine's refusal of a document that takes away a + // human seat a person holds. + heldSeat = &engineRefusal{status: http.StatusConflict, body: `{"error":"seat_held",` + + `"detail":"This change removes human seats that people still hold: founder (held by a person).",` + + `"held":{"founder":{"person":"p-1","kind":"person","stage":"active"}}}`} + // managedDocument is the engine's refusal of a write by a token that + // api.auth.company_writers does not name. + managedDocument = &engineRefusal{status: http.StatusForbidden, body: `{"error":"config_managed",` + + `"managed_by":["ci"],"detail":"the company document is managed by ci"}`} +) + func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { e.mu.Lock() defer e.mu.Unlock() @@ -201,9 +237,8 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { "applied_epoch": len(e.revisions) - e.appliedLag}) case r.Method == http.MethodGet && r.URL.Path == configRoute: e.readActive(w, active) - case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: - answer(http.StatusBadRequest, map[string]any{"error": "validation_error", "detail": "refused", - "problems": []map[string]string{{"path": "roles", "kind": "invalid", "message": "refused"}}}) + case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refusal != nil: + e.refusal.answer(w) case r.Method == http.MethodPut && r.URL.Path == configRoute && dryRun(r): answer(http.StatusOK, map[string]any{"base_revision_id": active.id}) case r.Method == http.MethodPut && r.URL.Path == configRoute: @@ -784,6 +819,26 @@ func (w *withdrawal) enforced(t *testing.T) (ctrl.Result, error) { return w.run.enforce(ctx, w.client, *w.run.desired, active) } +// rotatedValue is the value rotate gives the required Secret. +const rotatedValue = "tvly-rotated" + +// rotate gives the Secret behind the required value a new one, as somebody +// rotating a credential does: the next sync writes it into the engine's +// store, and what the engine runs reads it only once a revision activates +// after it. +func (w *withdrawal) rotate(t *testing.T) { + t.Helper() + ctx := context.Background() + rotated := &corev1.Secret{} + if err := w.cluster.Get(ctx, types.NamespacedName{Namespace: "ns", Name: requiredSecret}, rotated); err != nil { + t.Fatal(err) + } + rotated.Data[requiredKey] = []byte(rotatedValue) + if err := w.cluster.Update(ctx, rotated); err != nil { + t.Fatal(err) + } +} + // told reports whether the reconcile recorded a Normal event of reason, and // warned whether a Warning one. func told(recorded []string, reason string) bool { @@ -977,7 +1032,7 @@ func TestARevisionThatLetsAValueGoGoesWithoutItWhateverTheNextDocumentReads(t *t t.Fatal(err) } w.run.doc = doc - w.engine.refuse = true + w.engine.refusal = invalidDocument if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -1046,7 +1101,9 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { } t.Run(name, func(t *testing.T) { w := newWithdrawal(t, tc.running) - w.engine.refuse = tc.refused + if tc.refused { + w.engine.refusal = invalidDocument + } if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -1407,7 +1464,7 @@ func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { } // The dry run is refused, and the activation owed then meets an engine // that cannot say who is asking, for two seconds. - w.engine.refuse = true + w.engine.refusal = invalidDocument w.engine.reloadDown = true result, err := w.run.converge(ctx, w.client) if err != nil { @@ -1421,6 +1478,72 @@ func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { } } +// TestADocumentTheEngineHoldsStillSettlesTheStore pins that a document the +// engine refuses for what only its own users can clear, a human seat somebody +// holds or a token it does not let change the document, settles what the +// store owes as a document it refuses as invalid does: the running revision is +// activated again without a value whose Secret was deleted, and the refusal is +// reported and looked at again ten seconds on, or sooner where an activation +// that failed says so. It went back as an error, which nothing settled, so for +// as long as the seat was held the engine went on starting tool servers and +// executor runs with the deleted Secret's value. +func TestADocumentTheEngineHoldsStillSettlesTheStore(t *testing.T) { + for _, tc := range []struct { + refusal *engineRefusal + reason string + reloadDown bool + }{ + {refusal: heldSeat, reason: crewletaiv1alpha1.ReasonSeatHeld}, + {refusal: managedDocument, reason: crewletaiv1alpha1.ReasonWriteRefused}, + {refusal: heldSeat, reason: crewletaiv1alpha1.ReasonSeatHeld, reloadDown: true}, + } { + name := tc.reason + if tc.reloadDown { + name += ", and the activation owed fails" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // A rotated value is the change the engine is asked to apply: + // its own re-activation at the write fails, so the operator + // writes the document. + w.rotate(t) + w.engine.mu.Lock() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.refusal, w.engine.reloadDown = tc.refusal, tc.reloadDown + w.engine.mu.Unlock() + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + if got := w.run.configApplied; got == nil || got.Reason != tc.reason { + t.Errorf("ConfigApplied = %+v, want %s", got, tc.reason) + } + if tc.reloadDown { + recorded := drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretWithdrawalFailed ") && strings.Contains(e, "tries again in 2s") + }) || result.RequeueAfter != 2*time.Second { + t.Errorf("the next look is %s on and events %q; want SecretWithdrawalFailed promising 2s, and 2s", + result.RequeueAfter, recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Errorf("the revisions are %v, want r1 activated again as r2, without the registry", got) + } + if w.engine.holds(w.registry) || len(w.state(t).Withdrawn()) != 0 { + t.Errorf("the store holds the registry: %t, and the state owes %v; want it deleted and settled by r2", + w.engine.holds(w.registry), w.state(t).Withdrawn()) + } + if result.RequeueAfter != engineRetryInterval { + t.Errorf("the next look is %s on, want %s: nothing in the cluster changes when the engine's users "+ + "clear it", result.RequeueAfter, engineRetryInterval) + } + }) + } +} + // TestARevisionAdoptedInSyncIsRecordedAtItsEpoch pins that a revision of the // rendered document the operator did not activate, a reload of its document // here, is recorded at the epoch the fleet's activation names for it, and as From be2291fc0bd48ee0748ee4ff907051d3547c26cf Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 30 Oct 2026 16:46:12 +0100 Subject: [PATCH 15/36] fix(controller): reload for a value written while no write can follow The engine reads its secret store only when a revision activates. A value the sync writes, a credential somebody rotated or a value written back over one somebody else replaced, reaches the revision the engine runs only through the write of the document that follows it. When the engine refused that write (a document it finds invalid, a human seat somebody holds, a token it does not let change the document), or the document was held for a Secret, nothing followed: the operator activated the running revision again for a withdrawal it owed and for nothing else. So a key rotated because it leaked stayed in use by the revision the engine ran for as long as the document was refused, while every node that restarted read the new one. Now reactivate owes the running revision an activation for every store change it reads once it activates again (owedBy): a withdrawn name it lets go without, as before, and a name written that it references (its document's ${...} references). settle and a reconcile whose document is held both make it. A written name the running revision does not reference stays owed for the write that does. The summary, the Reactivated events and the error of a failed request say which changes the activation reads (storeChanges), and a failed activation owed only for written values is reported as SecretRefreshFailed rather than as a withdrawal that failed; one owed for a withdrawal is reported as before. The engine activates the revision it runs again itself at a sync's write of a value that revision reads (secretsync.Result.Reload), so this is the activation where that one failed: reactivate takes the engine's where it landed, as it does at a removal, and asks for one otherwise. The tests that pin the operator's activation make the engine's fail. TestAFailedReloadIsReadBack no longer owes the values pushed before r1 (owePushed, now gone): the reload it pins reads them too, and it expects the withdrawal alone owed. TestARotatedValueReachesTheRevisionTheEngineRunsWhileTheDocumentIsHeld fails with written values left out of the activation owed, with settle returning before it looks at them, with a value the running revision does not reference activated for, and with a failed activation for one reported as a failed withdrawal; TestAHeldDocumentHasTheEngineReadAValueWrittenBeforeIt with the held reconcile returning before it looks at them, and with its failed activation reported as a failed withdrawal. The applying concept page says that a written value no write can follow is activated for, and names SecretRefreshFailed; the credentials guide says a rotation reaches the engine while it refuses the document. Signed-off-by: Ali Yousefian --- docs/concepts/applying.md | 20 +- docs/guides/credentials.md | 9 + internal/controller/config_withdraw.go | 297 ++++++++++++++------ internal/controller/config_withdraw_test.go | 187 +++++++++--- 4 files changed, 380 insertions(+), 133 deletions(-) diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index 564163fd..b95cec01 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -251,7 +251,14 @@ flowchart TD ([What the engine has read](#what-the-engine-has-read)), not just in the reconcile that wrote it: a write that fails, or loses its race to another writer, leaves the next sync nothing to write while the engine may still - run a revision that read the store before it. + run a revision that read the store before it. Where no write is coming, + because the engine refuses the document or it is held, the revision the + engine runs is activated again for a value that revision references + (`Reactivated`), so that a credential somebody rotated reaches the + running engine whatever else is wrong with the document: by the engine + itself at the write, or, where its activation failed, by the operator; a + Warning `SecretRefreshFailed` says when neither could. A value the + running revision does not reference waits for the write that does. 5. **Dry-run** the document. The engine checks everything the API server cannot: rules that span objects, such as a Seat that asks for a sandbox the Company does not offer. Each problem the engine reports is traced back, @@ -642,10 +649,13 @@ it wrote back and who had replaced it, `SecretKept` naming each value of a deleted Secret it keeps in the store for the revision the engine runs, which requires it or whose reading the operator does not know, `Reactivated` when it activated the running revision again so the engine reads its store -without the value of a deleted Secret, a Warning `Reactivated` when the -revision activated again is not the document it read, or when a request that -failed finds another document running that activated after the deletion, a -Warning `SecretWithdrawalFailed` when it could not, +without the value of a deleted Secret, or with a value it wrote while no +write of the document was coming, a Warning `Reactivated` when the revision +activated again is not the document it read, or when a request that failed +finds another document running that activated after the change, a Warning +`SecretWithdrawalFailed` when it could not have the engine read its store +without a deleted Secret's value, a Warning `SecretRefreshFailed` when it +could not have it read a value it wrote, `Drifted`, `Invalid` with the engine's problems, and `EngineWarnings` when the engine applied a revision it has something to say about. Events, conditions and logs name Secrets, keys and store entries, diff --git a/docs/guides/credentials.md b/docs/guides/credentials.md index d198b885..d3b45488 100644 --- a/docs/guides/credentials.md +++ b/docs/guides/credentials.md @@ -423,6 +423,15 @@ kubectl get company acme -n acme -o jsonpath='{.status.config.revisionID}{"\n"}' A change that leaves every referenced value as it was (a label, an annotation, another key nothing references) writes nothing. +The new value reaches the engine even while it refuses the rendered +document (`ConfigApplied=False`, say for a seat somebody holds): no revision +of the document is written then, but the revision the engine runs is +activated again, unchanged, with a `Reactivated` event naming the store +entry, as long as that revision uses the value: the engine does so itself +as the value is written, and the operator asks for it where the engine's +activation failed. A Warning `SecretRefreshFailed` says it could not, and +when it tries again. + To rotate a provider's key without a moment where the vendor refuses the one in use, mind the order: write the new key into the Secret, wait for the revision that carries it, and only then revoke the old key at the vendor. diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index db8328dd..aa4502f7 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -157,7 +157,8 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err != nil { return ctrl.Result{}, err } - if len(read.Missing()) == 0 && len(released) == 0 && len(state.Withdrawn()) == 0 && len(state.Retained()) == 0 { + if len(read.Missing()) == 0 && len(released) == 0 && len(state.Withdrawn()) == 0 && len(state.Written()) == 0 && + len(state.Retained()) == 0 { // No value to take out of the store or to say it keeps, no // activation owed, and nothing said kept that has come back. return ctrl.Result{}, nil @@ -221,8 +222,11 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err := secretsync.SaveState(ctx, run.Client, stateRef, c, next); err != nil { return ctrl.Result{}, err } - if err := errors.Join(withdrawErr, reactivateErr); err != nil { - return run.withdrawalFailed(ctx, err), nil + if withdrawErr != nil { + return run.withdrawalFailed(ctx, withdrawErr), nil + } + if reactivateErr != nil { + return run.activationFailed(ctx, reactivateErr, next), nil } return run.heldResync(ctrl.Result{}), nil } @@ -373,15 +377,15 @@ func (run *configRun) retainedSentence() string { // sync may have withdrawn a name the revision just applied started with. // active is the engine's document as the reconcile last read it, before the // sync, nil to read it; reload is the engine's own re-activation at the -// sync's removals (secretsync.Result.Reload), which reactivate takes for the -// activation where it landed. Its error is the API server's; an activation -// that failed is reported (withdrawalFailed), and retry is when that report -// says the operator tries again, zero where nothing failed. +// sync's last write or removal (secretsync.Result.Reload), which reactivate +// takes for the activation where it landed. Its error is the API server's; an +// activation that failed is reported (activationFailed), and retry is when +// that report says the operator tries again, zero where nothing failed. func (run *configRun) settle( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, ) (retry time.Duration, err error) { - if len(state.Withdrawn()) == 0 { + if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { return 0, nil } next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state, ref) @@ -389,7 +393,7 @@ func (run *configRun) settle( return 0, err } if reactivateErr != nil { - return run.withdrawalFailed(ctx, reactivateErr).RequeueAfter, nil + return run.activationFailed(ctx, reactivateErr, next).RequeueAfter, nil } return 0, nil } @@ -418,40 +422,138 @@ func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Resu return ctrl.Result{RequeueAfter: retry} } +// activationFailed reports err, which kept the engine from reading the store +// changes state still owes an activation for: a withdrawal where it owes one, +// as withdrawalFailed reports it; otherwise only values written, which the +// revision the engine runs read before they were written, so it may go on +// starting tool servers and executor runs with the values the Secrets held +// before, a credential somebody rotated included (SecretRefreshFailed). It +// returns when to try again, as engineRetry says for err. +func (run *configRun) activationFailed(ctx context.Context, err error, state secretsync.State) ctrl.Result { + if len(state.Withdrawn()) > 0 || len(state.Written()) == 0 { + return run.withdrawalFailed(ctx, err) + } + retry := run.engineRetry(err) + logf.FromContext(ctx).Error(err, "Failed to have the engine read the values the operator wrote into its secret store") + run.event(corev1.EventTypeWarning, "SecretRefreshFailed", "ReloadConfig", "%s", ended(fmt.Sprintf( + "The engine may still start a tool server or an executor run with a value a Secret held before the operator "+ + "wrote its current one into the engine's secret store (%s), which the revision it runs read before; the "+ + "operator tries again in %s: %s", strings.Join(state.Written(), ", "), retry, err))) + return ctrl.Result{RequeueAfter: retry} +} + +// storeChanges are the changes of the engine's secret store that the revision +// the engine runs has not read and reads once it activates again: names +// withdrawn that it lets go without, and names written that it references. +// A withdrawn name it references as required, or one its reading the operator +// does not know, it keeps until the next revision runs, and a written name it +// does not reference changes nothing it runs. +type storeChanges struct { + withdrawn, written []string +} + +// owedBy is what an activation of active again would read of the store +// changes state records. +func owedBy(state secretsync.State, active *engine.ActiveConfig) storeChanges { + letGo := state.LetGo(active.RevisionID) + references := storeNamesIn(active.Document) + return storeChanges{ + withdrawn: slices.DeleteFunc(state.Withdrawn(), func(name string) bool { return !slices.Contains(letGo, name) }), + written: slices.DeleteFunc(state.Written(), func(name string) bool { return !slices.Contains(references, name) }), + } +} + +func (c storeChanges) empty() bool { return len(c.withdrawn) == 0 && len(c.written) == 0 } + +// owedIn reports whether state still owes an activation for any of c. +func (c storeChanges) owedIn(state secretsync.State) bool { + withdrawn, written := state.Withdrawn(), state.Written() + return slices.ContainsFunc(c.withdrawn, func(name string) bool { return slices.Contains(withdrawn, name) }) || + slices.ContainsFunc(c.written, func(name string) bool { return slices.Contains(written, name) }) +} + +// reads is how the engine reads its store once a revision activates after c: +// without what was withdrawn, with what was written. +func (c storeChanges) reads() string { + var parts []string + if len(c.withdrawn) > 0 { + parts = append(parts, "without "+strings.Join(c.withdrawn, ", ")) + } + if len(c.written) > 0 { + parts = append(parts, "with the values now written under "+strings.Join(c.written, ", ")) + } + return strings.Join(parts, " and ") +} + +// why is reads with the reason for each change, which revision, the one +// activated again, lets go without or references. +func (c storeChanges) why(revision string) string { + var parts []string + if len(c.withdrawn) > 0 { + parts = append(parts, fmt.Sprintf("without %s, whose Secret or key no longer exists and which %s lets go "+ + "without", strings.Join(c.withdrawn, ", "), revision)) + } + if len(c.written) > 0 { + parts = append(parts, fmt.Sprintf("with the values now written under %s, which %s references and read "+ + "before they were written", strings.Join(c.written, ", "), revision)) + } + return strings.Join(parts, ", and ") +} + +// made is what the operator did to the store. +func (c storeChanges) made() string { + switch { + case len(c.written) == 0: + return fmt.Sprintf("removed %s from the engine's secret store", strings.Join(c.withdrawn, ", ")) + case len(c.withdrawn) == 0: + return fmt.Sprintf("wrote %s into the engine's secret store", strings.Join(c.written, ", ")) + } + return fmt.Sprintf("removed %s from and wrote %s into the engine's secret store", strings.Join(c.withdrawn, ", "), + strings.Join(c.written, ", ")) +} + // reactivate activates the engine's running revision again when state records -// withdrawn names it lets go without (State.LetGo), and returns state with -// every withdrawal settled: the engine has read its store without them, runs -// nothing that references them, or runs a revision that reads them as -// required, or one whose reading state does not record, which keeps what it -// read until the next one runs. active is the engine's document as this -// reconcile read it before it removed anything, nil to read it here. +// a change of the store it reads once it does (owedBy): a withdrawn name it +// lets go without (State.LetGo), or a value written under a name it +// references, which it read before the write: a rotated credential, or a +// value the sync wrote back. It returns state with every withdrawal settled: +// the engine has read its store without them, runs nothing that references +// them, or runs a revision that reads them as required, or one whose reading +// state does not record, which keeps what it read until the next one runs. A +// value written under a name the running revision does not reference stays +// owed, for the next write of the document to read. active is the engine's +// document as this reconcile read it, nil to read it here. +// +// It is the activation for every store change no write of the document is +// coming to settle: a document the engine refused, or one held for a Secret. +// A value rotated while the document was refused went unread by the revision +// the engine runs for as long as it was refused, while every node that +// restarted read the new one. // // The engine activates its revision again itself when a write or a removal // changes a value that revision reads (Crewlet's ADR-0070), and reload is // what the last such request answered (secretsync.Result.Reload), made by // the engine call reloadAction (secretsync.Result.ReloadAction): where it -// landed, the engine read its store without every name removed, and it is -// the activation; asking for another would activate the same document twice. +// landed, it came after every change the sync made, and it is the +// activation; asking for another would activate the same document twice. // Where it is nil or failed, the operator asks for one (POST /config/reload). // // Neither names a revision: the engine stores again whichever revision the // fleet's activation pointer names when the request lands, built on it, and // a write landing inside the request fails it as a lost race. So the -// revision activated again is read back, and recorded as letting go what the -// one read lets go only where it is that document; otherwise the state goes -// on recording the one read, which is no longer the fleet's, and a Warning -// says so. -// Before the operator's request the withdrawals are marked by the fleet's -// activation epoch, and state saved in ref (markChanges): a request that -// fails, or loses its race to another writer, is read back, and a revision -// the fleet runs above that epoch came after every withdrawal, whoever -// activated it, and settles them (reloadFailed). +// revision it activates again is read back, and recorded as letting go what +// the one read lets go only where it is that document; otherwise the state +// goes on recording the one read, which is no longer the fleet's, and a +// Warning says so. Before the operator's request the changes are marked by +// the fleet's activation epoch, and state saved in ref (markChanges): a +// request that fails, or loses its race to another writer, is read back, and a revision +// the fleet runs above that epoch came after every change, whoever activated +// it, and settles them (reloadFailed). func (run *configRun) reactivate( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, ) (secretsync.State, error) { - withdrawn := state.Withdrawn() - if len(withdrawn) == 0 { + if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { return state, nil } if active == nil { @@ -463,9 +565,8 @@ func (run *configRun) reactivate( if active == nil { return state.Reactivated(), nil } - letGo := state.LetGo(active.RevisionID) - held := slices.DeleteFunc(withdrawn, func(name string) bool { return !slices.Contains(letGo, name) }) - if len(held) == 0 { + owed := owedBy(state, active) + if owed.empty() { return state.Reactivated(), nil } // WHO ACTIVATED IT: the engine, at the request of the sync that made @@ -482,23 +583,23 @@ func (run *configRun) reactivate( if state, err = run.markChanges(ctx, eng, state, ref); err != nil { return state, err } - if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(held)); err != nil { - return run.reloadFailed(ctx, eng, active, state, held, - fmt.Errorf("activating revision %s again, so that the engine reads its secret store without %s: %w", - active.RevisionID, strings.Join(held, ", "), err)) + if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(owed)); err != nil { + return run.reloadFailed(ctx, eng, active, state, owed, + fmt.Errorf("activating revision %s again, so that the engine reads its secret store %s: %w", + active.RevisionID, owed.reads(), err)) } } now, err := activeConfig(ctx, eng) switch { case err != nil: - run.reactivatedUnconfirmed(active, reloaded, by, held, "the operator could not read it back: "+err.Error()) + run.reactivatedUnconfirmed(active, reloaded, by, owed, "the operator could not read it back: "+err.Error()) case now == nil || now.RevisionID != reloaded.RevisionID: - run.reactivatedUnconfirmed(active, reloaded, by, held, "the engine runs another revision since") + run.reactivatedUnconfirmed(active, reloaded, by, owed, "the engine runs another revision since") case !bytes.Equal(now.Document, active.Document): - run.reactivatedUnconfirmed(active, reloaded, by, held, + run.reactivatedUnconfirmed(active, reloaded, by, owed, "its document is not that one: somebody activated another revision before the request landed") default: - run.recordReactivation(active, reloaded, by, held, nil) + run.recordReactivation(active, reloaded, by, owed, nil) return state.Activated(reloaded.RevisionID, state.LetGoKeys(active.RevisionID)), nil } // Whatever its document, the revision the request activated read the @@ -507,22 +608,22 @@ func (run *configRun) reactivate( } // reloadFailed is state once a request to activate active again, so that the -// engine reads its store without held, failed with failed: settled as far as -// the fleet's revision, read afresh, activated after the withdrawals, since +// engine reads the store changes owed, failed with failed: settled as far as +// the fleet's revision, read afresh, activated after the changes, since // nothing read before the request saw what activated since, and failed where -// any of held is still owed. A revision the fleet's activation names above -// the epoch the withdrawals were marked by (markChanges) came after every one -// of them, whoever activated it: the request itself, where it landed and its +// any of owed is still owed. A revision the fleet's activation names above +// the epoch the changes were marked by (markChanges) came after every one of +// them, whoever activated it: the request itself, where it landed and its // answer did not, or another writer, who won the race with it. Where its -// document is active's, the fleet runs that document again without held, as -// a reactivation the operator confirmed does, and it is recorded as one -// (recordReactivation, whose event says the request failed); otherwise it is -// a revision whose reading the operator does not know, which a Warning says, -// and the state goes on recording the one read, as reactivatedUnconfirmed -// leaves it. A request that failed outright leaves the fleet on active, which -// settles nothing. +// document is active's, the fleet runs that document again reading the store +// as owed says, as a reactivation the operator confirmed does, and it is +// recorded as one (recordReactivation, whose event says the request failed); +// otherwise it is a revision whose reading the operator does not know, which +// a Warning says, and the state goes on recording the one read, as +// reactivatedUnconfirmed leaves it. A request that failed outright leaves the +// fleet on active, which settles nothing. func (run *configRun) reloadFailed( - ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, held []string, + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, owed storeChanges, failed error, ) (secretsync.State, error) { now, err := activeConfig(ctx, eng) @@ -534,58 +635,67 @@ func (run *configRun) reloadFailed( return state, failed } settled := state.Ran(fleet.Activation.Epoch) - if owed := settled.Withdrawn(); slices.ContainsFunc(held, func(name string) bool { return slices.Contains(owed, name) }) { + if owed.owedIn(settled) { return state, failed } activation := engine.ApplyResult{RevisionID: now.RevisionID, Epoch: fleet.Activation.Epoch} if bytes.Equal(now.Document, active.Document) { - run.recordReactivation(active, activation, "", held, failed) + run.recordReactivation(active, activation, "", owed, failed) return settled.Adopted(now.RevisionID, state.LetGoKeys(active.RevisionID)), nil } run.event(corev1.EventTypeWarning, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( "Did not activate revision %s again: revision %s, another document, activated as epoch %d after the operator "+ - "removed %s from the engine's secret store, and read the store without them; the operator takes it to read "+ - "every value as required, as it does any revision it has no record of activating, and the request failed: %s", - active.RevisionID, now.RevisionID, fleet.Activation.Epoch, strings.Join(held, ", "), failed))) + "%s, and read the store %s; the operator takes it to read every value as required, as it does any revision "+ + "it has no record of activating, and the request failed: %s", + active.RevisionID, now.RevisionID, fleet.Activation.Epoch, owed.made(), owed.reads(), failed))) return settled, nil } // reactivationSummary is the summary of the revision that activates the -// running document again without held. -func reactivationSummary(held []string) string { +// running document again to read the store changes owed. +func reactivationSummary(owed storeChanges) string { const shown = 4 - text := strings.Join(held[:min(len(held), shown)], ", ") - if len(held) > shown { - text += fmt.Sprintf(" and %d more", len(held)-shown) + list := func(names []string) string { + text := strings.Join(names[:min(len(names), shown)], ", ") + if len(names) > shown { + text += fmt.Sprintf(" and %d more", len(names)-shown) + } + return text + } + var parts []string + if len(owed.withdrawn) > 0 { + parts = append(parts, "without "+list(owed.withdrawn)+", whose Secret or key no longer exists") } - return "crewlet-operator: activated again without " + text + ", whose Secret or key no longer exists" + if len(owed.written) > 0 { + parts = append(parts, "to read the values now written under "+list(owed.written)) + } + return "crewlet-operator: activated again " + strings.Join(parts, ", and ") } // recordReactivation records the running document activated again as -// reloaded, without held, by the engine at the request of the sync's engine -// call by (WriteSecret, PutSecrets or DeleteSecret: a write or a removal of -// a value the revision reads), or at the operator's request, by empty. A -// revision the operator recorded as its own is replaced by the one that -// activates it again, with the same content, so that the next one it applies -// does not take it for an edit somebody else made; one it did not write stays -// somebody else's. failed is the error the operator's request answered where -// the engine was read back running the document again anyway (reloadFailed), -// nil where it answered the activation. +// reloaded, reading the store changes owed, by the engine at the request of +// the sync's engine call by (WriteSecret, PutSecrets or DeleteSecret: a write +// or a removal of a value the revision reads), or at the operator's request, +// by empty. A revision the operator recorded +// as its own is replaced by the one that activates it again, with the same +// content, so that the next one it applies does not take it for an edit +// somebody else made; one it did not write stays somebody else's. failed is +// the error the operator's request answered where the engine was read back +// running the document again anyway (reloadFailed), nil where it answered the +// activation. func (run *configRun) recordReactivation( - active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string, failed error, + active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, owed storeChanges, failed error, ) { if failed == nil { who, action := reactivatedBy(by) run.event(corev1.EventTypeNormal, "Reactivated", action, "%s revision %s again, unchanged, as revision %s (epoch %d), so that the engine reads its secret store "+ - "without %s: the Secret or key each stands for no longer exists, and the revision lets it go without its "+ - "value.", who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", ")) + "%s.", who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, owed.why("the revision")) } else { run.event(corev1.EventTypeNormal, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( - "The engine runs revision %s again, unchanged, as revision %s (epoch %d), activated after the operator removed "+ - "%s from its secret store, whose Secret or key no longer exists and which the revision lets go without its "+ - "value, so it reads the store without them; the operator's request to activate it again failed: %s", - active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", "), failed))) + "The engine runs revision %s again, unchanged, as revision %s (epoch %d), activated after the operator %s, "+ + "so it reads the store %s; the operator's request to activate it again failed: %s", + active.RevisionID, reloaded.RevisionID, reloaded.Epoch, owed.made(), owed.why("the revision"), failed))) } recorded := run.company.Status.Config if run.config != nil { @@ -603,27 +713,28 @@ func (run *configRun) recordReactivation( } } -// reactivatedUnconfirmed reports an activation of active again, without -// held, by the engine at the sync's engine call by or at the operator's -// request (by empty), -// that activated reloaded, which the operator could not confirm is active's -// document, for why: it takes reloaded to read every name as required, and a -// revision somebody else activated may have needed a value it removed for -// active, which let it go. +// reactivatedUnconfirmed reports an activation of active again, reading the +// store changes owed, by the engine at the sync's engine call by or at the +// operator's request (by empty), that activated reloaded, which the operator +// could not confirm is active's document, for why: it takes reloaded to read every +// name as required, and, where owed withdrew a name, a revision somebody else +// activated may have needed a value it removed for active, which let it go. func (run *configRun) reactivatedUnconfirmed( - active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string, why string, + active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, owed storeChanges, why string, ) { who, action := reactivatedBy(by) // why ends with what the read back met, which may be the engine's own // sentence: the sentence that quotes it is ended by ended. activated := ended(fmt.Sprintf("%s the engine's document again as revision %s (epoch %d), so that the engine "+ - "reads its secret store without %s, whose Secret or key no longer exists and which revision %s lets go without; "+ - "but the operator cannot confirm that revision %s is revision %s's document, as %s", - who, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", "), active.RevisionID, reloaded.RevisionID, + "reads its secret store %s; but the operator cannot confirm that revision %s is revision %s's document, as %s", + who, reloaded.RevisionID, reloaded.Epoch, owed.why("revision "+active.RevisionID), reloaded.RevisionID, active.RevisionID, why)) - run.event(corev1.EventTypeWarning, "Reactivated", action, - "%s It takes revision %s to read every value as required, and one that reads a value it removed so goes without "+ - "what it starts with it until the Secret exists again.", activated, reloaded.RevisionID) + takes := fmt.Sprintf("It takes revision %s to read every value as required", reloaded.RevisionID) + if len(owed.withdrawn) > 0 { + takes += ", and one that reads a value it removed so goes without what it starts with it until the Secret " + + "exists again" + } + run.event(corev1.EventTypeWarning, "Reactivated", action, "%s %s.", activated, takes) } // reactivatedBy is who an activation again is said to be by, and the action diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 070973a5..ce809b39 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -1,6 +1,7 @@ package controller import ( + "bytes" "context" "encoding/json" "errors" @@ -607,11 +608,6 @@ type withdrawal struct { registry string // requiring is a document whose server requires the registry. requiring []byte - // pushed is what the operator pushed for r1, both values, and owe makes - // the state record them pushed after it (owePushed). - pushed secretsync.Desired - recorded string - optional []secretref.Key // stateWrites counts the writes of the secret sync's state since the // reconcile began, and failWrite, when it is not 0, is the one that // fails, the API server not answering. @@ -684,13 +680,14 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { t.Fatal(err) } // Pushed before r1 activated, so that r1 read them: nothing is - // re-activated for them, and nothing is owed for them after it. + // re-activated for them. fe.reloadAnswer = new(map[string]any(nil)) state, _, err := secretsync.Sync(ctx, eng, present, secretsync.State{}, operatorLogin, discardState) if err != nil { t.Fatal(err) } fe.reloadAnswer = nil + // r1 activated after both values were pushed, so it read them. state = state.Reloaded() if r.recorded != "" { state = state.Activated(r.recorded, runningDoc.Optional) @@ -705,7 +702,6 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { w.stateWrites.Store(0) w.engine, w.client, w.cluster = fe, eng, cluster - w.pushed, w.recorded, w.optional = present, r.recorded, runningDoc.Optional w.registry = secretsync.StoreName(secretref.Key{Namespace: "ns", Name: optionalSecret, Key: optionalKey}) w.again(t, company) if !fe.holds(w.registry) { @@ -714,30 +710,6 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { return w } -// owePushed makes the state the one a sync leaves that pushed both values -// again after r1 activated, whose writes the engine re-activated nothing at: -// the revision recorded as before, and both values owed until a revision -// activates after them. -func (w *withdrawal) owePushed(t *testing.T) { - t.Helper() - ctx := context.Background() - w.engine.reloadAnswer = new(map[string]any(nil)) - state, _, err := secretsync.Sync(ctx, w.client, w.pushed, secretsync.State{}, operatorLogin, discardState) - if err != nil { - t.Fatal(err) - } - w.engine.reloadAnswer = nil - if w.recorded != "" { - state = state.Adopted(w.recorded, w.optional) - } - if len(state.Written()) == 0 { - t.Fatal("the sync pushed nothing again") - } - if err := secretsync.SaveState(ctx, w.cluster, w.ref(), w.run.company, state); err != nil { - t.Fatal(err) - } -} - // again makes w.run a new reconcile of company, as a reconcile after the last // one sees it, rendering optionalRun's document. func (w *withdrawal) again(t *testing.T, company *crewletaiv1alpha1.Company) { @@ -1544,6 +1516,152 @@ func TestADocumentTheEngineHoldsStillSettlesTheStore(t *testing.T) { } } +// TestARotatedValueReachesTheRevisionTheEngineRunsWhileTheDocumentIsHeld pins +// that a value the sync writes into the store while the engine refuses the +// document, a credential somebody rotated, is read by the revision the engine +// runs: the operator activates that revision again for it, as it does for the +// value of a deleted Secret, since no write of the document is coming to do +// it. Only a withdrawal got that, so the running revision kept the old value, +// a revoked key say, for as long as the document was refused, while every +// node that restarted read the new one. A revision that does not reference the +// value is not activated again for it, and an activation that fails says so +// and comes back when it promises. +func TestARotatedValueReachesTheRevisionTheEngineRunsWhileTheDocumentIsHeld(t *testing.T) { + for _, tc := range []struct { + name string + refusal *engineRefusal + unreferenced, fails bool + }{ + {name: "the engine finds the document invalid", refusal: invalidDocument}, + {name: "a seat somebody holds", refusal: heldSeat}, + {name: "the running revision does not reference the value", refusal: invalidDocument, unreferenced: true}, + {name: "the activation fails", refusal: invalidDocument, fails: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // Nothing is withdrawn: the registry's Secret is back, and the + // engine runs a revision that read every value. + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.settled(t) + if owed := w.state(t).Written(); len(owed) != 0 { + t.Fatalf("the state owes %v before the rotation, want nothing", owed) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.rotate(t) + w.engine.mu.Lock() + // The engine's own re-activation at the write fails: the + // activation is the operator's. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.refusal, w.engine.reloadDown = tc.refusal, tc.fails + if tc.unreferenced { + // Somebody activated a document that names no such value. + doc := w.engine.revisions[len(w.engine.revisions)-1].document + w.engine.activateOther(bytes.ReplaceAll(doc, []byte("${"+tavily+"}"), []byte("tvly-literal"))) + } + w.engine.mu.Unlock() + before := w.engine.ids() + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + w.engine.mu.Lock() + stored := w.engine.store[tavily] + w.engine.mu.Unlock() + if stored != rotatedValue { + t.Fatalf("the store holds %q under %s, want the rotated value", stored, tavily) + } + recorded := drained(w.events) + ids, written := w.engine.ids(), w.state(t).Written() + switch { + case tc.unreferenced: + if !slices.Equal(ids, before) || !slices.Equal(written, []string{tavily}) { + t.Errorf("the revisions are %v and the state owes %v; want %v, nothing activated again for a value "+ + "the running revision does not name, and the value owed to the next write", ids, written, before) + } + case tc.fails: + if !slices.Equal(written, []string{tavily}) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretRefreshFailed ") && strings.Contains(e, "tries again in 2s") + }) || result.RequeueAfter != 2*time.Second { + t.Errorf("the state owes %v, the next look is %s on and events %q; want the value owed, "+ + "SecretRefreshFailed promising 2s, and 2s", written, result.RequeueAfter, recorded) + } + default: + if want := append(slices.Clone(before), fmt.Sprintf("r%d", len(before)+1)); !slices.Equal(ids, want) || + len(written) != 0 { + t.Errorf("the revisions are %v and the state owes %v; want %v, the running revision activated "+ + "again for the rotated value, and nothing owed", ids, written, want) + } + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Reactivated ") && strings.Contains(e, tavily) + }) { + t.Errorf("events %q, want Reactivated naming %s", recorded, tavily) + } + } + }) + } +} + +// TestAHeldDocumentHasTheEngineReadAValueWrittenBeforeIt pins that a reconcile +// whose document is held makes the activation a value written earlier is +// owed, where the revision the engine runs references it: a reconcile wrote a +// rotated value and went no further, and the next one finds the document +// held. It returned at once when nothing was withdrawn, missing or kept, so +// the running revision went on with the old value for as long as the +// document was held. An activation that fails is reported as one owed for a +// value written, not as a withdrawal that failed. +func TestAHeldDocumentHasTheEngineReadAValueWrittenBeforeIt(t *testing.T) { + for _, fails := range []bool{false, true} { + t.Run(map[bool]string{false: "the activation lands", true: "the activation fails"}[fails], func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.rotate(t) + // A reconcile wrote the rotated value, whose re-activation by the + // engine failed, and went no further. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + owed, _, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, w.ref(), w.run.company, owed); err != nil { + t.Fatal(err) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + if !slices.Equal(owed.Written(), []string{tavily}) { + t.Fatalf("the state owes %v, want the rotated value", owed.Written()) + } + w.engine.mu.Lock() + w.engine.reloadDown = fails + w.engine.mu.Unlock() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + state := w.state(t) + if fails { + if !slices.Equal(state.Written(), []string{tavily}) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretRefreshFailed ") && strings.Contains(e, "tries again in 2s") + }) || warned(recorded, "SecretWithdrawalFailed") || result.RequeueAfter != 2*time.Second { + t.Errorf("the state owes %v, the next look is %s on and events %q; want the value owed, "+ + "SecretRefreshFailed promising 2s and no failed withdrawal, and 2s", + state.Written(), result.RequeueAfter, recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) || len(state.Written()) != 0 { + t.Errorf("the revisions are %v and the state owes %v; want r1 activated again for the rotated value, "+ + "and nothing owed", got, state.Written()) + } + }) + } +} + // TestARevisionAdoptedInSyncIsRecordedAtItsEpoch pins that a revision of the // rendered document the operator did not activate, a reload of its document // here, is recorded at the epoch the fleet's activation names for it, and as @@ -1909,7 +2027,6 @@ func TestAFailedReloadIsReadBack(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() w := newWithdrawal(t, runningLetsGo) - w.owePushed(t) w.engine.mu.Lock() w.engine.reloadDown = true // The engine's own re-activation at the removal fails: the operator asks. @@ -1929,10 +2046,10 @@ func TestAFailedReloadIsReadBack(t *testing.T) { recorded := drained(w.events) state := w.state(t) if tc.other == nothing { - if !slices.Equal(state.Withdrawn(), []string{w.registry}) || len(state.Written()) == 0 || + if !slices.Equal(state.Withdrawn(), []string{w.registry}) || len(state.Written()) != 0 || !warned(recorded, "SecretWithdrawalFailed") { - t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want the registry and the "+ - "values pushed before it owed, and the failure reported", state.Withdrawn(), state.Written(), recorded) + t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want the registry owed, r1 having "+ + "read every value, and the failure reported", state.Withdrawn(), state.Written(), recorded) } return } From 6e48c8db7595f137f60bf0126ea45d04007efd1b Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 18:18:19 +0000 Subject: [PATCH 16/36] test(controller): build the samples, scheme and REST mapper once The controller package's tests that drive a reconcile against an engine double spend most of their time building their fixtures rather than running the code they test: each parsed every file in config/samples again (decodeSamples, through optionalRun, called twice per withdrawal), built a runtime scheme of client-go's types and the crewlet.ai types, and gave the fake client no REST mapper, so it derived a static one from that scheme each time. Profiled, those three were half of the CPU of the package's tests outside the envtest suite. They run one after another beside the envtest suite in one test binary, which runs on go test's default ten-minute timeout, and on a 4-CPU machine the package already takes eight of those minutes. The samples are parsed once per test binary (samples, a sync.OnceValues) and each call of decodeSamples deep-copies them into its namespace, so no test shares an object with another; the scheme and its REST mapper are built once (testScheme, testRESTMapper), being read-only once built, and every fake client in the package takes them through fakeClient. Those tests took 23.3 s and take 12.8 s, measured with the same load on the machine. Signed-off-by: Ali Yousefian --- internal/controller/config_fleet_test.go | 18 +------ internal/controller/config_optional_test.go | 3 +- internal/controller/config_placement_test.go | 8 +-- internal/controller/config_unit_test.go | 53 ++++++++++++++++++-- internal/controller/config_withdraw_test.go | 11 +--- 5 files changed, 54 insertions(+), 39 deletions(-) diff --git a/internal/controller/config_fleet_test.go b/internal/controller/config_fleet_test.go index 0a25677e..9c727faf 100644 --- a/internal/controller/config_fleet_test.go +++ b/internal/controller/config_fleet_test.go @@ -12,11 +12,8 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/engine" @@ -215,19 +212,12 @@ func TestSeatReadyFollowsTheFleet(t *testing.T) { // reconcile that would not have looked again. func TestPlacementResyncPollsOnlyWhileSeatsSettle(t *testing.T) { const workload = fleetWorkload - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } statefulSet := func(ready int32) *appsv1.StatefulSet { return fleetStatefulSet(workload, ready) } start := time.Date(2026, 10, 9, 8, 0, 0, 0, time.UTC) reconciler := &CompanyConfigReconciler{ResyncInterval: time.Minute} look := func(at time.Time, ready int32, fleet *engine.Fleet, result ctrl.Result) time.Duration { t.Helper() - reconciler.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(statefulSet(ready)).Build() + reconciler.Client = fakeClient().WithObjects(statefulSet(ready)).Build() reconciler.now = func() time.Time { return at } run, _ := seatRun("") run.CompanyConfigReconciler = reconciler @@ -305,10 +295,6 @@ func TestSeatsPastAPodsShareAreSettling(t *testing.T) { workload = fleetWorkload seatBo = "bo" ) - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } nodes := func(n int) []engine.FleetNode { var out []engine.FleetNode for i := range n { @@ -340,7 +326,7 @@ func TestSeatsPastAPodsShareAreSettling(t *testing.T) { bo.Name, bo.Status.Handle = seatBo, seatBo run.parts = append(run.parts, newPart(bo)) bo.Status.Placement = ana.obj.(*crewletaiv1alpha1.Seat).Status.Placement - run.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(fleetStatefulSet(workload, c.ready)).Build() + run.Client = fakeClient().WithObjects(fleetStatefulSet(workload, c.ready)).Build() run.fleet = c.fleet got, err := run.settlingSeats(context.Background()) if err != nil { diff --git a/internal/controller/config_optional_test.go b/internal/controller/config_optional_test.go index e71f3871..e8655019 100644 --- a/internal/controller/config_optional_test.go +++ b/internal/controller/config_optional_test.go @@ -11,7 +11,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/render" @@ -74,7 +73,7 @@ func optionalRun(t *testing.T, secrets ...*corev1.Secret) (*configRun, *part, *e run := &configRun{ CompanyConfigReconciler: &CompanyConfigReconciler{ Recorder: recorder, - APIReader: fake.NewClientBuilder().WithObjects(objectsOf(secrets)...).Build(), + APIReader: fakeClient().WithObjects(objectsOf(secrets)...).Build(), }, company: company, class: class, parts: []*part{p, newPart(seat)}, } diff --git a/internal/controller/config_placement_test.go b/internal/controller/config_placement_test.go index 8abb2a64..9abcd1b3 100644 --- a/internal/controller/config_placement_test.go +++ b/internal/controller/config_placement_test.go @@ -6,9 +6,7 @@ import ( "testing" "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/engine" @@ -49,13 +47,9 @@ func TestPlacementRecordKeepsThePreviousPinsUntilApplied(t *testing.T) { // on a client that serves its status subresource. func placementCompany(t *testing.T, recorded crewletaiv1alpha1.RevisionPlacement) (client.Client, *crewletaiv1alpha1.Company) { t.Helper() - scheme := runtime.NewScheme() - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } company := &crewletaiv1alpha1.Company{Namespace: "ns", Name: fleetCompany} company.Status.Config = crewletaiv1alpha1.ConfigStatus{RevisionID: "r1", Placement: recorded} - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(company).WithStatusSubresource(company).Build() + c := fakeClient().WithObjects(company).WithStatusSubresource(company).Build() got := &crewletaiv1alpha1.Company{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(company), got); err != nil { t.Fatal(err) diff --git a/internal/controller/config_unit_test.go b/internal/controller/config_unit_test.go index 919c26cd..5fe7e450 100644 --- a/internal/controller/config_unit_test.go +++ b/internal/controller/config_unit_test.go @@ -7,12 +7,19 @@ import ( "regexp" "slices" "strings" + "sync" "testing" "time" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/meta/testrestmapper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/yaml" @@ -24,8 +31,28 @@ import ( ) // decodeSamples decodes every object in config/samples, the Nimbus company, -// its parts and its class, the namespaced ones into the namespace given. +// its parts and its class, the namespaced ones into the namespace given. Each +// call has objects of its own, copied from one parse per test binary: the +// samples are read by dozens of tests, and parsing them again for each was +// most of what those tests cost. func decodeSamples(namespace string) ([]client.Object, error) { + parsed, err := samples() + if err != nil { + return nil, err + } + objects := make([]client.Object, 0, len(parsed)) + for _, obj := range parsed { + obj = obj.DeepCopyObject().(client.Object) + if _, cluster := obj.(*crewletaiv1alpha1.EngineClass); !cluster { + obj.SetNamespace(namespace) + } + objects = append(objects, obj) + } + return objects, nil +} + +// samples is every object in config/samples, decoded once, in no namespace. +var samples = sync.OnceValues(func() ([]client.Object, error) { files, err := filepath.Glob(filepath.Join("..", "..", "config", "samples", "v1alpha1_*.yaml")) if err != nil { return nil, err @@ -72,13 +99,31 @@ func decodeSamples(namespace string) ([]client.Object, error) { if err := yaml.UnmarshalStrict([]byte(manifest), obj); err != nil { return nil, fmt.Errorf("%s: %w", file, err) } - if typeMeta.Kind != render.KindEngineClass { - obj.SetNamespace(namespace) - } objects = append(objects, obj) } } return objects, nil +}) + +// testScheme holds the API types the fake clients of these tests serve, built +// once per test binary: a scheme is read-only once built, and building one, +// with the REST mapper a fake client otherwise derives from it, cost each +// test that made a client more than the test itself. +var testScheme = sync.OnceValue(func() *runtime.Scheme { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(crewletaiv1alpha1.AddToScheme(scheme)) + return scheme +}) + +// testRESTMapper maps every kind testScheme holds, built once with it. +var testRESTMapper = sync.OnceValue(func() meta.RESTMapper { + return testrestmapper.TestOnlyStaticRESTMapper(testScheme()) +}) + +// fakeClient is a fake client builder over testScheme and testRESTMapper. +func fakeClient() *fake.ClientBuilder { + return fake.NewClientBuilder().WithScheme(testScheme()).WithRESTMapper(testRESTMapper()) } // inputOf sorts objects into the input of a render. diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index ce809b39..6dae8a20 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -23,11 +23,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/yaml" @@ -649,13 +647,6 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { } fe, eng := newStoreEngine(t, served) - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } company := run1.company.DeepCopy() company.Spec.Mission = "" company.UID = "6f1c1a5e-0000-4000-8000-000000000002" @@ -664,7 +655,7 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { Type: crewletaiv1alpha1.ConditionRuntimeReady, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonValid, }) cluster := interceptor.NewClient( - fake.NewClientBuilder().WithScheme(scheme).WithObjects(company, tavily, npm).WithStatusSubresource(company).Build(), + fakeClient().WithObjects(company, tavily, npm).WithStatusSubresource(company).Build(), interceptor.Funcs{Apply: func(ctx context.Context, c client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { if n := w.stateWrites.Add(1); n == w.failWrite.Load() { return errStateWrite From 96d0593cc265aee9a7956648182acda48156fdca Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sun, 25 Oct 2026 18:18:43 +0000 Subject: [PATCH 17/36] build: give make test a timeout the controller suite fits in make test ran every package on go test's default timeout of ten minutes per test binary. internal/controller is one binary holding the envtest suite, which runs every config spec against a real engine, and the package took 478 s on a 4-CPU machine under the race detector, 80 per cent of that window. With more engine-double tests in the package it took 596 s, and a run beside another job crossed the window and failed with "test timed out after 10m0s" while every test was passing; a slower runner fails the same way. The e2e target already names its own (60m). The test target now passes -timeout 20m: twice the suite's measured time, so a runner half as fast still finishes, while a hung suite is still stopped well inside a CI job's limit. The comment beside the target says what the value is sized to. Signed-off-by: Ali Yousefian --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0d627fbd..cc225458 100644 --- a/Makefile +++ b/Makefile @@ -85,12 +85,16 @@ gateway-api-crds: ## Copy the Gateway API CRDs of the release go.mod requires in # test/docs) runs the crewlet binary `make crewlet` installed # (internal/engine/enginetest), named explicitly so a CREWLET_BIN in the # caller's environment cannot swap in another engine; test/chart renders the -# chart with the pinned helm. +# chart with the pinned helm. The timeout is per test binary and sized to the +# longest: internal/controller, whose envtest suite runs every config spec +# against a real engine and takes about eight minutes on a 4-CPU runner under +# the race detector, which go test's default of ten would fail on any slower +# one. Twenty still stops a hung suite. .PHONY: test test: manifests generate fmt vet setup-envtest crewlet helm ## Run the unit, envtest, engine and chart suites of both modules. KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" \ CREWLET_BIN="$(CREWLET)" HELM="$(HELM)" \ - go test -race $$(go list ./... | grep -v /test/e2e) -coverprofile cover.out + go test -race -timeout 20m $$(go list ./... | grep -v /test/e2e) -coverprofile cover.out cd api && go test -race ./... -coverprofile cover.out KIND_CLUSTER ?= crewlet-operator-test-e2e From 16a9e994f0605ce934e9606c44858859359d977d Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 30 Oct 2026 16:51:30 +0100 Subject: [PATCH 18/36] fix(controller): never write twice over an activation left unconfirmed When the engine stores a written revision and then loses the answer to the compare-and-set that activates it, it answers PUT /config with 503 unavailable, "outcome": "unknown", the stored_revision_id and the node that stored it, under the Retry-After of an undecided answer (crewlet's RefuseActivationUnknown, on main and in crewlet#142 alike): the fleet may run the revision or may not. The operator returned that like any other failure and kept nothing of it. Where the write had landed, the next reconcile read a revision it had not recorded: where its document is the rendered one, it adopted it as a revision it did not write, and where the render had moved on since, it reported its own write as drift it corrected and read every value the write's document lets go as required. Where the write had not landed yet, the next reconcile sent it again, and if the first landed before the second was decided, the second lost the race to it. Under Seed the write that seeded the engine was reported as a document the engine had before the operator seeded one. Only a read can settle it, and two reads do: GET /config (the fleet's activation pointer, crewlet ADR-0057) names the fleet's revision, and GET /fleet's activation the epoch it runs it as. So the write is recorded in status.config.unconfirmed (its revision, the revision it was built on, its appliedHash, when, its object counts, and what it replaced where that was a revision somebody else wrote with other content: replaced Drifted, and driftedAt), and settled by the next reconcile before anything is decided (convergeOnce). The engine runs it: it is recorded as the operator's write at the activation's epoch, with an Activated event, and reported as a confirmed write is, a drift it corrected included (reportReplaced, which recordApply calls too); it is dated when the operator found it, since nothing says when its activation landed, so the engine is held to applying it from then, as an adopted revision is. A fleet that cannot be read, or answers without its activation, leaves it recorded and writes nothing; one whose activation names another revision is a lost race (activationAhead), which converge reads again from the start, as it does for an adopted revision; for that, each attempt reads the fleet after its own read of the document, since the confirmation runs before keepServing reads it afresh. The next look is when the engine's Retry-After says (engineRetry), ten seconds where it sends none. The engine runs the revision the write was built on: it stays, and the write is sent again on the same condition, which the engine refuses if the first one landed meanwhile. Any other: it can no longer land, and goes. A write into an engine that had no document moves to status.config.overtaken instead, for Seed, which takes it for the seed whatever revision runs now, since nothing the engine says tells a seed that landed and was replaced from another writer's create that won; it stays until a reconcile records a revision in sync or the seed, or confirms a write of the operator's running, beside which no create can land or be the seed. A write built on the one confirmed, made in the moment that one landed, stays unconfirmed. Both lists are bounded at the CRD's 16, the newest kept. What the write's activation read of the store is settled two ways. The store changes it read were marked with the fleet's epoch before it was sent, and a revision the engine runs above that epoch settles them, whoever activated it (settleRead). So a write that landed and was activated again before the next read (the engine's reload after it mints a credential, a person's crewlet config reload, the operator's own) is in sync where its document is the rendered one, and nothing activates it again. A change no read of the pointer followed, because the fleet could not be read, which is when the engine is likeliest to lose an answer, or because the engine had no document yet, cannot be settled that way, and the next reconcile sent the same document again over a write that had read it. So the changes the write was sent after await the revision the 503 names (State.Awaiting, saved as the answer comes back, in the state Secret's awaited-revisions), and finding that revision running settles them (State.Found, in settleRead), however the pointer reads. Only an activation the engine confirmed whose record is lost while the pointer could not be read still costs one more activation. What a write needs beside that is how its document reads every name, which nothing else says once the render has moved on: the state records it under the write's appliedHash before every write is sent (State.Pending, saved even where no store change is marked; the state Secret holds it under pending-activations), and a revision the engine runs that is a write of the operator's, whose reading the state does not record, records that reading (settlePending, State.Confirmed): a confirmed unconfirmed write, and a confirmed write whose record in the state the reconcile that made it did not get as far as saving, where status.config still names it. The readings the status no longer names an unconfirmed write for go (KeepPending), so the state does not grow with every write. The pending readings sit behind one pointer of the State beside the reading of the revision it knew, which a confirmation makes of one of them (readingRecord), and the awaited revisions beside the changes they settle, so that a State, which callers pass by value, stays inside the size a value parameter is held to: 64 bytes, where it was 72. A reconcile whose document is held does the same against the revision it reads, and leaves status.config and the report to the next one that applies a document. It reads the engine while a write is unconfirmed even where nothing else is owed: returning first, it left the value of a deleted Secret that a running write lets go, and the revision the state records does not, for that write's tool servers to start with until the document applied. Where it activates such a write again for a value it withdrew, the write's record moves on to the revision that activates it (recordReactivation), which the next reconcile then confirms and reports: left on the write, the reload read as a revision somebody else activated, and the write over it as drift corrected. A reload the operator asks for is a write all the same, and the engine answers its lost compare-and-set the same 503. The stored revision may land yet, so it is recorded from the answer, before anything is read back: a read that fails, or finds the revision running while the fleet cannot be read, says nothing else. Where the revision read is the operator's, the stored one is recorded as an unconfirmed write of its document (reloadUnconfirmed), with its reading saved before the reload and the digest of that document (documentSHA256): the engine activates again whichever revision is active when the request lands, so a revision found running another document reloaded one somebody activated in between, and is not the operator's (runsItsDocument). The changes it was asked after await it. The record replaces nothing of its own: its base is the operator's write, and where that write is recorded too, the confirmation of either reports what the write replaced, once (confirmUnconfirmed follows the bases); copied onto the reload, it named the operator's own write as drift. A revision the read back settles as the reactivation is recorded once: recordReactivation drops an unconfirmed entry of it, which the API server refused beside a second one. A confirmation sets a flag of its own (confirmed) rather than wrote, which says this reconcile activated what it rendered: humanSeats read the rendered document for the human seats IdentityClaimed names, which a confirmation of an earlier render, with this reconcile's refused, are not. The in-sync path and the Applied event read either flag, and the flag is reset with every fresh read. The applying concept page, the troubleshooting guide and the status fields' docs say all of this. The engine client tells this answer from a lost race (UnconfirmedActivation). The withdraw tests' storeEngine answers as crewlet#142 does where the tests need it: a write refused on its If-Match (412 revision_advanced, a failed precondition, naming the revision that runs and the base as the header gave it) or If-None-Match (412 already_configured), an engine with no document, and GET /fleet refused as a lease table its coordination store did not answer is (the code's sentence, the condition's own words as the detail, and the five seconds of the health tick); and a write or a reload whose activation's answer is lost as crewlet's main words it, with the node, the base, the hint and the Retry-After it sends; its fleet's activation names the revision that runs, or nothing for an engine with none; a write it is to refuse is refused in put, as its refusal says; and it names each revision past every one it named and every one a test put in its history by hand, as the serving tests do. unconfirmedWrite rotates a value before the write: r1 is the rendered document and lets the withdrawn registry go, so the withdrawal alone is settled by activating r1 again, with no write to lose. TestARevisionTheEngineCannotOpenBlocksTheDocument counts the writes the engine refused on their condition as well as the ones it stored, and TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt counts the reading saved before the write among the state's writes. TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice, TestAnActivationThatLandsLateIsTheOperatorsOwn, TestAnActivationThatDidNotLandIsSentAgainOnItsCondition and TestASeedTheEngineCouldNotConfirmIsReportedAsTheSeed fail without the fix, as do the record, the unread-fleet and the bound cases; TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice also fails with the readings never dropped. TestAnUnconfirmedActivationOtherwiseOvertakenIsNotSentAgain fails with a write built on another revision kept as overtaken, TestAnUnconfirmedActivationIsNeverWrittenOverUnread with the fleet read skipped, and TestUnconfirmedActivationsAreBounded with either list untrimmed or keeping the oldest, or the constant moved off the CRD's maxItems. TestADriftCorrectionTheEngineCouldNotConfirmIsReportedOnceItRuns fails with the drift left off the record or the report left out of the confirmation; TestAConfirmationThatLosesItsRaceIsReadAgain with the lost race not wrapped as one; TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed with that write dropped; TestAConfirmedWriteIsReportedAppliedOnceTheEngineAppliesIt with the Applied event left to the wrote flag; and TestAConfirmedWriteLeavesTheSeatsToTheRevisionThatRuns with the confirmation setting wrote. TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound fails with the confirmation dated by the write. TestAWriteRecordsItsDocumentsReadingBeforeItIsSent fails with the reading not recorded, or not saved where no store change is marked. TestAHeldReconcileReleasesWhatARunningUnconfirmedWriteLetsGo fails with the held reconcile returning before it reads the engine, with what the confirmed write lets go not read again, with the reading not confirmed, with the reading not recorded before the write, and with the record not moved on to the reload. TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns fails with the stored reload not recorded. TestALandedWriteActivatedAgainStaysInSyncThroughAReconcileThatFails, TestAWriteAReactivationOvertookIsNotSentAgain, TestAHeldDocumentSettlesAnUnconfirmedWriteThatRuns and TestAConfirmationTheStateMissedIsSettledLater hold the activations that follow a write to what its epoch marks settle. In secretsync, TestAPendingReadingRecordsTheRevisionFoundRunningIt fails with a confirmation settling a store change, or dropping a reading, or with KeepPending keeping one it was not given, and TestPendingReadingsSurviveASaveAndALoad with what only a hand edit writes kept. TestAWriteWhoseAnswerIsLostWhileTheFleetIsUnreadableSettlesWhatItRead and TestAFirstWriteWhoseAnswerIsLostIsNotWrittenAgain fail with the write's changes not left awaiting the revision it stored, or with settleRead not settling them on finding it (the fleet answers the read of what every pod runs and not the ones after the changes: an unreadable fleet keeps every value, so nothing would be withdrawn); TestAReloadWhoseAnswerIsLostIsTheOperatorsWhenItsReadBackFails with the stored reload recorded only where the read back found the revision it read, with settleRead not settling what awaits, and with the reload's changes not left awaiting it; TestAReloadOfAnotherRevisionIsNotTheOperators with the document's digest not checked; TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce with recordReactivation keeping the entry it records; TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced with the reload keeping the write's replacement, or the confirmation reporting its own base; and TestAConfirmedWriteLeavesNothingOvertaken with the overtaken writes kept. In secretsync, TestARevisionStoredForAnActivationSettlesTheChangesItWasAskedAfter holds Awaiting and Found, the save and load of awaited-revisions, and a later change waiting for its own activation. The engine activates the revision it runs again itself at a sync's write or removal of a value that revision reads, so the operator's own write or reload, whose answer these tests lose, comes only where that activation failed: the tests make it fail (operatorActivates). Signed-off-by: Ali Yousefian --- api/v1alpha1/company_types.go | 126 +- api/v1alpha1/zz_generated.deepcopy.go | 34 + config/crd/bases/crewlet.ai_companies.yaml | 273 +++- .../templates/crd/companies.crewlet.ai.yaml | 273 +++- docs/concepts/applying.md | 154 +- docs/guides/troubleshooting.md | 24 +- docs/reference/api.md | 47 +- internal/controller/config_apply.go | 14 +- internal/controller/config_run.go | 312 +++- internal/controller/config_unconfirmed.go | 364 +++++ .../controller/config_unconfirmed_test.go | 1426 +++++++++++++++++ internal/controller/config_unreadable_test.go | 6 +- internal/controller/config_withdraw.go | 91 +- internal/controller/config_withdraw_test.go | 239 ++- internal/engine/client_test.go | 55 + internal/engine/engine_test.go | 3 +- internal/engine/errors.go | 24 +- internal/engine/types.go | 3 + internal/secretsync/state.go | 373 ++++- internal/secretsync/state_test.go | 214 ++- 20 files changed, 3800 insertions(+), 255 deletions(-) create mode 100644 internal/controller/config_unconfirmed.go create mode 100644 internal/controller/config_unconfirmed_test.go diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index 3554c9d3..05c8280f 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -1462,8 +1462,10 @@ type ConfigStatus struct { // appliedAt is when the operator activated the revision this records, // or found the engine running it where it adopted a revision of the - // rendered document it did not write (a reload): what the operator's - // wait for the engine to apply its epoch counts from. + // rendered document it did not write (a reload), or a write of its own + // whose activation the engine could not confirm (unconfirmed), which may + // have landed any time since it was written: what the operator's wait + // for the engine to apply its epoch counts from. // +optional AppliedAt *metav1.Time `json:"appliedAt,omitempty"` @@ -1471,6 +1473,45 @@ type ConfigStatus struct { // +optional Objects AppliedObjectCounts `json:"objects,omitzero"` + // unconfirmed are the revisions of the rendered document the operator + // wrote, which the engine stored, and whose activation the engine sent + // to its fleet and could not confirm (a 503 naming the revision it + // stored): the fleet may run one of them or none. Before it writes + // again, the operator reads which revision the engine runs. If it runs + // one of these, that write landed: status.config records it as the + // operator's write, as though the engine had confirmed it. If it runs the + // revision one of them was built on, that one has not landed and still + // may, so it stays, and the write is sent again on the same condition, + // which the engine refuses once one of them lands. Any other can no + // longer land, and goes, to overtaken where it was written into an + // engine that ran no document. A reload the operator asks for, to + // activate a revision of its own again, is recorded here too when the + // engine stores it and cannot confirm activating it, as a write of that + // revision's document, whether or not a read after it could say which + // revision runs. They are oldest first, and a seventeenth drops the + // oldest. + // +optional + // +listType=map + // +listMapKey=revisionID + // +kubebuilder:validation:MaxItems=16 + Unconfirmed []UnconfirmedRevision `json:"unconfirmed,omitempty"` + + // overtaken are writes from unconfirmed into an engine that ran no + // document that another revision the engine activated has overtaken. + // None of them can land any more, and under Seed one of them is taken as + // the seed whatever revision runs now: the engine stored it while it had + // none, so either it landed and was replaced since, or another writer's + // create reached the engine first, which nothing the engine says tells + // apart. Each is kept until a reconcile decides with it, and they go + // once status.config records a revision of the rendered document in + // sync, a write of it, or the seed. They are oldest first, and a + // seventeenth drops the oldest. + // +optional + // +listType=map + // +listMapKey=revisionID + // +kubebuilder:validation:MaxItems=16 + Overtaken []UnconfirmedRevision `json:"overtaken,omitempty"` + // placement is where the revision the engine runs pins its agent seats, // read from the engine's document under every management policy. The // operator keeps a seat pool's pods running while the running revision @@ -1483,6 +1524,87 @@ type ConfigStatus struct { Placement RevisionPlacement `json:"placement,omitzero"` } +// UnconfirmedRevision is a write of the company document whose activation the +// engine could not confirm. +type UnconfirmedRevision struct { + // revisionID is the revision the engine stored for the write. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + RevisionID string `json:"revisionID,omitempty"` + + // baseRevisionID is the revision the write was built on, which its + // activation was conditioned on. Empty for a write into an engine that + // ran no document, conditioned on there being none. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + BaseRevisionID string `json:"baseRevisionID,omitempty"` + + // appliedHash is the appliedHash of the write: the rendered document's + // and its values' hash as they were when it was written, which + // status.config.appliedHash records once the engine is seen running the + // revision. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + AppliedHash string `json:"appliedHash,omitempty"` + + // writtenAt is when the operator wrote it. A write the engine is seen + // running is recorded as applied when the operator found it, since + // nothing says when its activation landed; a seed taken from overtaken, + // which nothing waits on, as applied when it was written. + // +required + WrittenAt *metav1.Time `json:"writtenAt,omitempty"` + + // objects counts the objects the revision was rendered from, which + // status.config.objects records once the engine is seen running it. + // +optional + Objects AppliedObjectCounts `json:"objects,omitzero"` + + // replaced says what the write replaced where that was more than the + // operator's own document, which ConfigApplied and a Warning event report + // once the engine is seen running the revision, as they report a write the + // engine confirmed: Drifted, a revision the operator did not write whose + // document differed from the rendered one, at driftedAt. Unset for any + // other base, and for a reload: its base is a write of the operator's, + // and where that write is recorded here too, the revision found running + // reports what that write replaced, once, whichever of the two it is. + // +optional + Replaced ReplacedRevision `json:"replaced,omitempty"` + + // driftedAt names where the Drifted revision the write replaced differed + // from the rendered document, as ConfigApplied names it: up to five + // places, and how many more there were. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=32768 + DriftedAt string `json:"driftedAt,omitempty"` + + // documentSHA256 is set for a reload: the SHA-256, in hex, of the + // document of the revision it activates again, as the engine answered + // it. The engine activates again whichever revision is active when the + // request lands, so the revision it stored is the operator's only where + // its document is that one; one found running another document was a + // reload of a revision somebody activated in between, and goes. + // +optional + // +kubebuilder:validation:MinLength=64 + // +kubebuilder:validation:MaxLength=64 + // +kubebuilder:validation:Pattern=`^[0-9a-f]{64}$` + DocumentSHA256 string `json:"documentSHA256,omitempty"` +} + +// ReplacedRevision is what a write of the company document replaced where +// that was more than the operator's own document. +// +kubebuilder:validation:Enum=Drifted +type ReplacedRevision string + +const ( + // ReplacedDrifted is a revision the operator did not write whose + // document differed from the rendered one. + ReplacedDrifted ReplacedRevision = "Drifted" +) + // RevisionPlacement is where a revision of the company document pins its // agent seats: the values of the two placement labels its roles name. Only // values that can name one of the company's pods are recorded: a pool name diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 160059d0..54f96596 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -500,6 +500,20 @@ func (in *ConfigStatus) DeepCopyInto(out *ConfigStatus) { *out = (*in).DeepCopy() } in.Objects.DeepCopyInto(&out.Objects) + if in.Unconfirmed != nil { + in, out := &in.Unconfirmed, &out.Unconfirmed + *out = make([]UnconfirmedRevision, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Overtaken != nil { + in, out := &in.Overtaken, &out.Overtaken + *out = make([]UnconfirmedRevision, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Placement.DeepCopyInto(&out.Placement) } @@ -3341,6 +3355,26 @@ func (in *TrackerSettings) DeepCopy() *TrackerSettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UnconfirmedRevision) DeepCopyInto(out *UnconfirmedRevision) { + *out = *in + if in.WrittenAt != nil { + in, out := &in.WrittenAt, &out.WrittenAt + *out = (*in).DeepCopy() + } + in.Objects.DeepCopyInto(&out.Objects) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnconfirmedRevision. +func (in *UnconfirmedRevision) DeepCopy() *UnconfirmedRevision { + if in == nil { + return nil + } + out := new(UnconfirmedRevision) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Unit) DeepCopyInto(out *Unit) { *out = *in diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index f4067fa3..c23fa782 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -1733,8 +1733,10 @@ spec: description: |- appliedAt is when the operator activated the revision this records, or found the engine running it where it adopted a revision of the - rendered document it did not write (a reload): what the operator's - wait for the engine to apply its epoch counts from. + rendered document it did not write (a reload), or a write of its own + whose activation the engine could not confirm (unconfirmed), which may + have landed any time since it was written: what the operator's wait + for the engine to apply its epoch counts from. format: date-time type: string appliedHash: @@ -1793,6 +1795,136 @@ spec: minimum: 0 type: integer type: object + overtaken: + description: |- + overtaken are writes from unconfirmed into an engine that ran no + document that another revision the engine activated has overtaken. + None of them can land any more, and under Seed one of them is taken as + the seed whatever revision runs now: the engine stored it while it had + none, so either it landed and was replaced since, or another writer's + create reached the engine first, which nothing the engine says tells + apart. Each is kept until a reconcile decides with it, and they go + once status.config records a revision of the rendered document in + sync, a write of it, or the seed. They are oldest first, and a + seventeenth drops the oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Drifted, a revision the operator did not write whose + document differed from the rendered one, at driftedAt. Unset for any + other base, and for a reload: its base is a write of the operator's, + and where that write is recorded here too, the revision found running + reports what that write replaced, once, whichever of the two it is. + enum: + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map placement: description: |- placement is where the revision the engine runs pins its agent seats, @@ -1843,6 +1975,143 @@ spec: maxLength: 128 minLength: 1 type: string + unconfirmed: + description: |- + unconfirmed are the revisions of the rendered document the operator + wrote, which the engine stored, and whose activation the engine sent + to its fleet and could not confirm (a 503 naming the revision it + stored): the fleet may run one of them or none. Before it writes + again, the operator reads which revision the engine runs. If it runs + one of these, that write landed: status.config records it as the + operator's write, as though the engine had confirmed it. If it runs the + revision one of them was built on, that one has not landed and still + may, so it stays, and the write is sent again on the same condition, + which the engine refuses once one of them lands. Any other can no + longer land, and goes, to overtaken where it was written into an + engine that ran no document. A reload the operator asks for, to + activate a revision of its own again, is recorded here too when the + engine stores it and cannot confirm activating it, as a write of that + revision's document, whether or not a read after it could say which + revision runs. They are oldest first, and a seventeenth drops the + oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Drifted, a revision the operator did not write whose + document differed from the rendered one, at driftedAt. Unset for any + other base, and for a reload: its base is a write of the operator's, + and where that write is recorded here too, the revision found running + reports what that write replaced, once, whichever of the two it is. + enum: + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map type: object endpoints: description: endpoints is where the company is reached. diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index da736a6e..a306fd09 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -1736,8 +1736,10 @@ spec: description: |- appliedAt is when the operator activated the revision this records, or found the engine running it where it adopted a revision of the - rendered document it did not write (a reload): what the operator's - wait for the engine to apply its epoch counts from. + rendered document it did not write (a reload), or a write of its own + whose activation the engine could not confirm (unconfirmed), which may + have landed any time since it was written: what the operator's wait + for the engine to apply its epoch counts from. format: date-time type: string appliedHash: @@ -1796,6 +1798,136 @@ spec: minimum: 0 type: integer type: object + overtaken: + description: |- + overtaken are writes from unconfirmed into an engine that ran no + document that another revision the engine activated has overtaken. + None of them can land any more, and under Seed one of them is taken as + the seed whatever revision runs now: the engine stored it while it had + none, so either it landed and was replaced since, or another writer's + create reached the engine first, which nothing the engine says tells + apart. Each is kept until a reconcile decides with it, and they go + once status.config records a revision of the rendered document in + sync, a write of it, or the seed. They are oldest first, and a + seventeenth drops the oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Drifted, a revision the operator did not write whose + document differed from the rendered one, at driftedAt. Unset for any + other base, and for a reload: its base is a write of the operator's, + and where that write is recorded here too, the revision found running + reports what that write replaced, once, whichever of the two it is. + enum: + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map placement: description: |- placement is where the revision the engine runs pins its agent seats, @@ -1846,6 +1978,143 @@ spec: maxLength: 128 minLength: 1 type: string + unconfirmed: + description: |- + unconfirmed are the revisions of the rendered document the operator + wrote, which the engine stored, and whose activation the engine sent + to its fleet and could not confirm (a 503 naming the revision it + stored): the fleet may run one of them or none. Before it writes + again, the operator reads which revision the engine runs. If it runs + one of these, that write landed: status.config records it as the + operator's write, as though the engine had confirmed it. If it runs the + revision one of them was built on, that one has not landed and still + may, so it stays, and the write is sent again on the same condition, + which the engine refuses once one of them lands. Any other can no + longer land, and goes, to overtaken where it was written into an + engine that ran no document. A reload the operator asks for, to + activate a revision of its own again, is recorded here too when the + engine stores it and cannot confirm activating it, as a write of that + revision's document, whether or not a read after it could say which + revision runs. They are oldest first, and a seventeenth drops the + oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Drifted, a revision the operator did not write whose + document differed from the rendered one, at driftedAt. Unset for any + other base, and for a reload: its base is a write of the operator's, + and where that write is recorded here too, the revision found running + reports what that write replaced, once, whichever of the two it is. + enum: + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map type: object endpoints: description: endpoints is where the company is reached. diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index b95cec01..e25a4479 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -225,10 +225,15 @@ flowchart TD the running document again (the request landed although its answer did not, or somebody reloaded the document), it is recorded as the reactivation, and where it is another document, a Warning `Reactivated` - says the operator activated nothing again. Nothing makes the engine's - reload or its deletion conditional on the revision the operator read, so - a revision somebody activates in that moment goes without a value it may - require until the Secret exists again. + says the operator activated nothing again. Where the engine still runs + the revision read, and answered that it stored the request's revision and + could not confirm activating it, that revision may land yet: the deletion + stays owed and is reported, and, where the revision read is the + operator's, the stored one is recorded as an unconfirmed write of its + document (step 6), so that once it runs it is the operator's. Nothing + makes the engine's reload or its deletion conditional on the revision the + operator read, so a revision somebody activates in that moment goes + without a value it may require until the Secret exists again. 4. **Compare.** The operator compares the engine's active document with the rendered one, and checks that the revision the engine runs activated after every value it wrote into the store. If both hold, there is nothing @@ -268,7 +273,53 @@ flowchart TD engine with no document yet, on the condition that it still has none). The revision's summary is `crewlet-operator:` followed by the objects whose part of the document changed. A write that loses a race with - another writer is started over once from a fresh read. + another writer is started over once from a fresh read. When the engine + stores the revision and then can't confirm that it activated it (a `503` + naming the revision it stored), the fleet may run it or may not, so the + operator records the write in `status.config.unconfirmed` and writes + nothing more until it has read which revision the engine runs: that one, + and it is recorded and reported as the operator's write, as though the + engine had confirmed it, a revision somebody else wrote that it replaced + included (`DriftCorrected`, with the `Drifted` event recorded then), at + the epoch the fleet's activation names and as applied when the operator + found it, since nothing says when its activation landed; the one the + write was built on, and the write is sent again on the same condition, + which the engine refuses if the first one landed meanwhile; any other, + and the write goes, since it can no longer land. If it landed and was + activated again since (the engine's own reload after it mints a + credential, a `crewlet config reload`, or the operator's own activation + of it again), the revision that runs activated after every value the + write's sync wrote and withdrew ([What the engine has read](#what-the-engine-has-read)), + so, where it is the rendered document, it is in sync and nothing is + written. Under `Seed`, a write into an engine that had no document is + kept in `status.config.overtaken` once overtaken, and is the seed, + whatever revision runs now: nothing the engine says tells a seed that + landed and was replaced from another writer's create that reached the + engine first. It stays recorded until a reconcile decides with it, so one + that fails first does not lose it. Before the write is sent, the operator + also records, under the write's `appliedHash`, which Secret keys its + document lets go without (`pending-activations`), since nothing else says + it once the render has moved on: a write the engine is found running that + `status.config` names as the operator's, confirmed or not, is recorded as + reading the store that way, even where the reconcile that made it did not + get as far as saving the record of the store after it, so that the value of + a deleted Secret it lets go is taken out of the store rather than kept for + it. Where the `status.config` record of the write is lost too, the revision + reads as one the operator has no record of activating, requiring every + value it references. A reconcile that holds its document reads the engine + while a write is unconfirmed, records how a write that runs reads the + store, and leaves `status.config` to the next one that applies a document; + where it activates that write again, the write's record moves on to the + revision that activates it, which the next reconcile then confirms. A + request to activate the running revision again that the engine stores and + cannot confirm activating is recorded the same way, as a write of the + operator's of that revision's document, from the answer, whatever the + operator can read back after it (step 3), with a digest of that document + (`documentSHA256`): the engine activates again whichever revision is active + when the request lands, so a revision found running another document was a + reload of one somebody activated in between, and is not the operator's. + What a write replaced is reported once, by whichever of the write and such + a reload of it is found running. 7. **Clean up** the store. The operator records the new revision in its record of the store, with the Secret keys its document lets go without, before it deletes anything, so that a deletion whose own record a failed @@ -356,20 +407,23 @@ dry-runs and compares, and reads no Secret value and writes nothing; under never again (see [Management policies](#management-policies)). The operator remembers what it pushed as keyed fingerprints of the values, in -a Secret of its own, `-secret-sync`, owned by the Company: the -engine never returns a value, so this is how the operator knows which ones -changed. A fingerprint without the key that Secret also holds says nothing -about a value. The same Secret holds the operator's record of the store's -other facts: the revision whose reading of the values it knows, with the -Secret keys that revision lets go without (`optional-keys`), the values it -deleted that a revision has yet to activate without (`withdrawn-names`), the -ones it wrote that a revision has yet to activate with (`written-names`), -each of those two with the activation epoch it waits past, and the ones it -said it keeps. It records nothing of the engine's pods: what each runs and -would start on is what `GET /fleet` says of it now, since a pod applies -revisions, and a data pod moves the revision its volume starts it on, whether -or not the operator is looking, and a pod the fleet does not list keeps every -value it may need instead (step 3). +a Secret of its own, `-secret-sync`, owned by the Company: the engine +never returns a value, so this is how the operator knows which ones changed. A +fingerprint without the key that Secret also holds says nothing about a value. +The same Secret holds the operator's record of the store's other facts: the +revision whose reading of the values it knows, with the Secret keys that +revision lets go without (`optional-keys`), the values it deleted that a +revision has yet to activate without (`withdrawn-names`), the ones it wrote +that a revision has yet to activate with (`written-names`), each of those two +with the activation epoch it waits past and the revisions it awaits +(`awaited-revisions`), the ones it said it keeps, and, for each write of the +document the engine may yet be found running before the operator's record of +the store has followed it, the Secret keys that write's document lets go +without, under its `appliedHash` (`pending-activations`). It records nothing +of the engine's pods: what each runs and would start on is what `GET /fleet` +says of it now, since a pod applies revisions, and a data pod moves the +revision its volume starts it on, whether or not the operator is looking, and +a pod the fleet does not list keeps every value it may need instead (step 3). The engine is reached at its first data pod, `http://-data-0.-data..svc:8080`, with the @@ -414,13 +468,19 @@ revision that activated before it. A write of the document that the engine runs is therefore never sent again because the record of its activation was lost: the next reconcile finds the revision running above the epoch every value was marked by, and takes it for in sync. That needs a read of the -pointer between the change and the activation. A change the operator could -not follow with one, because the fleet could not be read or the engine had no -document yet, is settled by the activation the operator sees answered, and -where the record of that answer is lost as well, the next reconcile -activates a revision again for it. So is a change whose activation the -engine made itself: the last write or removal of a sync activates the -revision inside the same request, before any read of the pointer can follow the change, so the +pointer between the change and the activation. A change the operator could not +follow with one, because the fleet could not be read or the engine had no +document yet, is settled by the activation the operator sees answered; where +the engine answers that it stored a revision for the activation and could not +confirm activating it, which is when the pointer is likeliest to be +unreadable, the change awaits that revision (`awaited-revisions`) and is +settled once the engine is read running it, however the pointer reads then. +Only where the engine confirmed the activation and the operator lost the +record of that answer as well does the next reconcile activate a revision +again for such a change: one write or reload of the document more, with the +same values. So is a change whose activation the engine made itself: the +last write or removal of a sync activates the revision inside the same +request, before any read of the pointer can follow the change, so the operator settles it from that request's answer once it has read back that the engine runs the revision it names, and a save lost after that costs one activation more. @@ -608,20 +668,30 @@ Besides the conditions: document as, its activation epoch, `appliedHash`, when the operator activated it or found it running (`appliedAt`), and how many objects of each kind it was rendered from (platform model providers included). - `status.config.placement` is where the document the engine runs pins its - agent seats: every value of `crewlet.ai/pool` (`data` and the Shared pools - that run a seat) and of `crewlet.ai/seat` (each seat of a PerSeat pool). It - is read from the engine's own document under every policy, so under Observe - and Seed too. A new revision's pins join the record before the operator's - write that activates it, and while the engine applies it the record is the - previous revision's pins and the new one's together, because the engine - keeps running the previous revision until it has applied the new one, and - for good if that apply fails. Once the engine reports the new revision's - epoch applied, the record is its pins alone; a document the engine owns - (Observe, Seed) is narrowed to only once the engine's own record of its last - apply names that revision and says it ended well. The runtime reconciler - reads it to keep a seat pool's pods running for as long as the running - revision pins a seat to them ([A pool removed while seats still run in + `status.config.unconfirmed` lists the operator's writes the engine stored + and couldn't confirm activating, at most 16, the oldest dropped first, until + a read of the revision the engine runs settles them (step 6 above), each + with what it replaced where that was somebody else's revision + (`replaced: Drifted`, and `driftedAt`, where it differed), for the report + made once the engine is seen running it. `status.config.overtaken` lists the + ones written into an engine that had no document that another revision + overtook, which can no longer land, and of which `Seed` takes one for the + seed, at most 16, until a reconcile records a revision of the rendered + document in sync, a write of it, or the seed. `status.config.placement` is + where the document the engine runs pins its agent seats: every value of + `crewlet.ai/pool` (`data` and the Shared pools that run a seat) and of + `crewlet.ai/seat` (each seat of a PerSeat pool). It is read from the + engine's own document under every policy, so under Observe and Seed too. A + new revision's pins join the record before the operator's write that + activates it, and while the engine applies it the record is the previous + revision's pins and the new one's together, because the engine keeps running + the previous revision until it has applied the new one, and for good if that + apply fails. Once the engine reports the new revision's epoch applied, the + record is its pins alone; a document the engine owns (Observe, Seed) is + narrowed to only once the engine's own record of its last apply names that + revision and says it ended well. The runtime reconciler reads it to keep a + seat pool's pods running for as long as the running revision pins a seat to + them ([A pool removed while seats still run in it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). - **Seat**: its handle, its identity in the engine (`seatID`, derived from the company name and the handle), the pool that runs it and the @@ -642,7 +712,9 @@ Besides the conditions: Every event the config reconciler records is on the Company, but a seat's pause, which is recorded on the Seat (`SeatPaused`, `SeatResumed`, and `Drifted` when it undid a change made in the engine): `Activated` -with the revision, its epoch and its summary, `Applied` once the engine has +with the revision, its epoch and its summary (for a write the engine couldn't +confirm, with the revision and its epoch once a read shows the engine runs +it), `Applied` once the engine has applied it, `ApplyFailed` when it did not, `SecretsSynced` with the store entries it wrote and deleted, a Warning `SecretOverwritten` naming each entry it wrote back and who had replaced it, `SecretKept` naming each value of a diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index ce12fd6e..1e027250 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -119,7 +119,7 @@ with your last change yet. | | False | `Drifted` | Under `Observe`, the engine runs another document; the message names the places that differ. | | | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write, or after the operator found it running where it adopts a revision it did not write. | | | False | `ApplyFailed` | The engine's first data pod, the one the operator reaches, did not apply the revision: it keeps running the one it ran before, and starts on the one its volume names if it restarts, the same one unless the engine is still moving it (the message says which), and the engine's secret store keeps every value of either meanwhile. The message carries the engine's status, its posture (`isolated` on a single node, `shed` or `stuck` beside peers that applied it) and its own account of the failure; see [The engine did not apply a revision](#the-engine-did-not-apply-a-revision). | -| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine asked to be asked again later (`503`: it is starting, or cannot read the coordination store that holds the fleet's revision). Retried every ten seconds, or sooner when the engine says when; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | +| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine answered `503`: it asked to be asked again later (it is starting, or cannot read the coordination store that holds the fleet's revision), or it stored the operator's write and couldn't confirm activating it. Looked at again every ten seconds, or sooner when the engine says when, and a write it couldn't confirm is not sent again until a read shows it did not land; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | @@ -369,6 +369,28 @@ requests to `http://-data-0.-data..svc:8080` fail: document it cannot vouch for, and the engine logs `config_request_unavailable`. It clears once the engine's broker is back; the data pod's log says why it is not. +- The engine stored the operator's write and couldn't confirm that it + activated it: the message names the revision. The engine may run it or may + not. Nothing needs doing: the operator keeps the write in + `status.config.unconfirmed`, reads which revision the engine runs before it + writes again, and records the write as its own if the engine runs it, with + a `Normal` event `Activated` and, where it replaced a revision somebody else + wrote, `ConfigApplied=True DriftCorrected` and a Warning `Drifted`, as for + a write the engine confirmed, dated when it found it running. It never + writes the document a second time on top of one that landed, nor on top of + one that landed and was activated again before it looked (a reload): that + revision activated after every value the write was made with, so where it + is the rendered document the engine runs it with them, and it is in sync. + Under `Seed` a write into an engine that had no document is kept in + `status.config.overtaken` once another revision runs, and reported as the + seed, whatever revision runs now. A request to activate the operator's own + revision again, for the value of a deleted Secret or one written since, + that the engine stored and couldn't confirm is kept the same way, whatever + the operator could read back after it, beside the `SecretWithdrawalFailed` + or `SecretRefreshFailed` Warning, and recorded as the operator's once it + runs that revision's document; one found running another document reloaded + a revision somebody activated before the request landed, and is not the + operator's. ## Nobody can sign in diff --git a/docs/reference/api.md b/docs/reference/api.md index 74582eff..2c1331c7 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -91,6 +91,7 @@ _Validation:_ _Appears in:_ - [ConfigStatus](#configstatus) +- [UnconfirmedRevision](#unconfirmedrevision) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -761,8 +762,10 @@ _Appears in:_ | `revisionID` _string_ | revisionID is the engine's id of the last revision the operator saw
run the rendered document: the one it applied, one the engine
activated with the same content (a reload), or, under Observe, the
one it found in sync. Under Seed it is the revision the document was
seeded as. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| | `epoch` _integer_ | epoch is the engine's activation counter for that revision. The
engine applies a revision asynchronously; it runs it once the applied
epoch its /health reports has reached this. | | Minimum: 0
Optional: \{\}
| | `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as the revision recorded runs
them: the operator's last write, or a revision it found running that
document after every value it wrote. The values enter it only as
keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| -| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator activated the revision this records,
or found the engine running it where it adopted a revision of the
rendered document it did not write (a reload): what the operator's
wait for the engine to apply its epoch counts from. | | Optional: \{\}
| +| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator activated the revision this records,
or found the engine running it where it adopted a revision of the
rendered document it did not write (a reload), or a write of its own
whose activation the engine could not confirm (unconfirmed), which may
have landed any time since it was written: what the operator's wait
for the engine to apply its epoch counts from. | | Optional: \{\}
| | `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from. | | MinProperties: 1
Optional: \{\}
| +| `unconfirmed` _[UnconfirmedRevision](#unconfirmedrevision) array_ | unconfirmed are the revisions of the rendered document the operator
wrote, which the engine stored, and whose activation the engine sent
to its fleet and could not confirm (a 503 naming the revision it
stored): the fleet may run one of them or none. Before it writes
again, the operator reads which revision the engine runs. If it runs
one of these, that write landed: status.config records it as the
operator's write, as though the engine had confirmed it. If it runs the
revision one of them was built on, that one has not landed and still
may, so it stays, and the write is sent again on the same condition,
which the engine refuses once one of them lands. Any other can no
longer land, and goes, to overtaken where it was written into an
engine that ran no document. A reload the operator asks for, to
activate a revision of its own again, is recorded here too when the
engine stores it and cannot confirm activating it, as a write of that
revision's document, whether or not a read after it could say which
revision runs. They are oldest first, and a seventeenth drops the
oldest. | | MaxItems: 16
Optional: \{\}
| +| `overtaken` _[UnconfirmedRevision](#unconfirmedrevision) array_ | overtaken are writes from unconfirmed into an engine that ran no
document that another revision the engine activated has overtaken.
None of them can land any more, and under Seed one of them is taken as
the seed whatever revision runs now: the engine stored it while it had
none, so either it landed and was replaced since, or another writer's
create reached the engine first, which nothing the engine says tells
apart. Each is kept until a reconcile decides with it, and they go
once status.config records a revision of the rendered document in
sync, a write of it, or the seed. They are oldest first, and a
seventeenth drops the oldest. | | MaxItems: 16
Optional: \{\}
| | `placement` _[RevisionPlacement](#revisionplacement)_ | placement is where the revision the engine runs pins its agent seats,
read from the engine's document under every management policy. The
operator keeps a seat pool's pods running while the running revision
pins a seat to them, even after the pool or the seat's placement is
gone from the spec: a pod stopped while its seat is still pinned to
it would leave the seat held by nobody, and a blocked document keeps
the last good revision running. Once a revision that pins nothing to
them runs, the pods are deleted. | | MinProperties: 1
Optional: \{\}
| @@ -2916,6 +2919,24 @@ _Appears in:_ | `Atlassian` | | +#### ReplacedRevision + +_Underlying type:_ _string_ + +ReplacedRevision is what a write of the company document replaced where +that was more than the operator's own document. + +_Validation:_ +- Enum: [Drifted] + +_Appears in:_ +- [UnconfirmedRevision](#unconfirmedrevision) + +| Field | Description | +| --- | --- | +| `Drifted` | ReplacedDrifted is a revision the operator did not write whose
document differed from the rendered one.
| + + #### RetentionSettings @@ -4086,6 +4107,30 @@ _Appears in:_ | `worklet` _[WorkletSettings](#workletsettings)_ | worklet is the policy of Worklet, for backend Worklet. | | MinProperties: 1
Optional: \{\}
| +#### UnconfirmedRevision + + + +UnconfirmedRevision is a write of the company document whose activation the +engine could not confirm. + + + +_Appears in:_ +- [ConfigStatus](#configstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `revisionID` _string_ | revisionID is the revision the engine stored for the write. | | MaxLength: 128
MinLength: 1
Required: \{\}
| +| `baseRevisionID` _string_ | baseRevisionID is the revision the write was built on, which its
activation was conditioned on. Empty for a write into an engine that
ran no document, conditioned on there being none. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| +| `appliedHash` _string_ | appliedHash is the appliedHash of the write: the rendered document's
and its values' hash as they were when it was written, which
status.config.appliedHash records once the engine is seen running the
revision. | | MaxLength: 128
MinLength: 1
Required: \{\}
| +| `writtenAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | writtenAt is when the operator wrote it. A write the engine is seen
running is recorded as applied when the operator found it, since
nothing says when its activation landed; a seed taken from overtaken,
which nothing waits on, as applied when it was written. | | Required: \{\}
| +| `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from, which
status.config.objects records once the engine is seen running it. | | MinProperties: 1
Optional: \{\}
| +| `replaced` _[ReplacedRevision](#replacedrevision)_ | replaced says what the write replaced where that was more than the
operator's own document, which ConfigApplied and a Warning event report
once the engine is seen running the revision, as they report a write the
engine confirmed: Drifted, a revision the operator did not write whose
document differed from the rendered one, at driftedAt. Unset for any
other base, and for a reload: its base is a write of the operator's,
and where that write is recorded here too, the revision found running
reports what that write replaced, once, whichever of the two it is. | | Enum: [Drifted]
Optional: \{\}
| +| `driftedAt` _string_ | driftedAt names where the Drifted revision the write replaced differed
from the rendered document, as ConfigApplied names it: up to five
places, and how many more there were. | | MaxLength: 32768
MinLength: 1
Optional: \{\}
| +| `documentSHA256` _string_ | documentSHA256 is set for a reload: the SHA-256, in hex, of the
document of the revision it activates again, as the engine answered
it. The engine activates again whichever revision is active when the
request lands, so the revision it stored is the operator's only where
its document is that one; one found running another document was a
reload of a revision somebody activated in between, and goes. | | MaxLength: 64
MinLength: 64
Pattern: `^[0-9a-f]\{64\}$`
Optional: \{\}
| + + #### Unit diff --git a/internal/controller/config_apply.go b/internal/controller/config_apply.go index a58f408f..43831340 100644 --- a/internal/controller/config_apply.go +++ b/internal/controller/config_apply.go @@ -44,7 +44,9 @@ const engineLagGrace = 45 * time.Second // applyDeadline is how long after a write the operator waits for an engine // whose posture still reads wait, or after it first finds the engine running // a revision of the rendered document it did not write, which it adopts -// (inSync). A node reports wait for the whole of an +// (inSync), or a write of its own whose activation the engine could not +// confirm, which may have landed any time since (confirmUnconfirmed). A node +// reports wait for the whole of an // apply, and turns serve when it succeeds or a diverged posture when an // attempt fails, so wait past this is one of two things the engine itself // never concludes: it is not polling its activation pointer at all, or an @@ -118,7 +120,8 @@ func (r *CompanyConfigReconciler) clock() time.Time { // records and has not diverged since, Applying while its posture reads wait, // and ApplyFailed when its posture reads diverged, or still wait // applyDeadline after the write, or after the operator found the revision it -// adopted (status.config.appliedAt). It runs on every look at the engine, the +// adopted, or a write of its own whose activation the engine could not +// confirm (status.config.appliedAt). It runs on every look at the engine, the // periodic resync included, so an engine that diverges after applying the // operator's revision, on one someone activates later, is reported too. // health and healthErr are what readHealth read; result is what the @@ -136,10 +139,7 @@ func (run *configRun) verifyApplied( return result } key := types.NamespacedName{Namespace: run.company.Namespace, Name: run.company.Name} - config := run.company.Status.Config - if run.config != nil { - config = *run.config - } + config := run.recordedConfig() var target int64 if config.Epoch != nil { target = *config.Epoch @@ -237,7 +237,7 @@ func (run *configRun) settled(key types.NamespacedName, target int64, revision s run.recordPlacement(*run.running, true) } current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied) - if run.wrote || (current != nil && + if run.wrote || run.confirmed != "" || (current != nil && (current.Reason == crewletaiv1alpha1.ReasonApplying || current.Reason == crewletaiv1alpha1.ReasonApplyFailed)) { run.event(corev1.EventTypeNormal, crewletaiv1alpha1.ReasonApplied, "ApplyConfig", "The engine applied revision %s (epoch %d).", revision, target) diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 5b75f031..c88165ce 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -91,6 +91,11 @@ type configRun struct { config *crewletaiv1alpha1.ConfigStatus // wrote is whether this reconcile activated a revision. wrote bool + // confirmed is the revision, empty for none, that this reconcile found + // the engine running and that the operator wrote and the engine could + // not confirm activating (confirmUnconfirmed): a write an earlier + // reconcile made, of what it rendered then. + confirmed string // objectsChecked and secretsChecked are whether the reconcile got as // far as resolving the parts' references to other objects, and to @@ -378,31 +383,7 @@ func (run *configRun) policy() crewletaiv1alpha1.ManagementPolicy { // error, and the reconcile is retried. func (run *configRun) converge(ctx context.Context, eng *engine.Client) (ctrl.Result, error) { for attempt := 1; ; attempt++ { - active, err := activeConfig(ctx, eng) - if err != nil { - return ctrl.Result{}, err - } - if active != nil { - run.active = active.Document - } - var result ctrl.Result - switch policy := run.policy(); { - case policy == crewletaiv1alpha1.ManagementPolicyObserve: - return run.observe(ctx, eng, active) - case policy == crewletaiv1alpha1.ManagementPolicySeed && active != nil: - if err := run.seeded(active); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: run.resyncInterval()}, nil - default: - if run.desired == nil { - blocked, err := run.resolveSecrets(ctx) - if blocked || err != nil { - return ctrl.Result{}, err - } - } - result, err = run.enforce(ctx, eng, *run.desired, active) - } + result, err := run.convergeOnce(ctx, eng) lost := errors.Is(err, engine.ErrRevisionAdvanced) || errors.Is(err, engine.ErrAlreadyConfigured) if lost && attempt == 1 { logf.FromContext(ctx).V(1).Info("Another writer changed the engine's document during the apply; reading it again") @@ -412,22 +393,74 @@ func (run *configRun) converge(ctx context.Context, eng *engine.Client) (ctrl.Re } } -// activationAhead is inSync's answer where the fleet's activation names a -// revision other than the one GET /config answered with: both read the -// fleet's activation pointer, so another write activated a revision between -// the two reads. It is a lost race (engine.ErrRevisionAdvanced), which -// converge reads again from the start once, as it reads a write that lost. +// convergeOnce is one attempt of converge, from a fresh read of the engine's +// document. +func (run *configRun) convergeOnce(ctx context.Context, eng *engine.Client) (ctrl.Result, error) { + // The fleet is read after this attempt's read of the document, never + // before it: confirmUnconfirmed, which runs before keepServing reads it + // afresh, takes the epoch the fleet's activation names for the revision + // this read returns, and an answer an attempt that lost its write read + // describes the engine from before that revision. + run.fleet, run.fleetErr = nil, nil + // A confirmation is of the revision the read runs: another one runs once + // the reconcile starts over. + run.confirmed = "" + active, err := activeConfig(ctx, eng) + if err != nil { + return ctrl.Result{}, err + } + if active != nil { + run.active = active.Document + } + // Before anything is decided from it: the revision the engine runs may be + // a write of the operator's that the engine could not confirm + // (config_unconfirmed.go). + if err := run.settleUnconfirmed(ctx, eng, active); err != nil { + return ctrl.Result{}, err + } + switch policy := run.policy(); { + case policy == crewletaiv1alpha1.ManagementPolicyObserve: + return run.observe(ctx, eng, active) + case policy == crewletaiv1alpha1.ManagementPolicySeed && active != nil: + if err := run.seeded(active); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: run.resyncInterval()}, nil + } + if run.desired == nil { + blocked, err := run.resolveSecrets(ctx) + if blocked || err != nil { + return ctrl.Result{}, err + } + } + return run.enforce(ctx, eng, *run.desired, active) +} + +// activationAhead is inSync's answer, and confirmUnconfirmed's, where the +// fleet's activation names a revision other than the one GET /config answered +// with: both read the fleet's activation pointer, so another write activated +// a revision between the two reads. It is a lost race +// (engine.ErrRevisionAdvanced), which converge reads again from the start +// once, as it reads a write that lost. type activationAhead struct { // read is the revision GET /config answered with. read string // activation is the fleet's. activation engine.Activation + // unconfirmed is whether read is a write of the operator's whose + // activation the engine could not confirm, rather than a revision of the + // rendered document the operator has not recorded. + unconfirmed bool } func (e *activationAhead) Error() string { - return fmt.Sprintf("the fleet runs the rendered document as revision %s, which the operator has not recorded, "+ - "and its activation named revision %s (epoch %d) by the time the operator read the fleet: another write "+ - "activated a revision between the two reads", e.read, e.activation.RevisionID, e.activation.Epoch) + runs := fmt.Sprintf("the rendered document as revision %s, which the operator has not recorded", e.read) + if e.unconfirmed { + runs = fmt.Sprintf("revision %s, which the operator wrote and the engine could not confirm activating", e.read) + } + return fmt.Sprintf("the fleet runs %s, and its activation named revision %s (epoch %d) by the time the operator "+ + "read the fleet: another write activated a revision between the two reads", runs, e.activation.RevisionID, + e.activation.Epoch) } func (e *activationAhead) Unwrap() error { return engine.ErrRevisionAdvanced } @@ -476,6 +509,7 @@ func (run *configRun) enforce( if err != nil { return ctrl.Result{}, err } + state = run.settlePending(state, active) run.told = state.Retained() diffs, err := diffWith(run.doc, active) if err != nil { @@ -519,7 +553,7 @@ func (run *configRun) enforce( } hash := appliedHash(run.doc, state, desired.Names()) - recorded := c.Status.Config + recorded := run.recordedConfig() // The engine runs the rendered document with every value only where the // revision it runs read the store after the last value the sync wrote: // the engine reads its store only when a revision activates. The state @@ -583,8 +617,13 @@ func (run *configRun) enforce( } // What the write's activation reads is marked before it is asked for, // so that a save lost after it still leaves what it read settled by the - // next read of the engine. - if state, err = run.markChanges(ctx, eng, state, stateRef); err != nil { + // next read of the engine; and so is how its document reads every name, + // for the revision the engine stores, which a later read may find + // running although its activation went unconfirmed, or its record was + // never saved (settlePending). + ahead := !state.Pends(hash) + state = state.Pending(hash, run.doc.Optional) + if state, err = run.markChanges(ctx, eng, state, stateRef, ahead); err != nil { return ctrl.Result{}, err } req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, state.Written())} @@ -600,10 +639,8 @@ func (run *configRun) enforce( } applied, err := eng.ApplyConfig(ctx, req) if err != nil { - if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef, err); refused { - return result, saveErr - } - return ctrl.Result{}, err + return run.writeFailed(ctx, eng, err, writeAttemptOf(req, hash, drifted, diffs), active, pushed.Reload, + pushed.ReloadAction, state, stateRef) } run.recordApply(applied, hash, drifted, active, diffs, req.Summary) @@ -701,6 +738,58 @@ func (run *configRun) refusedByEngine( return ctrl.Result{}, false, nil } +// writeAttempt is a write of the rendered document: the revision it was +// built on, which it is conditioned on ("" for none), its appliedHash, and +// what it replaced where that was more than the operator's own document +// (replaced, with driftedAt, the places a Drifted one differed at as +// ConfigApplied names them), which a write the engine could not confirm keeps +// for the report made once a read shows it runs. +type writeAttempt struct { + base, hash string + replaced crewletaiv1alpha1.ReplacedRevision + driftedAt string +} + +// writeAttemptOf is req, a write of the rendered document whose appliedHash +// is hash, as a writeAttempt: drifted where the revision it was built on is +// somebody else's, which differed from the rendered document at diffs. +func writeAttemptOf(req engine.ApplyRequest, hash string, drifted bool, diffs []render.Path) writeAttempt { + attempt := writeAttempt{base: req.IfMatch, hash: hash} + if drifted { + attempt.replaced, attempt.driftedAt = crewletaiv1alpha1.ReplacedDrifted, placesOf(diffs) + } + return attempt +} + +// writeFailed ends the reconcile where the engine did not take the write, +// failing with err. A document it refused is settled as one (refusedByEngine, +// with the sync's reload and reloadAction). +// A write it stored and could not confirm activating is recorded in +// status.config.unconfirmed, the secret sync's state having recorded what its +// activation reads before it was sent, and settled by the next read of the +// revision the engine runs, never by sending the write again before it +// (config_unconfirmed.go). Any other failure goes back. +func (run *configRun) writeFailed( + ctx context.Context, eng *engine.Client, err error, write writeAttempt, active *engine.ActiveConfig, + reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, +) (ctrl.Result, error) { + if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, reload, reloadAction, state, ref, err); refused { + return result, saveErr + } + if stored, ok := engine.UnconfirmedActivation(err); ok { + // How its document reads every name the state recorded before it + // was sent (markChanges), and the changes it was sent after await + // the revision the engine stored, which settles them once found + // running however the fleet's pointer reads (State.Awaiting). + after := run.unconfirmed(stored, write, err) + if saveErr := secretsync.SaveState(ctx, run.Client, ref, run.company, state.Awaiting(stored)); saveErr != nil { + return ctrl.Result{}, saveErr + } + return ctrl.Result{RequeueAfter: after}, nil + } + return ctrl.Result{}, err +} + // adopt is state recording the reading of active, the revision the engine // runs, where its document is the one this reconcile rendered: diffs, where // the two differ, is empty, and the engine's document references every store @@ -768,30 +857,37 @@ func (run *configRun) reloadedBySync( } // settleRead is state with every change of the store settled that the -// fleet's activation came after (State.Ran), and every change no read of the -// fleet's activation pointer has followed marked by the epoch it names now -// (State.Observed), and whether that moved a mark, which the caller saves -// before it asks for an activation (markChanges' rule: a save lost after the -// activation must leave a change the activation settles). It reads the fleet -// the reconcile last read (fleetOf), which keepServing reads afresh after -// this attempt's read of the document. active is the revision that read -// returned, which GET /config opened: the activation settles only where it is -// that revision. Both read the fleet's activation pointer, so another -// revision there is one a write activated between the two reads, perhaps one -// no node opens. state holds only what earlier reconciles changed, so any -// read of the fleet this attempt makes followed it. A fleet that cannot be -// read, or answers without its activation, settles and marks nothing. +// fleet's activation came after (State.Ran), or that awaits the revision +// this attempt read (State.Found: one the engine stored for an activation +// asked for after the change and could not confirm, which settles it however +// the pointer reads, and whether or not the fleet can be read), and every +// change no read of the fleet's activation pointer has followed marked by the +// epoch it names now (State.Observed), and whether that moved a mark, which +// the caller saves before it asks for an activation (markChanges' rule: a +// save lost after the activation must leave a change the activation settles). +// It reads the fleet the reconcile last read (fleetOf), which keepServing +// reads afresh after this attempt's read of the document. active is the +// revision that read returned, which GET /config opened: the activation +// settles only where it is that revision. Both read the fleet's activation +// pointer, so another revision there is one a write activated between the +// two reads, perhaps one no node opens. state holds only what earlier +// reconciles changed, so any read of the fleet this attempt makes followed +// it. A fleet that cannot be read, or answers without its activation, settles +// nothing by the pointer and marks nothing. func (run *configRun) settleRead( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, ) (secretsync.State, bool) { - if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { - return state, false + next := state + if active != nil { + next = next.Found(active.RevisionID) + } + if len(next.Withdrawn()) == 0 && len(next.Written()) == 0 { + return next, !next.SameMarks(state) } fleet, err := run.fleetOf(ctx, eng) if err != nil || fleet.Activation == nil { - return state, false + return next, !next.SameMarks(state) } - next := state if active != nil && fleet.Activation.RevisionID == active.RevisionID { next = next.Ran(fleet.Activation.Epoch) } @@ -808,24 +904,26 @@ func (run *configRun) settleRead( // by the next read of the engine (settleRead) rather than owed for good. A // fleet that cannot be read, or answers without its activation, leaves them // unread, for an activation the operator sees confirmed to settle; a save -// that fails is the caller's error, and no activation is asked for. +// that fails is the caller's error, and no activation is asked for. changed +// says the caller changed state since it was saved, so that it is saved +// whether or not a change of the store was marked. func (run *configRun) markChanges( - ctx context.Context, eng *engine.Client, state secretsync.State, ref types.NamespacedName, + ctx context.Context, eng *engine.Client, state secretsync.State, ref types.NamespacedName, changed bool, ) (secretsync.State, error) { - if !state.Unobserved() { - return state, nil + if state.Unobserved() { + fleet, err := eng.Fleet(ctx) + if err != nil || fleet.Activation == nil { + logf.FromContext(ctx).V(1).Info("Failed to read the fleet's activation pointer after a change of the "+ + "engine's secret store; the change waits for an activation the operator sees confirmed", + "error", fmt.Sprint(err)) + } else { + state, changed = state.Observed(fleet.Activation.Epoch), true + } } - fleet, err := eng.Fleet(ctx) - if err != nil || fleet.Activation == nil { - logf.FromContext(ctx).V(1).Info("Failed to read the fleet's activation pointer after a change of the engine's "+ - "secret store; the change waits for an activation the operator sees confirmed", "error", fmt.Sprint(err)) + if !changed { return state, nil } - state = state.Observed(fleet.Activation.Epoch) - if err := secretsync.SaveState(ctx, run.Client, ref, run.company, state); err != nil { - return state, err - } - return state, nil + return state, secretsync.SaveState(ctx, run.Client, ref, run.company, state) } // sync pushes the desired values into the engine's secret store, deletes @@ -945,11 +1043,12 @@ func diffWith(doc *render.Document, active *engine.ActiveConfig) ([]render.Path, } // keepInSync is enforce for an engine that runs the rendered document with -// every value it references: it records the revision (inSync), makes the -// activation the secret sync's state owes (settle), and comes back at the -// resync, or sooner where that activation failed and its report says so, or -// a value is kept for a revision other than the fleet's (heldResync). reload -// and reloadAction are the sync's, as settle takes them. +// every value it references, whose appliedHash is hash: it records the +// revision (inSync), makes the activation the secret sync's state owes +// (settle), and comes back at the resync, or sooner where that activation +// failed and its report says so, or a value is kept for a revision other than +// the fleet's (heldResync). reload and reloadAction are the sync's, as settle +// takes them. func (run *configRun) keepInSync( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, @@ -980,10 +1079,12 @@ func (run *configRun) keepInSync( // start. hash is what it runs (appliedHash), which the record takes: the one // recorded may be another, the operator's write of the same document whose // record the reconcile that made it never saved being taken for in sync too. +// The overtaken writes go (status.config.overtaken): the operator records a +// revision of its own document, which none of them can say more about. func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string) error { - recorded := run.company.Status.Config + recorded := run.recordedConfig() config := recorded.DeepCopy() - config.AppliedHash = hash + config.AppliedHash, config.Overtaken = hash, nil if recorded.RevisionID != active.RevisionID { // AT THE EPOCH IT ACTIVATED AS, which only the fleet's activation // says: the one recorded is the revision it replaces, so the @@ -1016,11 +1117,15 @@ func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *en run.recordPlacement(*run.rendered, false) // The reason of the last write (Applied, DriftCorrected) stays until // the next one: it is what that write did. - if current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied); current != nil && - current.Status == metav1.ConditionTrue { + current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied) + switch { + case run.confirmed != "" && run.configApplied != nil: + // That write is one this reconcile has just found running, which + // the engine could not confirm, and reported (confirmUnconfirmed). + case current != nil && current.Status == metav1.ConditionTrue: kept := *current run.configApplied = &kept - } else { + default: run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonApplied, Message: fmt.Sprintf("The engine runs the rendered document as revision %s.", active.RevisionID), @@ -1058,12 +1163,8 @@ func (run *configRun) recordApply( "Seeded into revision %s; under the Company's Seed policy the engine owns the document since, and changes to this object are not applied.", applied.RevisionID)} case drifted: - condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected - condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ - "revision %s replaced it.", active.RevisionID, placesOf(diffs), applied.RevisionID) - run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", - "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ - "under the Enforce policy revision %s replaced it.", active.RevisionID, placesOf(diffs), applied.RevisionID) + run.reportReplaced(&condition, applied.RevisionID, active.RevisionID, crewletaiv1alpha1.ReplacedDrifted, + placesOf(diffs)) } run.configApplied = &condition // Activated, not applied: the engine applies it asynchronously, and @@ -1076,6 +1177,27 @@ func (run *configRun) recordApply( } } +// reportReplaced says on condition, ConfigApplied for revision written, and in +// a Warning event what written replaced where that was more than the +// operator's own document: base, which was replaced (a Drifted revision the +// operator did not write, which differed from the rendered document at +// driftedAt). recordApply reports a write the engine confirmed with it, and +// confirmUnconfirmed one it could not confirm, once a read shows it runs, so +// both say the same. +func (run *configRun) reportReplaced( + condition *metav1.Condition, written, base string, replaced crewletaiv1alpha1.ReplacedRevision, driftedAt string, +) { + if replaced != crewletaiv1alpha1.ReplacedDrifted { + return + } + condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected + condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ + "revision %s replaced it.", base, driftedAt, written) + run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", + "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ + "under the Enforce policy revision %s replaced it.", base, driftedAt, written) +} + // observe compares the rendered document with the engine's and reports the // difference, writing nothing: no revision, and no Secret value. func (run *configRun) observe(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) (ctrl.Result, error) { @@ -1156,7 +1278,23 @@ func (run *configRun) seeded(active *engine.ActiveConfig) error { } run.recordPlacement(running, false) run.runningRevision = active.RevisionID - recorded := run.company.Status.Config.RevisionID + prior := run.recordedConfig() + recorded := prior.RevisionID + switch seed := run.overtakenSeed(); { + case recorded == "" && seed != nil: + // Seeded by a write the engine could not confirm, and replaced + // since: recorded as the seed, as a confirmed seed is. + config := prior.DeepCopy() + config.RevisionID, config.AppliedHash, config.AppliedAt = seed.RevisionID, seed.AppliedHash, seed.WrittenAt.DeepCopy() + config.Objects, config.Overtaken = seed.Objects, nil + run.config, recorded = config, seed.RevisionID + case recorded != "" && len(prior.Overtaken) > 0: + // The engine owns the document since the seed, and the operator + // writes nothing more that one of them could say anything about. + config := prior.DeepCopy() + config.Overtaken = nil + run.config = config + } message := fmt.Sprintf("The engine had a company document (revision %s) before the operator seeded one; "+ "under the Company's Seed policy the operator leaves it to the engine.", active.RevisionID) if recorded != "" { diff --git a/internal/controller/config_unconfirmed.go b/internal/controller/config_unconfirmed.go new file mode 100644 index 00000000..4552b486 --- /dev/null +++ b/internal/controller/config_unconfirmed.go @@ -0,0 +1,364 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// An activation the engine could not confirm. +// +// A write of the company document stores a revision, then asks the fleet to +// activate it by a compare-and-set on its activation pointer, conditioned on +// the revision the write was built on; so does a reload, which activates the +// running revision again, conditioned on that one. When the answer to the +// compare-and-set is lost, the engine answers 503 naming the revision it +// stored (engine.UnconfirmedActivation): the pointer may name it or may not. +// Nothing the engine says afterwards says which write a revision was, so +// without a record of it a write that did land read on the next reconcile as +// a revision somebody else activated: under Enforce, where its document is +// the rendered one, a revision the operator adopts as one it did not write, +// and where the render had moved on since, an edit somebody else made, which +// the next write reported as drift it corrected; under Seed, a document the +// engine had before the operator seeded one. +// +// So each such write is recorded (status.config.unconfirmed) with what a +// confirmed write records, and settled by the next read of the revision the +// engine runs, before anything is written. The engine runs one of them: that +// write landed, at the epoch the fleet's activation names, and it is recorded +// as the operator's, as a confirmed write is. The engine runs the revision +// one was built on: that one has not landed yet, and still may, so it stays +// and the write is sent again on the same condition, which the engine refuses +// once one of them lands (and the reconcile then reads it, and records it). +// Every other one can no longer land, since it was conditioned on a revision +// the fleet has moved past, and goes; but for one written into an engine that +// ran no document, which under Seed is the seed whatever revision runs now, +// since the engine stored it while it had none, so either it landed and was +// replaced since, or another writer's create reached the engine first, which +// nothing the engine says tells apart (status.config.overtaken, overtakenSeed). +// It stays recorded until a reconcile decides with it, since one that fails +// before it gets that far must not lose it. +// +// What the write's activation read of the secret store is settled two ways. +// The changes it read were marked with the fleet's activation epoch before it +// was sent (markChanges), and the next read of a revision the engine runs +// above that epoch settles them, whoever activated it (settleRead). A change +// no read of the pointer followed, which it cannot have where the pointer was +// unreadable, and that is when the engine is likeliest to lose an answer, +// awaits the revision the engine stored instead (secretsync.State.Awaiting, +// saved as the answer comes back), and finding that revision running settles +// it (secretsync.State.Found). What a write needs beside that is how its +// document reads every name, which the state records under the write's +// appliedHash before it is sent (secretsync.State.Pending), since nothing +// else says it once the render has moved on; the revision found running a +// write of the operator's records that reading (settlePending). A reconcile +// whose document is held settles the state the same way, and leaves +// status.config, and the report of the write, to the reconcile that applies +// one: its conditions say why it holds. + +// maxUnconfirmed bounds status.config.unconfirmed and status.config.overtaken +// (their MaxItems): writes whose answers are all lost come ten seconds apart +// while the engine cannot reach its coordination store, and an activation +// that has neither landed nor been refused sixteen writes later is not one a +// later read will see land. +const maxUnconfirmed = 16 + +// recordedConfig is status.config as this reconcile leaves it: what it decided +// to write over the Company's record, where it decided anything. +func (run *configRun) recordedConfig() crewletaiv1alpha1.ConfigStatus { + if run.config != nil { + return *run.config + } + return run.company.Status.Config +} + +// boundedWrites is writes without the oldest past maxUnconfirmed. +func boundedWrites(writes []crewletaiv1alpha1.UnconfirmedRevision) []crewletaiv1alpha1.UnconfirmedRevision { + if extra := len(writes) - maxUnconfirmed; extra > 0 { + return writes[extra:] + } + return writes +} + +// unconfirmedIndex is the index in writes of the write of revision, -1 for +// none. +func unconfirmedIndex(writes []crewletaiv1alpha1.UnconfirmedRevision, revision string) int { + if revision == "" { + return -1 + } + return slices.IndexFunc(writes, func(write crewletaiv1alpha1.UnconfirmedRevision) bool { + return write.RevisionID == revision + }) +} + +// ownWrite is the record that makes active, the revision the engine runs as +// this reconcile read it, a write of the operator's, as +// status.config.unconfirmed holds one: status.config where it records that +// revision, or a write the engine could not confirm activating that is it +// and runs its document (runsItsDocument). ok is false for a revision the +// operator did not write, for one it recorded without the hash of what it +// runs, and for none. +func (run *configRun) ownWrite(active *engine.ActiveConfig) (write crewletaiv1alpha1.UnconfirmedRevision, ok bool) { + if active == nil { + return crewletaiv1alpha1.UnconfirmedRevision{}, false + } + recorded := run.recordedConfig() + if recorded.RevisionID == active.RevisionID && recorded.AppliedHash != "" { + return crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: active.RevisionID, AppliedHash: recorded.AppliedHash, Objects: recorded.Objects, + }, true + } + if i := unconfirmedIndex(recorded.Unconfirmed, active.RevisionID); i >= 0 && + runsItsDocument(&recorded.Unconfirmed[i], active) { + return recorded.Unconfirmed[i], true + } + return crewletaiv1alpha1.UnconfirmedRevision{}, false +} + +// runsItsDocument reports whether active, the revision write records, runs +// the document write was recorded with: a reload's (documentSHA256), since +// the engine activates again whichever revision is active when the request +// lands, so the revision it stored runs somebody else's document where they +// activated one in between. A write of the rendered document runs what the +// engine stored for it, and its record names no document. +func runsItsDocument(write *crewletaiv1alpha1.UnconfirmedRevision, active *engine.ActiveConfig) bool { + return write.DocumentSHA256 == "" || active != nil && write.DocumentSHA256 == documentDigest(active.Document) +} + +// documentDigest is documentSHA256 of document: its SHA-256, in hex. +func documentDigest(document []byte) string { + sum := sha256.Sum256(document) + return hex.EncodeToString(sum[:]) +} + +// unconfirmed records write, whose activation the engine could not confirm, +// as revision stored, reports it on ConfigApplied, and returns when the next +// reconcile settles it (settleUnconfirmed): as soon as the engine's +// Retry-After says, the scale of one batch on its coordination store, and ten +// seconds on where it sends none (engineRetry). +func (run *configRun) unconfirmed(stored string, write writeAttempt, err error) time.Duration { + retry := run.engineRetry(err) + recorded := run.recordedConfig() + config := recorded.DeepCopy() + now := metav1.NewTime(run.clock()) + config.Unconfirmed = boundedWrites(append(config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: stored, BaseRevisionID: write.base, AppliedHash: write.hash, WrittenAt: &now, + Objects: run.objectCounts(), Replaced: write.replaced, DriftedAt: write.driftedAt, + })) + run.config = config + run.configApplied = &metav1.Condition{ + Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, + Message: ended(fmt.Sprintf("The engine stored the rendered document as revision %s and could not confirm that it "+ + "activated it, so it may run it or may not. The operator reads which revision the engine runs before it writes "+ + "again, in %s: %s", stored, retry, err)), + } + return retry +} + +// reloadUnconfirmed records stored, the revision a request to activate active +// again stored and could not confirm activating, as a write of the operator's +// of active's document, where active is the operator's (ownWrite): it was +// conditioned on active, and may land yet, and once a read finds it running it +// is recorded and reported as that write would be (settleUnconfirmed), rather +// than as a revision somebody else activated. A revision the operator did not +// write stays somebody else's when activated again. +// +// It is recorded with the digest of active's document (documentSHA256), since +// the request names no revision: where somebody activated another revision +// before it landed, the engine stored that one's document, which a read then +// finds running under stored, and that revision is not the operator's +// (runsItsDocument). And it replaced nothing of its own: its base is the +// operator's write, and what that write replaced, where it is recorded here +// too, is reported once, whichever of the two is confirmed +// (confirmUnconfirmed); copied onto this record, it named the operator's own +// write as one somebody else made, or as one stored without a seal. +func (run *configRun) reloadUnconfirmed(active *engine.ActiveConfig, stored string) { + own, ok := run.ownWrite(active) + recorded := run.recordedConfig() + if !ok || unconfirmedIndex(recorded.Unconfirmed, stored) >= 0 { + return + } + config := recorded.DeepCopy() + now := metav1.NewTime(run.clock()) + own.RevisionID, own.BaseRevisionID, own.WrittenAt = stored, active.RevisionID, &now + own.Replaced, own.DriftedAt, own.DocumentSHA256 = "", "", documentDigest(active.Document) + config.Unconfirmed = boundedWrites(append(config.Unconfirmed, own)) + run.config = config +} + +// overtakenBy splits writes, of status.config.unconfirmed, by runs, a revision +// none of them is that the engine runs: still are the ones built on runs, +// which may land yet, and overtaken the others written into an engine that ran +// no document, which Seed takes for the seed (overtakenSeed). Every other one +// can no longer land, and nothing reads it again. +func overtakenBy( + writes []crewletaiv1alpha1.UnconfirmedRevision, runs string, +) (still, overtaken []crewletaiv1alpha1.UnconfirmedRevision) { + for i := range writes { + switch writes[i].BaseRevisionID { + case runs: + still = append(still, writes[i]) + case "": + overtaken = append(overtaken, writes[i]) + } + } + return still, overtaken +} + +// settleUnconfirmed settles the writes status.config.unconfirmed records +// against active, the revision the engine runs as this reconcile read it. One +// of them is active, with the document it was recorded with +// (runsItsDocument): it is recorded as the operator's write +// (confirmUnconfirmed). Otherwise the ones built on active stay, and every +// other one goes, to status.config.overtaken where it was written into an +// engine that ran no document; a reload found running another document was +// built on a revision somebody else activated, and goes too. Under Observe +// nothing is settled: the operator writes nothing there, and Observe records +// the revision it finds in sync. +func (run *configRun) settleUnconfirmed(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) error { + recorded := run.recordedConfig() + if len(recorded.Unconfirmed) == 0 || run.policy() == crewletaiv1alpha1.ManagementPolicyObserve { + return nil + } + runs := "" + if active != nil { + runs = active.RevisionID + } + if i := unconfirmedIndex(recorded.Unconfirmed, runs); i >= 0 && runsItsDocument(&recorded.Unconfirmed[i], active) { + return run.confirmUnconfirmed(ctx, eng, &recorded, i) + } + still, overtaken := overtakenBy(recorded.Unconfirmed, runs) + if len(still) < len(recorded.Unconfirmed) { + config := recorded.DeepCopy() + config.Unconfirmed = still + config.Overtaken = boundedWrites(append(config.Overtaken, overtaken...)) + run.config = config + } + return nil +} + +// confirmUnconfirmed records recorded.Unconfirmed[i], a revision the engine +// runs, as the operator's write, and reports it, as recordApply records and +// reports one the engine confirmed: at the epoch the fleet's activation names, +// which only a read of it gives, since the engine answered none; from now, +// since nothing says when the activation landed, and the engine is held to +// applying its epoch applyDeadline after the operator found it running, as an +// adopted revision is; and saying what it replaced (reportReplaced), which the +// record kept, or where it is a reload of a write recorded here too, what +// that write replaced, which only it records (reloadUnconfirmed), and which +// then goes with it: the revision it activated again landed, since the reload +// runs. A write built on it stays unconfirmed, and every other one is +// overtaken by it, and so is every write status.config.overtaken keeps: none +// can land now, nor be the seed, since a write of the operator's runs. +// +// It fails where the fleet cannot be read, or answers without its activation, +// and with an *activationAhead, a lost race, where the activation names +// another revision: converge reads again from the start, and waits for the +// node it reads where the second read finds the same. +func (run *configRun) confirmUnconfirmed( + ctx context.Context, eng *engine.Client, recorded *crewletaiv1alpha1.ConfigStatus, i int, +) error { + write := recorded.Unconfirmed[i] + fleet, err := run.fleetOf(ctx, eng) + if err != nil { + return fmt.Errorf("the engine runs revision %s, which the operator wrote and the engine could not confirm "+ + "activating, and its fleet, which says as what epoch, could not be read: %w", write.RevisionID, err) + } + switch { + case fleet.Activation == nil: + // The read of the document named the revision, so it is active: + // the engine answered its fleet without reading the pointer. + return fmt.Errorf("the engine runs revision %s, which the operator wrote and the engine could not confirm "+ + "activating, and its fleet answered without its activation, which says as what epoch", write.RevisionID) + case fleet.Activation.RevisionID != write.RevisionID: + return &activationAhead{read: write.RevisionID, activation: *fleet.Activation, unconfirmed: true} + } + epoch := fleet.Activation.Epoch + // The write a reload activated again, where that is recorded here too, + // and so on down: the first of them says what the rendered document + // replaced. + first, landed := write, []string{write.RevisionID} + for { + j := unconfirmedIndex(recorded.Unconfirmed, first.BaseRevisionID) + if j < 0 || slices.Contains(landed, recorded.Unconfirmed[j].RevisionID) { + break + } + first = recorded.Unconfirmed[j] + landed = append(landed, first.RevisionID) + } + still, _ := overtakenBy(slices.DeleteFunc(slices.Clone(recorded.Unconfirmed), + func(w crewletaiv1alpha1.UnconfirmedRevision) bool { return slices.Contains(landed, w.RevisionID) }), + write.RevisionID) + found := metav1.NewTime(run.clock()) + run.confirmed = write.RevisionID + run.config = &crewletaiv1alpha1.ConfigStatus{ + RevisionID: write.RevisionID, Epoch: new(epoch), AppliedHash: write.AppliedHash, AppliedAt: &found, + Objects: write.Objects, Unconfirmed: still, + } + condition := metav1.Condition{ + Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonApplied, + Message: fmt.Sprintf("Applied the rendered document as revision %s.", write.RevisionID), + } + run.reportReplaced(&condition, write.RevisionID, first.BaseRevisionID, first.Replaced, first.DriftedAt) + run.configApplied = &condition + run.applied = appliedVerdict{kind: appliedInRevision, revision: write.RevisionID} + run.event(corev1.EventTypeNormal, "Activated", "ApplyConfig", + "The engine runs revision %s as epoch %d: the operator activated it, and the engine could not confirm the "+ + "activation when it was made.", write.RevisionID, epoch) + return nil +} + +// settlePending is state recording the reading of active, the revision the +// engine runs as this reconcile read it, where that is a write of the +// operator's whose reading state does not record (ownWrite): the write's +// activation went unconfirmed, or the save of its record after it was lost, +// and the reading state recorded under the write's appliedHash before it was +// sent says how its document reads every name (secretsync.State.Confirmed). +// The readings of the writes status.config.unconfirmed no longer names go: no +// revision the operator has not recorded can be one of them. A revision state +// already records keeps its reading, and so does one the operator did not +// write, or none, which the caller adopts where its document is the rendered +// one. +func (run *configRun) settlePending(state secretsync.State, active *engine.ActiveConfig) secretsync.State { + if active != nil && !state.Records(active.RevisionID) { + if own, ok := run.ownWrite(active); ok { + if confirmed, ok := state.Confirmed(active.RevisionID, own.AppliedHash); ok { + state = confirmed + } + } + } + recorded := run.recordedConfig() + keep := make([]string, len(recorded.Unconfirmed)) + for i := range recorded.Unconfirmed { + keep[i] = recorded.Unconfirmed[i].AppliedHash + } + return state.KeepPending(keep) +} + +// overtakenSeed is the write into an engine with no document, of +// status.config.overtaken, that seeded the engine under Seed, nil for none: the +// engine had no document when it was written, and runs one now, so that write +// landed, and the revision that runs replaced it, or another write reached +// the engine first in the moment between the reconcile's read and that write, +// which nothing the engine says tells apart, and which the operator takes for +// the first, whatever revision runs now. +func (run *configRun) overtakenSeed() *crewletaiv1alpha1.UnconfirmedRevision { + recorded := run.recordedConfig() + for i := range recorded.Overtaken { + if write := &recorded.Overtaken[i]; write.BaseRevisionID == "" { + return write + } + } + return nil +} diff --git a/internal/controller/config_unconfirmed_test.go b/internal/controller/config_unconfirmed_test.go new file mode 100644 index 00000000..9574f449 --- /dev/null +++ b/internal/controller/config_unconfirmed_test.go @@ -0,0 +1,1426 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/yaml" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretref" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// These tests hold what the operator does with a write whose activation the +// engine sent to its fleet and could not confirm: it never sends the write +// again on top of it, and a write that did land is recorded as its own, under +// Enforce and under Seed alike. The engine gives that answer only when its +// coordination store loses the answer to a compare-and-set, which a real +// engine cannot be made to do at will, so the storeEngine stands in for it. + +// outcome is how the activation of the storeEngine's next write goes. +type outcome int + +const ( + // confirmed: it activates, and the engine says so. + confirmed outcome = iota + // landed: it activates, and the engine's answer is lost. + landed + // pending: the revision is stored and not activated, and the engine's + // answer is lost. + pending +) + +// land activates the inert revision id, as a compare-and-set whose answer was +// lost lands after all. The caller holds e's lock. +func (e *storeEngine) land(id string) { + i := slices.IndexFunc(e.inert, func(r storeRevision) bool { return r.id == id }) + e.revisions = append(e.revisions, e.inert[i]) + e.inert = slices.Delete(e.inert, i, i+1) +} + +// r1Hash is the appliedHash a test records r1 under where r1 is the +// operator's own write. +const r1Hash = "sha256:r1" + +// answerLost is the engine's answer to a write whose activation it sent and +// whose answer its coordination store did not give (crewlet's +// internal/api/configapi, RefuseActivationUnknown): 503 unavailable with the +// unknown outcome, the revision it stored, the node whose history alone holds +// it, the revision the write was built on, and a hint saying how to look +// first, ended as httpjson.Sentence ends it, under the Retry-After of an +// undecided answer (authz.RetryUndecidedSeconds). +func answerLost(w http.ResponseWriter, stored, base string) { + hint := fmt.Sprintf("Revision %[1]s was stored on node %[2]s, and the coordination store did not answer its "+ + "activation, so whether the fleet runs it is unknown. GET /config names the active revision once the store "+ + "answers again. If it is not %[1]s, the revision is in node %[2]s's history alone: send GET "+ + "/config/revisions/%[1]s/diff, and POST /config/revisions/%[1]s/revert to make it the company, to that "+ + "node \u2014 by its own address where a shared one fronts several, since any other node holds no copy and "+ + "answers 404. Sending the write again makes another revision.", stored, storeNode) + body := map[string]string{ + "error": "unavailable", "message": "Can't answer right now. Try again in a moment, or ask another node.", + "outcome": "unknown", "stored_revision_id": stored, "node": storeNode, "hint": hint, + } + if base != "" { + body["your_base"] = base + } + w.Header().Set("Retry-After", "2") + answerJSON(w, http.StatusServiceUnavailable, body) +} + +// unconfirmedWrite is the withdrawal's reconcile under Enforce after the +// operator's write of the rendered document, built on r1, came back +// unconfirmed as revision r2, which how lands or keeps inert; and the Company +// as that reconcile's status write leaves it. A reconcile that fails is +// reported as the reconciler reports it, and retried. +func unconfirmedWrite(t *testing.T, how outcome) (*withdrawal, *crewletaiv1alpha1.Company) { + t.Helper() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + // r1 is the rendered document and lets the registry go, so the + // withdrawal alone is settled by activating r1 again; a rotated value + // is what has the operator write the document, where the engine's own + // re-activation at the sync's write fails. + w.rotate(t) + w.engine.activation = how + operatorActivates(w) + reconcileOnce(t, w) + w.engine.activation, w.engine.reloadAnswer = confirmed, nil + company := w.statusWritten() + return w, company +} + +// operatorActivates makes the engine's own re-activation at a sync's write or +// removal fail, as an engine whose coordination store does not answer it +// does, so that the activation is the operator's: the write or the reload +// whose answer these tests lose. +func operatorActivates(w *withdrawal) { + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") +} + +// reconcileOnce runs w's reconcile up to its status write, as the reconciler +// does: an error it fails with is reported on ConfigApplied. +func reconcileOnce(t *testing.T, w *withdrawal) { + t.Helper() + if _, err := w.run.converge(context.Background(), w.client); err != nil { + t.Logf("the reconcile failed, and is retried: %v", err) + w.run.unavailable(err) + } +} + +// TestAnActivationTheEngineCouldNotConfirmIsRecorded: the write the engine +// could not confirm is recorded with what a confirmed one records, and the +// reconcile asks again soon, reporting it. +func TestAnActivationTheEngineCouldNotConfirmIsRecorded(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || got[0].AppliedHash == "" || + got[0].WrittenAt.IsZero() { + t.Fatalf("status.config.unconfirmed is %+v, want r2, built on r1, with its hash and when", got) + } + c := w.run.configApplied + if c == nil || c.Status != metav1.ConditionFalse || c.Reason != crewletaiv1alpha1.ReasonEngineUnavailable || + !strings.Contains(c.Message, "revision r2 and could not confirm") { + t.Errorf("ConfigApplied is %+v, want False, EngineUnavailable, saying r2 could not be confirmed", c) + } +} + +// TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice: the write did +// land, so the next reconcile finds the engine running it, records it as the +// operator's at the epoch the fleet's activation names, and writes nothing; +// and its activation is recorded as a confirmed write's is, as the one that +// read the store without the withdrawal's registry, which the sync before it +// deleted, so nothing activates r2 again for it either. The reading recorded +// for it goes, nothing unconfirmed naming it any more. +func TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) { + t.Fatalf("the write's sync withdrew %v, want the registry", state.Withdrawn()) + } + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone: the engine ran it", written) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 activated without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !state.Records("r2") || + (len(claims) == 1 && state.Pends(claims[0].AppliedHash)) { + t.Errorf("the state owes %v and records %q, want nothing owed, r2's reading, and no reading left pending, "+ + "since nothing unconfirmed names it", state.Withdrawn(), state.Recorded()) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.AppliedHash != claims[0].AppliedHash || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r2's write recorded, with its hash, and nothing unconfirmed", config) + } + recorded := drained(w.events) + if i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated ") }); i < 0 || + !strings.Contains(recorded[i], "revision r2 as epoch 2") { + t.Errorf("events %q, want an Activated event saying the engine runs r2 as epoch 2", recorded) + } +} + +// TestAnActivationThatLandsLateIsTheOperatorsOwn: the write had not landed by +// the reconcile that wrote it, and lands before the next one reads, which +// records it as the operator's write and says so, rather than adopting it as +// a revision of the rendered document somebody else activated. +func TestAnActivationThatLandsLateIsTheOperatorsOwn(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + written, _ := w.engine.writes() + if config := w.statusWritten().Status.Config; !slices.Equal(written, []string{"r2"}) || len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed, want r2 alone, recorded", written, config.Unconfirmed) + } + if recorded := drained(w.events); !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) { + t.Errorf("events %q, want an Activated event saying the engine runs r2, the operator's write, as epoch 2", recorded) + } +} + +// TestAnActivationThatDidNotLandIsSentAgainOnItsCondition: the engine still +// runs r1, so the write is sent again on r1, and lands; and where the first +// one lands just ahead of it, the engine refuses it, and the reconcile finds +// the first one running and records it rather than writing a third. +func TestAnActivationThatDidNotLandIsSentAgainOnItsCondition(t *testing.T) { + t.Run("it never lands", func(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.again(t, company) + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) || len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed; want r3 written again on r1, and nothing "+ + "unconfirmed", written, config.Unconfirmed) + } + }) + t.Run("it lands ahead of the second", func(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.beforeWrite = func(e *storeEngine) { e.land("r2") } + w.again(t, company) + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + if written, refused := w.engine.writes(); !slices.Equal(written, []string{"r2"}) || refused != 1 || + len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v, %d refused, and keeps %+v unconfirmed; want the second write refused "+ + "and r2 recorded", written, refused, config.Unconfirmed) + } + if recorded := drained(w.events); !told(recorded, "Activated") { + t.Errorf("events %q, want an Activated event for r2", recorded) + } + }) +} + +// TestAnUnconfirmedActivationIsNeverWrittenOverUnread: the engine runs the +// unconfirmed revision and its fleet cannot be read, so the operator cannot +// tell at what epoch: it writes nothing, keeps the record, and says so. +func TestAnUnconfirmedActivationIsNeverWrittenOverUnread(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.fleetDown = true + w.again(t, company) + if _, err := w.run.converge(context.Background(), w.client); err == nil { + t.Fatal("the reconcile succeeded with no read of the fleet's activation") + } + kept := w.statusWritten().Status.Config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) || len(kept) != 1 { + t.Errorf("the operator wrote %v and keeps %+v, want r2 alone, still unconfirmed", written, kept) + } +} + +// TestAnUnconfirmedActivationOtherwiseOvertakenIsNotSentAgain: somebody +// activated r3 since, so the write, conditioned on r1, can never land: it +// goes rather than being sent again, and not to status.config.overtaken, +// which keeps only the writes into an engine that ran no document; and the +// next write, r4 on r3, whose answer is lost too, is recorded unconfirmed. +func TestAnUnconfirmedActivationOtherwiseOvertakenIsNotSentAgain(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.mu.Lock() + w.engine.activateOther(w.requiring) + w.engine.mu.Unlock() + w.again(t, company) + w.engine.activation = pending + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + got := config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r4"}) || len(got) != 1 || + got[0].RevisionID != "r4" || got[0].BaseRevisionID != "r3" { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed, want r2 not sent again and r4, built on r3, kept", + written, got) + } + if over := config.Overtaken; len(over) != 0 { + t.Errorf("status.config.overtaken is %+v, want nothing: r2 was built on r1, and no Seed reads it", over) + } +} + +// TestALandedWriteActivatedAgainStaysInSyncThroughAReconcileThatFails: the +// write landed and was activated again (r3) before the next reconcile read it, +// and that reconcile failed before it decided anything with it: its save of +// the secret sync's state did not go through. Nothing records the write any +// more, r3 having overtaken it, and nothing needs to: the withdrawal its sync +// made and the values it wrote were marked with the fleet's epoch before it +// was sent, and r3 activated above it, so the next reconcile finds r3 in sync, +// records it at its own epoch, and owes nothing for it. Kept owed, the +// withdrawal had the operator activate r3 again, and the values had it write +// the same document again as r4. +func TestALandedWriteActivatedAgainStaysInSyncThroughAReconcileThatFails(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by the engine"); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 0 || len(config.Overtaken) != 0 { + t.Fatalf("status.config after a reconcile that failed is %+v, want r2 gone: r3 overtook it", config) + } + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry: the reconcile that failed saved nothing", state.Withdrawn()) + } + + w.failWrite.Store(0) + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r1 to r3: the operator wrote, or activated, what already ran", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || config.Epoch == nil || + *config.Epoch != 3 { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash, the values it was written with", config) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || len(state.Written()) != 0 { + t.Errorf("the state owes withdrawals %v and writes %v, want nothing: r3 activated after both", state.Withdrawn(), + state.Written()) + } +} + +// TestAHeldDocumentSettlesAnUnconfirmedWriteThatRuns: the first reconcile +// after the write holds its document, and the engine runs the write, which +// landed, or a reload of it. Either read the store without the withdrawal's +// registry, which the sync before the write deleted, so the held reconcile +// settles it and activates nothing again for it. Settled only by a reconcile +// that applies a document, it was owed there, and the held reconcile activated +// the running revision again: an epoch every node applied for nothing. The +// record of the write, and the report, are left to the reconcile that applies +// a document. +func TestAHeldDocumentSettlesAnUnconfirmedWriteThatRuns(t *testing.T) { + for _, reloaded := range []bool{false, true} { + t.Run(map[bool]string{false: "the write runs", true: "a reload of the write runs"}[reloaded], func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + want := []string{"r1", "r2"} + if reloaded { + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + want = append(want, "r3") + } + w.again(t, company) + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + if ids := w.engine.ids(); !slices.Equal(ids, want) { + t.Errorf("the engine's revisions are %v, want %v: what runs read the store without the registry", ids, want) + } + if recorded := drained(w.events); told(recorded, "Reactivated") { + t.Errorf("events %q, want no Reactivated: nothing was owed", recorded) + } + state := w.state(t) + if len(state.Withdrawn()) != 0 || (!reloaded && !state.Records("r2")) { + t.Errorf("the state owes %v and records %q, want nothing owed, and r2's reading where r2 runs", + state.Withdrawn(), state.Recorded()) + } + if w.run.config != nil || w.run.configApplied != nil { + t.Errorf("the held reconcile recorded %+v and reported %+v, want both left to the one that applies a "+ + "document", w.run.config, w.run.configApplied) + } + }) + } +} + +// TestAConfirmationThatLosesItsRaceIsReadAgain: the engine runs the write as +// this reconcile reads it, and by the read of its fleet, which says as what +// epoch, a reload has activated after it. That is a lost race, read again +// from the start, which finds the reload running the write's document with its +// values: in sync, and recorded at its own epoch, with nothing written. A fleet +// answering without its activation is no race, and the reconcile fails over +// it without claiming one, and records nothing. +func TestAConfirmationThatLosesItsRaceIsReadAgain(t *testing.T) { + t.Run("another revision activated", func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { e.activateOther(e.active().document) } + w.engine.mu.Unlock() + w.again(t, company) + if _, err := w.run.converge(context.Background(), w.client); err != nil { + t.Fatalf("converge() = %v, want the reconcile to read the engine again and find it in sync", err) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r1 to r3: nothing written", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.Epoch == nil || *config.Epoch != 3 || + config.AppliedHash != claims[0].AppliedHash { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash", config) + } + }) + t.Run("the fleet answers without its activation", func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.mu.Lock() + w.engine.noActivation = true + w.engine.mu.Unlock() + w.again(t, company) + _, err := w.run.converge(context.Background(), w.client) + if err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || strings.Contains(err.Error(), "instead") { + t.Errorf("converge() = %v, want an error that claims no race", err) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone", written) + } + if got := w.statusWritten().Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r2" { + t.Errorf("status.config.unconfirmed is %+v, want r2 kept", got) + } + }) +} + +// TestAConfirmedWriteIsReportedAppliedOnceTheEngineAppliesIt: the write's own +// reconcile reported it as one the engine could not confirm, and the next +// finds it running and its epoch applied: an Applied event names it, as one +// does for a write the engine confirmed, although the stored ConfigApplied is +// neither Applying nor ApplyFailed. +func TestAConfirmedWriteIsReportedAppliedOnceTheEngineAppliesIt(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + meta.SetStatusCondition(&company.Status.Conditions, *w.run.configApplied) + w.again(t, company) + reconcileOnce(t, w) + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + var applied []string + for _, e := range drained(w.events) { + if strings.HasPrefix(e, "Normal "+crewletaiv1alpha1.ReasonApplied+" ") { + applied = append(applied, e) + } + } + if len(applied) != 1 || !strings.Contains(applied[0], "revision r2 (epoch 2)") { + t.Errorf("Applied events %q, want one naming revision r2 at epoch 2", applied) + } +} + +// TestAValueWrittenBackIsSettledByTheWriteThatCarriedIt: the store lost the +// operator's value, so the sync wrote it back and the document was written +// again, as r3, whose activation the engine could not confirm. The engine runs +// r3, or a reload of it, either of which read the store with the value, so the +// next reconcile owes nothing for it and writes nothing. Kept owed, the value +// had the operator write the document once more. +func TestAValueWrittenBackIsSettledByTheWriteThatCarriedIt(t *testing.T) { + for _, reloaded := range []bool{false, true} { + t.Run(map[bool]string{false: "the write runs", true: "a reload of the write runs"}[reloaded], func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(w.state(t).Written(), []string{tavily}) { + t.Fatalf("the operator wrote %v and the state owes %v; want r3 written for the value written back, "+ + "and the value owed", written, w.state(t).Written()) + } + want := []string{"r1", "r2", "r3"} + if reloaded { + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + want = append(want, "r4") + } + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, want) { + t.Errorf("the engine's revisions are %v, want %v: what runs read the value written back", ids, want) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing", got) + } + }) + } +} + +// TestASeedTheEngineCouldNotConfirmIsReportedAsTheSeed: under Seed, a write +// into an engine with no document that landed unconfirmed is the seed, not a +// document the engine had before the operator seeded one. +func TestASeedTheEngineCouldNotConfirmIsReportedAsTheSeed(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = crewletaiv1alpha1.ManagementPolicySeed + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the seed", written) + } + seed := written[0] + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != seed || got[0].BaseRevisionID != "" { + t.Errorf("status.config.unconfirmed is %+v, want the seed %s, built on nothing", got, seed) + } + w.engine.activation = confirmed + w.again(t, company) + reconcileOnce(t, w) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonSeeded || + !strings.Contains(c.Message, "seeded with the rendered document as revision "+seed) { + t.Errorf("ConfigApplied is %+v, want Seeded, naming %s as the seed", c, seed) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{seed}) { + t.Errorf("the operator wrote %v, want the seed alone", written) + } +} + +// TestALandedWriteActivatedAgainIsNotWrittenAgain: the write landed, and was +// activated again before the next reconcile read it, by the engine's own +// reload after it minted a credential or by a person's crewlet config reload, +// so the engine runs the rendered document as r3, with the values r2 was +// written with. Dropped as a write that can no longer land, it left +// status.config's hash the one before it, and the operator wrote the same +// document a third time. It is in sync instead: r3 is recorded at its own +// epoch with r2's hash, and the withdrawal r2's sync made, which r3 activated +// after, is settled, so nothing activates it again either. +func TestALandedWriteActivatedAgainIsNotWrittenAgain(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Fatalf("the engine's revisions are %v, want r1 to r3: the operator wrote, or activated, what already ran", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || + config.Epoch == nil || *config.Epoch != 3 || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash, and nothing unconfirmed", config) + } + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue { + t.Errorf("ConfigApplied is %+v, want True: the engine runs the rendered document", c) + } + if state := w.state(t); len(state.Withdrawn()) != 0 { + t.Errorf("the state owes %v, want nothing: r3 activated after the registry was deleted", state.Withdrawn()) + } +} + +// TestASeedActivatedAgainIsReportedAsTheSeed: under Seed, the write into an +// engine with no document landed unconfirmed and was activated again before +// the next reconcile read it, so the engine runs another revision. It is +// still the seed, and the engine runs the revision that replaced it, not a +// document it had before the operator seeded one. +func TestASeedActivatedAgainIsReportedAsTheSeed(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = crewletaiv1alpha1.ManagementPolicySeed + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the seed", written) + } + seed := written[0] + reloaded, err := w.client.ReloadConfig(context.Background(), "reloaded by the engine") + if err != nil { + t.Fatal(err) + } + company = w.statusWritten() + w.again(t, company) + reconcileOnce(t, w) + want := fmt.Sprintf("seeded with the rendered document as revision %s and owns the document since; it runs revision %s", + seed, reloaded.RevisionID) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonSeeded || !strings.Contains(c.Message, want) { + t.Errorf("ConfigApplied is %+v, want Seeded, saying %q", c, want) + } + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != seed || len(config.Unconfirmed) != 0 || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want the seed %s recorded, and nothing unconfirmed or overtaken", config, seed) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{seed}) { + t.Errorf("the operator wrote %v, want the seed alone", written) + } + + // The engine owns the document since the seed: a write still recorded + // as overtaken, as a status written before the seed was recorded may + // hold, says nothing more, and goes. + company.Status.Config.Overtaken = []crewletaiv1alpha1.UnconfirmedRevision{{ + RevisionID: "r9", AppliedHash: "sha256:other", WrittenAt: &metav1.Time{}, + }} + w.again(t, company) + reconcileOnce(t, w) + if config := w.statusWritten().Status.Config; config.RevisionID != seed || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want the seed %s recorded, and nothing overtaken", config, seed) + } +} + +// TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed: a write the operator +// built on r2 in the moment r2 landed, before any read showed it running, may +// land yet once r2 is confirmed: it stays unconfirmed, and is confirmed in +// turn once it runs. Taken for a write r2 overtook, it was adopted as a +// revision the operator had not written once it landed. +func TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.mu.Lock() + w.engine.inert = append(w.engine.inert, storeRevision{id: w.engine.mint(), document: w.engine.active().document}) + w.engine.mu.Unlock() + company.Status.Config.Unconfirmed = append(company.Status.Config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: "r3", BaseRevisionID: "r2", AppliedHash: company.Status.Config.Unconfirmed[0].AppliedHash, + WrittenAt: &metav1.Time{}, + }) + w.again(t, company) + reconcileOnce(t, w) + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != "r2" || len(config.Unconfirmed) != 1 || + config.Unconfirmed[0].RevisionID != "r3" || len(config.Overtaken) != 0 { + t.Fatalf("status.config is %+v, want r2 recorded and r3, built on it, still unconfirmed", config) + } + w.engine.mu.Lock() + w.engine.land("r3") + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 recorded as the operator's, and nothing unconfirmed", config) + } + if recorded := drained(w.events); !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") + }) { + t.Errorf("events %q, want an Activated event saying the engine runs r3, the operator's write, as epoch 3", recorded) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone", written) + } +} + +// TestAWriteAReactivationOvertookIsNotSentAgain: a held reconcile activates +// r1 again (r3), for the withdrawal of the registry, while a write built on it +// (r2) is unconfirmed. r2 can no longer land, and r3 activated after the +// changes r2's sync marked, so it read the values r2 was written with: the +// reload is recorded as the operator's, r2 is left for the next read to drop, +// and the next reconcile finds r3 in sync with the document and writes +// nothing. +func TestAWriteAReactivationOvertookIsNotSentAgain(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + claims := company.Status.Config.Unconfirmed + w.again(t, company) + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r3"}) { + t.Fatalf("the engine's revisions are %v, want r1 activated again as r3", ids) + } + if config := company.Status.Config; len(claims) != 1 || config.RevisionID != "r3" || len(config.Unconfirmed) != 1 { + t.Fatalf("status.config is %+v, want r3 recorded and r2 left for the next read", config) + } + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: r3 read the values r2 was written with", written) + } + config := w.statusWritten().Status.Config + if config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || len(config.Unconfirmed) != 0 || + len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want r3 with r2's hash, and r2 gone", config) + } +} + +// TestAConfirmationTheStateMissedIsSettledLater: the reconcile that found the +// write running recorded it in status.config and could not save the secret +// sync's state, so the next one reads what the state records of the write as +// the operator's, and settles the withdrawal its activation read the store +// without, rather than activating r2 again for it: whether that reconcile +// applies a document or holds it. +func TestAConfirmationTheStateMissedIsSettledLater(t *testing.T) { + for _, held := range []bool{false, true} { + t.Run(map[bool]string{false: "the next document is applied", true: "the next document is held"}[held], func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + company = w.statusWritten() + if company.Status.Config.RevisionID != "r2" || len(company.Status.Config.Unconfirmed) != 0 { + t.Fatalf("status.config is %+v, want r2 recorded as the operator's", company.Status.Config) + } + if state := w.state(t); state.Records("r2") { + t.Fatal("the state records r2's reading, which the failed write was to save") + } + w.failWrite.Store(0) + w.again(t, company) + if held { + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + } else { + reconcileOnce(t, w) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 activated without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !state.Records("r2") { + t.Errorf("the state owes %v and records %q, want nothing owed and r2's reading", state.Withdrawn(), state.Recorded()) + } + }) + } +} + +// TestAConfirmedWriteLeavesTheSeatsToTheRevisionThatRuns: the write a +// reconcile finds running is one an earlier reconcile made, of what it +// rendered then, so the human seats IdentityClaimed names are those of the +// revision the engine runs, here one with a seat ana, and not those of this +// reconcile's render, which the engine refuses. Taken for this reconcile's own +// write, a confirmation named seats the engine does not have. +func TestAConfirmedWriteLeavesTheSeatsToTheRevisionThatRuns(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.mu.Lock() + w.engine.revisions[len(w.engine.revisions)-1].document = []byte("name: Acme\nroles:\n - name: Ana\n handle: ana\n kind: human\n") + w.engine.refusal = invalidDocument + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + if w.run.confirmed != "r2" { + t.Fatalf("the reconcile confirmed %q, want r2", w.run.confirmed) + } + if got := w.run.humanSeats(context.Background()); !slices.Equal(got, []string{seatAna}) { + t.Errorf("humanSeats() = %v, want [%s]: the engine runs r2", got, seatAna) + } +} + +// TestUnconfirmedActivationsAreBounded: a write whose answer is lost every +// time is recorded no more than maxUnconfirmed times, the newest kept, since +// the API server refuses a status that holds more: the generated CRD's +// maxItems, which this holds the constant to. +func TestUnconfirmedActivationsAreBounded(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "config", "crd", "bases", "crewlet.ai_companies.yaml")) + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatal(err) + } + for _, field := range []string{"unconfirmed", "overtaken"} { + var bound *int64 + for _, v := range crd.Spec.Versions { + if v.Name == "v1alpha1" { + config := v.Schema.OpenAPIV3Schema.Properties["status"].Properties["config"] + bound = config.Properties[field].MaxItems + } + } + if bound == nil { + t.Fatalf("the Company CRD does not bound status.config.%s; the operator does, at %d", field, maxUnconfirmed) + } + if *bound != maxUnconfirmed { + t.Fatalf("the Company CRD bounds status.config.%s at %d items, the operator at %d", field, *bound, maxUnconfirmed) + } + } + w, company := unconfirmedWrite(t, pending) + for range maxUnconfirmed { + w.again(t, company) + w.engine.activation = pending + reconcileOnce(t, w) + company = w.statusWritten() + } + written, _ := w.engine.writes() + got := make([]string, 0, len(company.Status.Config.Unconfirmed)) + for _, write := range company.Status.Config.Unconfirmed { + got = append(got, write.RevisionID) + } + if len(written) != maxUnconfirmed+1 || !slices.Equal(got, written[1:]) { + t.Errorf("after %d writes %v, status.config.unconfirmed holds %v; want the newest %d", len(written), written, got, + maxUnconfirmed) + } + + // Writes into an engine that ran no document move to overtaken once the + // engine runs a revision none of them is, beside the ones already there, + // and the oldest go: sixteen older ones here, which a list cut at its + // other end would keep instead. The reconcile fails before it writes, as + // the record is kept through one that does. + company = w.run.company.DeepCopy() + company.Status.Config.Unconfirmed, company.Status.Config.Overtaken = nil, nil + want := make([]string, 0, maxUnconfirmed) + for i := range maxUnconfirmed { + company.Status.Config.Overtaken = append(company.Status.Config.Overtaken, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: fmt.Sprintf("old%d", i), AppliedHash: "sha256:old", WrittenAt: &metav1.Time{}, + }) + seed := fmt.Sprintf("seed%d", i) + company.Status.Config.Unconfirmed = append(company.Status.Config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: seed, AppliedHash: "sha256:seed", WrittenAt: &metav1.Time{}, + }) + want = append(want, seed) + } + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + w.failWrite.Store(0) + got = got[:0] + for _, write := range w.statusWritten().Status.Config.Overtaken { + got = append(got, write.RevisionID) + } + if config := w.statusWritten().Status.Config; !slices.Equal(got, want) || len(config.Unconfirmed) != 0 { + t.Errorf("status.config.overtaken holds %v and unconfirmed %d writes; want the newest %d, %v, and none", + got, len(config.Unconfirmed), maxUnconfirmed, want) + } +} + +// TestADriftCorrectionTheEngineCouldNotConfirmIsReportedOnceItRuns holds a +// write over a revision somebody else activated with other content, whose +// activation the engine could not confirm, to the report a confirmed one gets +// once a read shows the engine runs it: ConfigApplied DriftCorrected naming +// the revision it replaced and where it differed, and a Warning Drifted +// event, kept through the rest of that reconcile. Confirmed from the record +// as it was, it read Applied, then "The engine runs the rendered document", +// and nothing ever said a revision somebody else wrote had gone. +func TestADriftCorrectionTheEngineCouldNotConfirmIsReportedOnceItRuns(t *testing.T) { + w := newWithdrawal(t, runningOther) + operatorActivates(w) + // The operator's last write was r0: r1 is somebody else's. + company := w.run.company.DeepCopy() + company.Status.Config.RevisionID = "r0" + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company = w.statusWritten() + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].Replaced != crewletaiv1alpha1.ReplacedDrifted || + got[0].DriftedAt == "" { + t.Fatalf("status.config.unconfirmed is %+v, want r2, recorded as replacing a Drifted r1, with where", got) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonDrifted) { + t.Errorf("events %q, want no Drifted before the write is seen running: it may never land", recorded) + } + + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone", written) + } + want := fmt.Sprintf("Revision r1, which the operator did not write, differed from the rendered document at %s; "+ + "revision r2 replaced it.", got[0].DriftedAt) + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue || + c.Reason != crewletaiv1alpha1.ReasonDriftCorrected || c.Message != want { + t.Errorf("ConfigApplied is %+v, want True DriftCorrected, %q", c, want) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonDrifted+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "ran revision r1, which the operator did not write") || + !strings.Contains(recorded[i], "revision r2 replaced it") { + t.Errorf("events %q, want a Warning Drifted saying r2 replaced r1", recorded) + } +} + +// TestAHeldReconcileReleasesWhatARunningUnconfirmedWriteLetsGo: r1 requires +// the registry, and the operator's write r2, which lets it go, landed with its +// answer lost. Then the registry's Secret is deleted while the document is +// held by a render that no longer reads it at all, so the held reconcile reads +// nothing missing and owes nothing for r1. It reads the engine all the same +// while a write of the operator's may run: r2 runs, and the reading the state +// recorded before r2 was sent says it lets the registry go, so the value is +// taken out of the store and r2 activated again (r3) for it. Returning before +// the read, it left the deleted Secret's value for r2's tool server to start +// with until the document was applied. The record of r2 moves on to r3, which +// the next reconcile confirms as the operator's, rather than taking r3 for a +// revision somebody else activated and reporting the write over it as drift. +func TestAHeldReconcileReleasesWhatARunningUnconfirmedWriteLetsGo(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + claims := company.Status.Config.Unconfirmed + if len(claims) != 1 || claims[0].RevisionID != "r2" || !w.engine.holds(w.registry) || len(w.state(t).Retained()) != 0 { + t.Fatalf("status.config.unconfirmed is %+v, the store holds the registry: %t, and the state retains %v; want "+ + "r2 recorded, the registry pushed, and nothing retained", claims, w.engine.holds(w.registry), + w.state(t).Retained()) + } + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + if w.engine.holds(w.registry) { + t.Error("the engine's store still holds the registry, whose Secret is gone and which r2, which runs, lets go") + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r2 activated again as r3 without the registry", ids) + } + state := w.state(t) + if !state.Records("r3") || !slices.Equal(state.LetGo("r3"), []string{w.registry}) || len(state.Withdrawn()) != 0 { + t.Errorf("the state records %q letting %v go and owes %v; want r3 letting the registry go and nothing owed", + state.Recorded(), state.LetGo(state.Recorded()), state.Withdrawn()) + } + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != "r1" || len(config.Unconfirmed) != 1 || + config.Unconfirmed[0].RevisionID != "r3" || config.Unconfirmed[0].AppliedHash != claims[0].AppliedHash { + t.Fatalf("status.config is %+v, want r1 recorded and r2's record moved on to r3", config) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if warned(recorded, crewletaiv1alpha1.ReasonDrifted) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") + }) { + t.Errorf("events %q, want r3 confirmed as the operator's, at epoch 3, and no drift reported", recorded) + } +} + +// TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns: a held +// reconcile takes the registry, which r1, the operator's revision, lets go, +// out of the store and activates r1 again; the engine stores the reload as r2 +// and loses the answer to its activation. Where r2 runs by the read back, it +// activated after the withdrawal, which is settled, and it is recorded as the +// reactivation. Where it has not landed yet it may still: the withdrawal stays +// owed and reported, and r2 is recorded as a write of r1's document, so that +// once it lands the next reconcile records it as the operator's, at its epoch, +// and owes nothing for the withdrawal, rather than taking it for a revision +// somebody else activated. +func TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns(t *testing.T) { + for _, how := range []outcome{landed, pending} { + t.Run(map[outcome]string{landed: "it landed", pending: "it lands later"}[how], func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = how + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.reloadActivation = confirmed + recorded := drained(w.events) + state := w.state(t) + company = w.statusWritten() + if how == landed { + if config := company.Status.Config; len(state.Withdrawn()) != 0 || config.RevisionID != "r2" || + len(config.Unconfirmed) != 0 || !told(recorded, "Reactivated") || warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes %v, status.config is %+v, events %q; want the withdrawal settled, r2 recorded "+ + "as the reactivation, and no failure reported", state.Withdrawn(), config, recorded) + } + return + } + config := company.Status.Config + if !slices.Equal(state.Withdrawn(), []string{w.registry}) || !warned(recorded, "SecretWithdrawalFailed") || + config.RevisionID != "r1" || len(config.Unconfirmed) != 1 || config.Unconfirmed[0].RevisionID != "r2" || + config.Unconfirmed[0].BaseRevisionID != "r1" || config.Unconfirmed[0].AppliedHash != r1Hash { + t.Fatalf("the state owes %v, status.config is %+v, events %q; want the registry owed, the failure reported, "+ + "and r2 recorded unconfirmed on r1 with r1's hash", state.Withdrawn(), config, recorded) + } + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + recorded = drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) { + t.Errorf("events %q, want r2 confirmed as the operator's, at epoch 2", recorded) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 read the store without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !slices.Equal(state.LetGo("r2"), []string{w.registry}) { + t.Errorf("the state owes %v and lets %v go for r2; want nothing owed and the registry let go", + state.Withdrawn(), state.LetGo("r2")) + } + }) + } +} + +// TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound: the write's +// answer was lost three minutes before the reconcile that finds it running, +// and the engine is one epoch behind. Nothing says when the activation landed, +// so the engine is held to applying it from when the operator found it, as an +// adopted revision is: Applying, not an apply that failed. Dated by the write, +// the confirmation read as ApplyFailed, with a Warning, in the reconcile that +// found it. +func TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + written := metav1.NewTime(time.Date(2026, 10, 25, 8, 0, 0, 0, time.UTC)) + company.Status.Config.Unconfirmed[0].WrittenAt = &written + found := written.Add(3 * time.Minute) + w.again(t, company) + w.run.now = func() time.Time { return found } + w.engine.mu.Lock() + w.engine.appliedLag = 1 + w.engine.mu.Unlock() + reconcileOnce(t, w) + if config := w.run.config; config == nil || config.RevisionID != "r2" || config.AppliedAt == nil || + !config.AppliedAt.Time.Equal(found) { + t.Fatalf("status.config is %+v, want r2 confirmed, applied when the operator found it, %s", w.run.config, found) + } + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonApplying { + t.Errorf("ConfigApplied is %+v, want Applying: the operator found r2 a moment ago", c) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonApplyFailed) { + t.Errorf("events %q, want no ApplyFailed", recorded) + } +} + +// TestAWriteRecordsItsDocumentsReadingBeforeItIsSent: the state owes nothing, +// so no change of the store is marked before the write, and the reading of the +// document the write carries is saved before it is sent all the same, under +// the write's hash: its answer lost, nothing else says how the revision the +// engine stored reads the store, once the render has moved on. +func TestAWriteRecordsItsDocumentsReadingBeforeItIsSent(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + reconcileOnce(t, w) + company := w.statusWritten() + if state := w.state(t); state.Unobserved() || !state.Records("r2") || company.Status.Config.RevisionID != "r2" { + t.Fatalf("after the first write the state records %q, with changes unread: %t; want r2, and none", + state.Recorded(), state.Unobserved()) + } + w.engine.mu.Lock() + w.engine.activateOther(w.requiring) + w.engine.mu.Unlock() + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + got := w.statusWritten().Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r4" { + t.Fatalf("status.config.unconfirmed is %+v, want r4, written over the r3 somebody activated", got) + } + if !w.state(t).Pends(got[0].AppliedHash) { + t.Errorf("the state records no reading under r4's hash, %s: saved after the write, its answer lost left none", + got[0].AppliedHash) + } +} + +// TestAWriteWhoseAnswerIsLostWhileTheFleetIsUnreadableSettlesWhatItRead: the +// fleet's activation pointer cannot be read after the changes the write is +// sent after, so nothing marks the rotated value and the withdrawn registry, +// and the engine loses the write's answer. The changes await the revision it +// stored instead, and the next reconcile, finding that revision running, +// settles them: no second write of the same document, and no reload of it. +// Left unread, no epoch the pointer named later could settle them, and the +// operator wrote the document again over its own landed write. The fleet +// answers the read that says what every pod runs, before the changes: one it +// does not answer keeps every value, so the registry would stay. +func TestAWriteWhoseAnswerIsLostWhileTheFleetIsUnreadableSettlesWhatItRead(t *testing.T) { + for _, down := range []bool{true, false} { + t.Run(map[bool]string{true: "the fleet answers 503", false: "the fleet answers without its activation"}[down], func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.rotate(t) + if down { + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + } + w.engine.noActivation = !down + w.engine.activation = landed + reconcileOnce(t, w) + if state := w.state(t); !state.Unobserved() { + t.Fatalf("the state owes %v and %v, all marked; want them unread, as the fleet could not be read", + state.Withdrawn(), state.Written()) + } + w.engine.fleetDown, w.engine.noActivation, w.engine.activation = false, false, confirmed + w.again(t, w.statusWritten()) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: r2 ran, and read every change", written) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: nothing activated r2 again", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || len(state.Written()) != 0 { + t.Errorf("the state owes %v and %v, want nothing", state.Withdrawn(), state.Written()) + } + }) + } +} + +// TestAFirstWriteWhoseAnswerIsLostIsNotWrittenAgain: under Enforce the engine +// runs no document, so the fleet names no activation to mark the values the +// sync pushed by, and the engine loses the answer to the first write. Once a +// read finds the revision it stored running, that revision read every value, +// and the operator writes nothing more. Marked by nothing, the values stayed +// owed, and the operator wrote the same document a second time. +func TestAFirstWriteWhoseAnswerIsLostIsNotWrittenAgain(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.rotate(t) + w.engine.activation = landed + reconcileOnce(t, w) + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the first write", written) + } + w.engine.activation = confirmed + w.again(t, w.statusWritten()) + reconcileOnce(t, w) + if again, _ := w.engine.writes(); !slices.Equal(again, written) { + t.Errorf("the operator wrote %v, want %v alone", again, written) + } + if state := w.state(t); len(state.Written()) != 0 { + t.Errorf("the state owes %v, want nothing: %s read every value", state.Written(), written[0]) + } +} + +// TestAReloadWhoseAnswerIsLostIsTheOperatorsWhenItsReadBackFails: a held +// reconcile activates r1, the operator's, again for the registry it withdrew, +// the engine stores the reload as r2 and loses the answer, and the read back +// fails: GET /config, or the fleet with r2 running. r2 is recorded as the +// operator's from the answer, so once it runs the next reconcile confirms it +// as the operator's, at its epoch, and settles the withdrawal, which awaits +// r2 where no read of the fleet marked it before the reload. Recorded only +// where the read back found r1 still running, it was recorded nowhere, and r2 +// was taken for a revision somebody else activated; and a withdrawal no read +// marked had r2 activated again for it. The fleet answers the read that says +// what every pod runs, before the withdrawal: one it does not answer keeps +// every value, and nothing is withdrawn to reload for. +func TestAReloadWhoseAnswerIsLostIsTheOperatorsWhenItsReadBackFails(t *testing.T) { + for _, c := range []struct { + name string + how outcome + fail func(e *storeEngine) + // fleetLost makes the fleet unreadable after the read that says + // what every pod runs, so no read marks the withdrawal before the + // reload is asked for. + fleetLost bool + }{ + {name: "GET /config fails, the reload landed", how: landed, fail: func(e *storeEngine) { e.configDown = true }}, + {name: "GET /config fails, the reload lands later", how: pending, fail: func(e *storeEngine) { e.configDown = true }}, + {name: "the fleet cannot be read, the reload landed", how: landed, fail: func(e *storeEngine) { e.fleetDown = true }}, + { + name: "the fleet cannot be read across the reload, which landed", how: landed, + fail: func(e *storeEngine) { e.configDown = true }, fleetLost: true, + }, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = c.how + w.engine.beforeReload = c.fail + if c.fleetLost { + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.configDown, w.engine.fleetDown, w.engine.reloadActivation = false, false, confirmed + if c.how == pending { + w.engine.land("r2") + } + r1 := w.engine.revisions[0].document + w.engine.mu.Unlock() + company = w.statusWritten() + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || + got[0].DocumentSHA256 != documentDigest(r1) { + t.Fatalf("status.config.unconfirmed is %+v, want r2, a reload of r1, with r1's document's digest", got) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) || warned(recorded, crewletaiv1alpha1.ReasonDrifted) { + t.Errorf("events %q, want r2 confirmed as the operator's, at epoch 2, and no drift reported", recorded) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 read the store without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 { + t.Errorf("the state owes %v, want the withdrawal settled by r2", state.Withdrawn()) + } + }) + } +} + +// TestAReloadOfAnotherRevisionIsNotTheOperators: the operator asks to activate +// r1 again, and somebody activates r2, another document, before the request +// lands, so the engine stores r3 as a reload of r2, loses the answer, and the +// read back fails. r3 is recorded as a reload of r1, with r1's document's +// digest; found running r2's document, it is not the operator's: nothing +// confirms it as the operator's write, and the next write over it reports it +// as the drift it is. +func TestAReloadOfAnotherRevisionIsNotTheOperators(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = landed + w.engine.beforeReload = func(e *storeEngine) { + e.activateOther(w.requiring) + e.configDown = true + } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.configDown, w.engine.reloadActivation = false, confirmed + w.engine.mu.Unlock() + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r3" { + t.Fatalf("status.config.unconfirmed is %+v, want r3 recorded, as nothing said what it reloaded", got) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3") + }) { + t.Errorf("events %q, want r3 not confirmed as the operator's: it runs r2's document", recorded) + } + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonDriftCorrected || + !strings.HasPrefix(c.Message, "Revision r3, which the operator did not write") { + t.Errorf("ConfigApplied is %+v, want DriftCorrected over r3", c) + } + if config := w.statusWritten().Status.Config; len(config.Unconfirmed) != 0 || config.RevisionID != "r4" { + t.Errorf("status.config is %+v, want r4 recorded and nothing unconfirmed", config) + } +} + +// TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce: a held reconcile +// finds r2, a write of the operator's whose activation went unconfirmed, +// running, and its reload of r2 is stored as r3 and left unconfirmed too. In +// the next held reconcile r3 lands while that reconcile's own reload of r2 +// fails, so the read back finds r3 running r2's document and records it as +// the reactivation. r3 is recorded once: as two entries of one revision, the +// API server refused the status, and every condition the reconcile decided +// was lost. +func TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 2 || config.Unconfirmed[1].RevisionID != "r3" { + t.Fatalf("status.config is %+v, want r2 unconfirmed and the reload r3 recorded on it", config) + } + w.again(t, company) + w.engine.reloadActivation = confirmed + w.engine.reloadDown = true + w.engine.beforeReload = func(e *storeEngine) { e.land("r3") } + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + config := w.statusWritten().Status.Config + ids := make([]string, len(config.Unconfirmed)) + for i := range config.Unconfirmed { + ids[i] = config.Unconfirmed[i].RevisionID + } + if slices.Contains(ids, "r2") || !slices.Equal(slices.Compact(slices.Sorted(slices.Values(ids))), slices.Sorted(slices.Values(ids))) || + !slices.Contains(ids, "r3") { + t.Errorf("status.config.unconfirmed names %v, want r3 once, as the reactivation of r2's record", ids) + } +} + +// TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced: the +// operator's write r2 replaced r1, somebody else's document, and its answer +// was lost; a held reconcile then found r2 running and its reload of r2 was +// stored as r3, its answer lost too. Once r3 runs, its confirmation reports +// that the rendered document replaced r1, once, and never names r2, the +// operator's own write, as somebody else's: whether r2 was confirmed first or +// not at all. +func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing.T) { + for _, c := range []struct { + name string + // confirmFirst confirms r2 in a reconcile of its own before r3 lands. + confirmFirst bool + }{ + {name: "r3 lands first"}, + {name: "r2 confirmed first", confirmFirst: true}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningOther) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.RevisionID = "r0" + w.again(t, company) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r2" || got[0].Replaced == "" { + t.Fatalf("status.config.unconfirmed is %+v, want r2 recorded as replacing r1", got) + } + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + w.engine.reloadActivation = confirmed + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 2 || got[1].RevisionID != "r3" || got[1].Replaced != "" || + got[1].DriftedAt != "" { + t.Fatalf("status.config.unconfirmed is %+v, want r3, built on r2, replacing nothing of its own", got) + } + var reports []string + if c.confirmFirst { + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + reports = append(reports, drained(w.events)...) + company = w.statusWritten() + } + w.engine.mu.Lock() + w.engine.land("r3") + w.engine.mu.Unlock() + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + reports = append(reports, drained(w.events)...) + reason := crewletaiv1alpha1.ReasonDrifted + var warnings []string + for _, e := range reports { + if strings.HasPrefix(e, "Warning "+reason+" ") { + warnings = append(warnings, e) + } + } + // What the event says of the revision replaced: "revision r1, + // which the operator did not write". + names := func(revision string) bool { + return strings.Contains(warnings[0], "revision "+revision+", which") + } + if len(warnings) != 1 || !names("r1") || names("r2") { + t.Errorf("Warning %s events %q, want one, naming r1 and never r2", reason, warnings) + } + if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 recorded as the operator's and nothing unconfirmed", config) + } + }) + } +} + +// TestAConfirmedWriteLeavesNothingOvertaken: a create the engine could not +// confirm was overtaken, and a later write of the operator's is confirmed +// running, by a reconcile that then ends before anything is in sync: a value +// rotated since needs a write (the engine's own re-activation at it failing), +// and the engine cannot decide the dry run. +// Nothing overtaken can land now, nor be the seed, so the record of it goes +// with the confirmation, as the API's own doc says it does once +// status.config records a write; kept, it stood beside that write until a +// later reconcile found the document in sync. +func TestAConfirmedWriteLeavesNothingOvertaken(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + now := metav1.NewTime(time.Unix(1, 0)) + company.Status.Config.Overtaken = []crewletaiv1alpha1.UnconfirmedRevision{ + {RevisionID: "r0", AppliedHash: "sha256:r0", WrittenAt: &now}, + } + w.again(t, company) + rotated := &corev1.Secret{} + if err := w.cluster.Get(ctx, types.NamespacedName{Namespace: "ns", Name: requiredSecret}, rotated); err != nil { + t.Fatal(err) + } + rotated.Data[requiredKey] = []byte(rotatedValue + "-again") + if err := w.cluster.Update(ctx, rotated); err != nil { + t.Fatal(err) + } + operatorActivates(w) + w.engine.dryRunDown = true + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone: the dry run after the confirmation was not decided", written) + } + if config := w.statusWritten().Status.Config; config.RevisionID != "r2" || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want r2 recorded and nothing overtaken", config) + } +} diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go index c33b7f65..07574499 100644 --- a/internal/controller/config_unreadable_test.go +++ b/internal/controller/config_unreadable_test.go @@ -97,9 +97,9 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { if result.RequeueAfter != w.run.resyncInterval() { t.Errorf("the next look is %s on, want the resync interval, %s", result.RequeueAfter, w.run.resyncInterval()) } - if ids := w.engine.ids(); len(ids) != 1 { - t.Errorf("the engine holds revisions %v: the operator wrote over a revision the engine cannot open, "+ - "want nothing written", ids) + if written, refused := w.engine.writes(); len(written) != 0 || refused != 0 { + t.Errorf("the operator wrote %v (and %d refused) over a revision the engine cannot open, want nothing", + written, refused) } if recorded := drained(w.events); !warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { t.Errorf("events %q, want a Warning RevisionUnreadable", recorded) diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index aa4502f7..9b6223d5 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -158,9 +158,11 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) return ctrl.Result{}, err } if len(read.Missing()) == 0 && len(released) == 0 && len(state.Withdrawn()) == 0 && len(state.Written()) == 0 && - len(state.Retained()) == 0 { + len(state.Retained()) == 0 && len(run.recordedConfig().Unconfirmed) == 0 { // No value to take out of the store or to say it keeps, no - // activation owed, and nothing said kept that has come back. + // activation owed, nothing said kept that has come back, and no + // write of the operator's that may run, which may let go what the + // revision the state records does not (settlePending). return ctrl.Result{}, nil } eng, err := run.Engines.Engine(ctx, c) @@ -171,6 +173,18 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err != nil { return run.withdrawalFailed(ctx, err), nil } + // The revision that runs may be a write of the operator's the engine + // could not confirm activating, whose reading the state recorded before + // it was sent, and whose document, where the render has moved on, lets + // go of what the one the state records does not. + settled := run.settlePending(state, active) + if settled.Recorded() != state.Recorded() { + // What the revision the state now records goes without. + if released, err = secretsync.Absent(ctx, run.APIReader, settled.LetGoKeys(settled.Recorded())); err != nil { + return ctrl.Result{}, err + } + } + state = settled if active != nil { if run.doc != nil { // Held for a Secret, not refused by the render: there is a @@ -579,8 +593,17 @@ func (run *configRun) reactivate( if by != "" { reloaded.RevisionID, reloaded.Epoch = reload.RevisionID, reload.Epoch } else { + // How the revision the request may store reads every name, should + // the engine not confirm activating it and a later read find it + // running (config_unconfirmed.go): as the one it activates again + // does. + own, ok := run.ownWrite(active) + ahead := ok && !state.Pends(own.AppliedHash) + if ahead { + state = state.Pending(own.AppliedHash, state.LetGoKeys(active.RevisionID)) + } var err error - if state, err = run.markChanges(ctx, eng, state, ref); err != nil { + if state, err = run.markChanges(ctx, eng, state, ref, ahead); err != nil { return state, err } if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(owed)); err != nil { @@ -622,10 +645,25 @@ func (run *configRun) reactivate( // a Warning says, and the state goes on recording the one read, as // reactivatedUnconfirmed leaves it. A request that failed outright leaves the // fleet on active, which settles nothing. +// +// One whose revision the engine stored and could not confirm activating may +// yet land. That is decided from the answer, before anything is read back, +// since a read that fails, or finds the revision running with the fleet's +// pointer unreadable, leaves nothing else to say it: where active is the +// operator's, the revision is recorded as its own write of the same document, +// for the next read to settle as it settles one (reloadUnconfirmed), and the +// changes it was asked for after await it, so that finding it running +// settles them (State.Awaiting) however the pointer reads. Recorded only +// where the read back still found active, it was taken, once it landed, for a +// revision somebody else activated. func (run *configRun) reloadFailed( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, owed storeChanges, failed error, ) (secretsync.State, error) { + if stored, ok := engine.UnconfirmedActivation(failed); ok { + run.reloadUnconfirmed(active, stored) + state = state.Awaiting(stored) + } now, err := activeConfig(ctx, eng) if err != nil || now == nil || now.RevisionID == active.RevisionID { return state, failed @@ -679,10 +717,19 @@ func reactivationSummary(owed storeChanges) string { // by empty. A revision the operator recorded // as its own is replaced by the one that activates it again, with the same // content, so that the next one it applies does not take it for an edit -// somebody else made; one it did not write stays somebody else's. failed is -// the error the operator's request answered where the engine was read back -// running the document again anyway (reloadFailed), nil where it answered the -// activation. +// somebody else made; so is a write of the operator's the engine could not +// confirm activating and runs, whose record (status.config.unconfirmed) moves +// to the revision that activates it again, for the next reconcile that +// applies a document to confirm and report (config_unconfirmed.go); one it +// did not write stays somebody else's. failed is the error the operator's +// request answered where the engine was read back running the document again +// anyway (reloadFailed), nil where it answered the activation. +// +// reloaded is recorded once: a record status.config.unconfirmed already holds +// of it, a reload the engine stored and could not confirm activating +// (reloadUnconfirmed), goes, as the read back settled it. Kept, it was a +// second entry of one revision, which the API server refuses, or a revision +// the next reconcile confirmed a second time. func (run *configRun) recordReactivation( active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, owed storeChanges, failed error, ) { @@ -697,20 +744,30 @@ func (run *configRun) recordReactivation( "so it reads the store %s; the operator's request to activate it again failed: %s", active.RevisionID, reloaded.RevisionID, reloaded.Epoch, owed.made(), owed.why("the revision"), failed))) } - recorded := run.company.Status.Config - if run.config != nil { - recorded = *run.config + recorded := run.recordedConfig() + config := recorded.DeepCopy() + if i := unconfirmedIndex(config.Unconfirmed, reloaded.RevisionID); i >= 0 { + config.Unconfirmed = slices.Delete(config.Unconfirmed, i, i+1) + run.config = config } - if recorded.RevisionID != active.RevisionID { + now := metav1.NewTime(run.clock()) + switch i := unconfirmedIndex(config.Unconfirmed, active.RevisionID); { + case recorded.RevisionID == active.RevisionID: + config.RevisionID, config.Epoch, config.AppliedAt = reloaded.RevisionID, new(reloaded.Epoch), &now + if run.applied.kind == appliedInRevision && run.applied.revision == active.RevisionID { + run.applied.revision = reloaded.RevisionID + } + case i >= 0: + // Confirmed only in the secret sync's state (settlePending), by a + // reconcile that holds its document: the record goes on to the + // revision that runs it now, which is the write's as much as the + // one it activated again was, rather than staying with one the + // next read finds overtaken. + config.Unconfirmed[i].RevisionID, config.Unconfirmed[i].WrittenAt = reloaded.RevisionID, &now + default: return } - config := recorded.DeepCopy() - now := metav1.NewTime(run.clock()) - config.RevisionID, config.Epoch, config.AppliedAt = reloaded.RevisionID, new(reloaded.Epoch), &now run.config = config - if run.applied.kind == appliedInRevision && run.applied.revision == active.RevisionID { - run.applied.revision = reloaded.RevisionID - } } // reactivatedUnconfirmed reports an activation of active again, reading the diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 6dae8a20..7b896560 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -44,13 +44,20 @@ import ( // storeEngine answers the engine's /health, /config, /config/revisions, // /fleet and /secrets routes from a revision history, the last of which is -// the fleet's, and a store, as the operator's client reads them. A document -// it is to refuse is refused as refusal says. -// beforeReload, afterReload and afterApply, when set, run once, under its -// lock, just before a reload is answered and just after a reload or an apply -// is: somebody else's write landing between two of the operator's. Its fleet -// answers the activation pointer beside its nodes: the fleet's revision at -// its place in the history, which is its epoch. +// the fleet's (none, an engine that runs no document yet), and a store, as the +// operator's client reads them. A write of the document is refused on the +// condition it names, as the engine refuses it, and so is every document while +// it is to refuse them, as refusal says; written are the revisions the writes +// it took stored, and refused counts the ones it refused on their condition. +// beforeReload, afterReload, beforeWrite and afterApply, when set, run once, +// under its lock, just before a reload or a write is answered and just after +// a reload or an apply is: somebody else's write landing between two of the +// operator's. activation is how the next write's activation goes, and inert +// the revisions writes stored whose activation has not landed +// (config_unconfirmed_test.go). minted counts the revisions it has named, +// active or not, so that no two share an id. Its fleet answers the activation +// pointer beside its nodes: the fleet's revision at its place in the history, +// which is its epoch. // // A write into its store (PUT /secrets, PUT /secrets/{name}), and a removal // of a value (DELETE /secrets/{name}), re-activates the fleet's revision when its @@ -65,13 +72,15 @@ import ( // refused the fleet's, "" for one that has recorded no attempt and so not // said); runsNone names nodes beside those that refused the fleet's revision // and run no company, as a seat pod that started on nothing and refused it -// does; fleetDown makes GET /fleet unanswerable, and missing names the -// revisions GET /config/revisions does not find. configDown, secretsDown and -// reloadDown make it answer a read of the fleet's document, a write to the -// store, and a reload of the fleet's revision, as an engine that cannot decide -// who is asking does; unreadable, when set, gives -// the body it answers a read of the fleet's document with, as an engine whose -// keyring does not open that revision does (config_unreadable_test.go). +// does; fleetDown makes GET /fleet unanswerable, as an engine whose +// coordination store does not answer the read of its lease table +// (fleetUnreadable), and missing names the revisions GET /config/revisions +// does not find. configDown, secretsDown and reloadDown make it answer a read +// of the fleet's document, a write to the store, and a reload of the fleet's +// revision, as an engine that cannot decide who is asking does; unreadable, +// when set, gives the body it answers a read of the fleet's document with, as +// an engine whose keyring does not open that revision does +// (config_unreadable_test.go). // // Each node's row says what it starts on as the engine's does: a data node // (dataNode) the revision it runs, and a seat node, whose store is scratch, @@ -81,6 +90,9 @@ import ( type storeEngine struct { mu sync.Mutex revisions []storeRevision + minted int + written []string + refused int store map[string]string // refusal, when set, is how the engine refuses every PUT of the document, // the dry run included: invalidDocument, a document it finds invalid, or @@ -92,7 +104,7 @@ type storeEngine struct { configDown, secretsDown bool reloadDown bool beforeReload, afterReload func(e *storeEngine) - afterApply func(e *storeEngine) + beforeWrite, afterApply func(e *storeEngine) reloadAnswer *map[string]any afterSecretsReload func(e *storeEngine) serving map[string]string @@ -101,6 +113,11 @@ type storeEngine struct { fleetDown bool missing map[string]bool unreadable func(revision string) string + activation outcome + // reloadActivation is how the activation of the next reload goes, as + // activation is the next write's. + reloadActivation outcome + inert []storeRevision // beforeConfig and beforeFleet run once, before the engine answers the // next read of the fleet's document or of its fleet: an activation // somebody else makes in that moment. @@ -128,6 +145,14 @@ func dataNode(node string) bool { // otherwise. const storeNode = "acme-data-0" +// mint names the engine's next revision: one past every revision it has +// named, active or not, and past every one a test put in its history by +// hand, so that no two share an id. The caller holds e's lock. +func (e *storeEngine) mint() string { + e.minted = max(e.minted, len(e.revisions)+len(e.inert)) + 1 + return fmt.Sprintf("r%d", e.minted) +} + // down reports whether r is a request configDown, secretsDown, reloadDown or // dryRunDown makes the engine refuse: a read of the fleet's document, a write // to the store, one name's or a batch's, a reload of the fleet's revision, or @@ -158,6 +183,19 @@ func cannotDecide(w http.ResponseWriter) { `"message":"Can't check who you are right now. Try again in a moment."}`)) } +// fleetUnreadable is the engine's answer to GET /fleet when its coordination +// store does not answer the read of the lease table, as crewlet#142 words it: +// the unavailable code's own sentence, the condition's as the detail, and the +// five seconds of the health tick that clears it. +func fleetUnreadable(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"unavailable",` + + `"message":"Can't answer right now. Try again in a moment, or ask another node.",` + + `"detail":"This node couldn't reach the rest of the fleet. Try again in a moment."}`)) +} + type storeRevision struct { id, summary string document []byte @@ -166,7 +204,7 @@ type storeRevision struct { // newStoreEngine runs the engine with document as its active revision r1. func newStoreEngine(t *testing.T, document []byte) (*storeEngine, *engine.Client) { t.Helper() - e := &storeEngine{revisions: []storeRevision{{id: "r1", document: document}}, store: map[string]string{}} + e := &storeEngine{revisions: []storeRevision{{id: "r1", document: document}}, minted: 1, store: map[string]string{}} server := httptest.NewServer(http.HandlerFunc(e.serve)) t.Cleanup(server.Close) return e, engine.New(server.URL, "token") @@ -180,9 +218,7 @@ type engineRefusal struct { } func (refusal *engineRefusal) answer(w http.ResponseWriter) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(refusal.status) - _, _ = io.WriteString(w, refusal.body) + answerRaw(w, refusal.status, refusal.body) } var ( @@ -207,15 +243,13 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) answer := func(status int, v any) { answerJSON(w, status, v) } e.before(r) - active := e.revisions[len(e.revisions)-1] + active := e.active() activate := func(document []byte) storeRevision { - revision := storeRevision{id: fmt.Sprintf("r%d", len(e.revisions)+1), document: document, summary: r.Header.Get("X-Summary")} + revision := storeRevision{id: e.mint(), document: document, summary: r.Header.Get("X-Summary")} e.revisions = append(e.revisions, revision) return revision } - activated := func(revision storeRevision) { - answer(http.StatusCreated, map[string]any{revisionIDField: revision.id, "epoch": len(e.revisions)}) - } + activated := func(revision storeRevision) { e.activated(answer, revision) } if e.down(r) { cannotDecide(w) return @@ -232,22 +266,18 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { if e.appliedLag > 0 { posture = engine.PostureWait } - answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": posture, + answer(http.StatusOK, map[string]any{"status": "ok", "configured": active.id != "", "posture": posture, "applied_epoch": len(e.revisions) - e.appliedLag}) + case r.Method == http.MethodGet && r.URL.Path == configRoute && active.id == "": + answerRaw(w, http.StatusNotFound, `{"error":"no_active_revision"}`) case r.Method == http.MethodGet && r.URL.Path == configRoute: e.readActive(w, active) - case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refusal != nil: - e.refusal.answer(w) - case r.Method == http.MethodPut && r.URL.Path == configRoute && dryRun(r): - answer(http.StatusOK, map[string]any{"base_revision_id": active.id}) case r.Method == http.MethodPut && r.URL.Path == configRoute: - applied := activate(body) - if e.afterApply != nil { - hook := e.afterApply - e.afterApply = nil - hook(e) - } - activated(applied) + e.put(w, r, body, answer) + case r.Method == http.MethodPost && r.URL.Path == reloadRoute && e.reloadActivation == pending: + stored := storeRevision{id: e.mint(), document: active.document, summary: r.Header.Get("X-Summary")} + e.inert = append(e.inert, stored) + answerLost(w, stored.id, active.id) case r.Method == http.MethodPost && r.URL.Path == reloadRoute: reloaded := activate(active.document) if e.afterReload != nil { @@ -255,6 +285,10 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { e.afterReload = nil hook(e) } + if e.reloadActivation == landed { + answerLost(w, reloaded.id, active.id) + return + } activated(reloaded) case r.Method == http.MethodGet && r.URL.Path == secretsRoute: listing := make([]map[string]string, 0, len(e.store)) @@ -375,8 +409,7 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st revisionID, isRevision := strings.CutPrefix(path, "/config/revisions/") switch { case path == fleetRoute && e.fleetDown: - w.Header().Set("Retry-After", "1") - answerJSON(w, http.StatusServiceUnavailable, refusal("unavailable", "coordination is unreachable")) + fleetUnreadable(w) case path == fleetRoute: serving := e.serving if serving == nil { @@ -417,7 +450,7 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st nodes = append(nodes, row(node, len(e.revisions), active.id, engine.ApplyError, "")) } answerJSON(w, http.StatusOK, map[string]any{"this_node": storeNode, "nodes": nodes, "target_epoch": len(e.revisions), - "activation": e.activation()}) + "activation": e.pointer()}) case isRevision: i := slices.IndexFunc(e.revisions, func(rev storeRevision) bool { return rev.id == revisionID }) if i < 0 || e.missing[revisionID] { @@ -463,15 +496,16 @@ type activationAnswer struct { Epoch int `json:"epoch"` } -// activation is the activation pointer the engine answers beside its nodes: -// the fleet's revision at the epoch it activated as, which is its place in -// the history, nil where noActivation says the engine could not read it. The -// caller holds e's lock. -func (e *storeEngine) activation() *activationAnswer { - if e.noActivation { +// pointer is the activation pointer the engine answers beside its nodes: the +// fleet's revision at the epoch it activated as, which is its place in the +// history; nil for an engine with no revision, and where noActivation says the +// engine could not read it. The caller holds e's lock. +func (e *storeEngine) pointer() *activationAnswer { + running := e.active().id + if running == "" || e.noActivation { return nil } - return &activationAnswer{RevisionID: e.revisions[len(e.revisions)-1].id, Epoch: len(e.revisions)} + return &activationAnswer{RevisionID: running, Epoch: len(e.revisions)} } // dryRun reports whether r asks only for a check of the document. @@ -496,15 +530,100 @@ func refusal(code, detail string) map[string]any { // its keyring does not open. func (e *storeEngine) readActive(w http.ResponseWriter, active storeRevision) { if e.unreadable != nil { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusConflict) - _, _ = w.Write([]byte(e.unreadable(active.id))) + answerRaw(w, http.StatusConflict, e.unreadable(active.id)) return } w.Header().Set("ETag", `"`+active.id+`"`) _, _ = w.Write(active.document) } +// active is the revision the engine runs, the zero revision for none. The +// caller holds e's lock. +func (e *storeEngine) active() storeRevision { + if n := len(e.revisions); n > 0 { + return e.revisions[n-1] + } + return storeRevision{} +} + +// answerRaw answers with status and body, the engine's JSON as it words it. +func answerRaw(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) +} + +// put answers PUT /config, a write of the document whose body is body or its +// dry run, as the engine does: a document it is to refuse is refused as refusal +// says; a write is refused when its condition (If-Match, If-None-Match) no +// longer holds, and otherwise stored and activated, the engine's answer to the +// activation lost as activation says. The caller holds e's lock. +func (e *storeEngine) put(w http.ResponseWriter, r *http.Request, body []byte, answer func(int, any)) { + if e.refusal != nil { + e.refusal.answer(w) + return + } + if dryRun(r) { + answer(http.StatusOK, map[string]any{"base_revision_id": e.active().id}) + return + } + if hook := e.beforeWrite; hook != nil { + e.beforeWrite = nil + hook(e) + } + running := e.active().id + if base := strings.Trim(r.Header.Get("If-Match"), `"`); base != "" && base != running { + // A failed precondition: the write named in If-Match a revision + // that no longer runs, which the engine answers 412 whether it + // found it so before the write or lost the race to another after, + // naming the revision that runs and the base as the header gave it. + e.refused++ + answerJSON(w, http.StatusPreconditionFailed, struct { + Code string `json:"error"` + Current string `json:"current_revision_id"` + Base string `json:"your_base"` + }{"revision_advanced", running, r.Header.Get("If-Match")}) + return + } + if r.Header.Get("If-None-Match") == "*" && running != "" { + e.refused++ + answerRaw(w, http.StatusPreconditionFailed, fmt.Sprintf(`{"error":"already_configured","current_revision_id":%q}`, running)) + return + } + revision := storeRevision{id: e.mint(), document: body, summary: r.Header.Get("X-Summary")} + e.written = append(e.written, revision.id) + switch e.activation { + case confirmed: + e.revisions = append(e.revisions, revision) + if e.afterApply != nil { + hook := e.afterApply + e.afterApply = nil + hook(e) + } + e.activated(answer, revision) + return + case landed: + e.revisions = append(e.revisions, revision) + case pending: + e.inert = append(e.inert, revision) + } + answerLost(w, revision.id, strings.Trim(r.Header.Get("If-Match"), `"`)) +} + +// activated answers a write or a reload that activated revision, the +// newest, at an epoch per activation. The caller holds e's lock. +func (e *storeEngine) activated(answer func(int, any), revision storeRevision) { + answer(http.StatusCreated, map[string]any{"revision_id": revision.id, "epoch": len(e.revisions)}) +} + +// writes are the revisions the engine's writes of the document stored, and +// how many it refused on their condition. +func (e *storeEngine) writes() ([]string, int) { + e.mu.Lock() + defer e.mu.Unlock() + return slices.Clone(e.written), e.refused +} + // ids are the engine's revisions, oldest first. func (e *storeEngine) ids() []string { e.mu.Lock() @@ -534,7 +653,7 @@ func (e *storeEngine) holds(name string) bool { // edit through the engine's own API: the caller holds e's lock. func (e *storeEngine) activateOther(document []byte) { e.revisions = append(e.revisions, storeRevision{ - id: fmt.Sprintf("r%d", len(e.revisions)+1), document: document, summary: "somebody else's", + id: e.mint(), document: document, summary: "somebody else's", }) } @@ -1130,16 +1249,18 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { for _, tc := range []struct { name string // fails is the write of the state in the first reconcile that - // fails: 1 before the apply, 2 its record, 3 after the deletion. + // fails: 1 the sync's before the apply, 2 the reading of the + // document saved before the write, 3 the apply's record, 4 after + // the deletion. fails int32 // held is whether the next reconcile's document is another one, // held for a Secret, so that only the record says how the revision // the engine runs reads the value. held bool }{ - {name: "the record of the apply is lost", fails: 2}, - {name: "the record of the deletion is lost", fails: 3}, - {name: "the record of the deletion is lost, and the next document is held", fails: 3, held: true}, + {name: "the record of the apply is lost", fails: 3}, + {name: "the record of the deletion is lost", fails: 4}, + {name: "the record of the deletion is lost, and the next document is held", fails: 4, held: true}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() @@ -1152,13 +1273,13 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { // r2 reads, is removed: a record of the deletion lost is one of // that activation lost too. first := []string{"r1", "r2"} - if tc.fails == 3 { + if tc.fails == 4 { first = append(first, "r3") } if got, deleted := w.engine.ids(), !w.engine.holds(w.registry); !slices.Equal(got, first) || - deleted != (tc.fails == 3) { + deleted != (tc.fails == 4) { t.Fatalf("the revisions are %v and the registry is deleted: %t, want %v, and deleted: %t", - got, deleted, first, tc.fails == 3) + got, deleted, first, tc.fails == 4) } w.failWrite.Store(0) @@ -1190,7 +1311,7 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { // is one whose reading the operator does not know, which needs no // activation for a value it already went without. want := []string{"r1", "r2", "r3"} - if tc.fails == 3 && !tc.held { + if tc.fails == 4 && !tc.held { want = append(want, "r4") } last := want[len(want)-1] @@ -1202,7 +1323,7 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { if len(state.Withdrawn()) != 0 { t.Errorf("the state owes %v, want nothing", state.Withdrawn()) } - if letsGo := tc.fails != 3 || !tc.held; letsGo != slices.Equal(state.LetGo(last), []string{w.registry}) { + if letsGo := tc.fails != 4 || !tc.held; letsGo != slices.Equal(state.LetGo(last), []string{w.registry}) { t.Errorf("the state lets %v go for %s, want the registry: %t", state.LetGo(last), last, letsGo) } }) diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 2e1e4d54..3d316920 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -1384,6 +1384,61 @@ func TestActiveConfigWhileTheFleetCannotBeRead(t *testing.T) { } } +// unknownOutcome is the engine's answer to a write it stored and whose +// activation its coordination store did not answer (crewlet's +// internal/api/configapi, RefuseActivationUnknown): 503 unavailable with the +// unknown outcome, the revision it stored, the node whose history alone holds +// it, the revision the write was built on, and a hint saying how to look +// first, ended as httpjson.Sentence ends it. +func unknownOutcome(stored, node, base string) string { + hint := fmt.Sprintf("Revision %[1]s was stored on node %[2]s, and the coordination store did not answer its "+ + "activation, so whether the fleet runs it is unknown. GET /config names the active revision once the store "+ + "answers again. If it is not %[1]s, the revision is in node %[2]s's history alone: send GET "+ + "/config/revisions/%[1]s/diff, and POST /config/revisions/%[1]s/revert to make it the company, to that "+ + "node — by its own address where a shared one fronts several, since any other node holds no copy and "+ + "answers 404. Sending the write again makes another revision.", stored, node) + body, _ := json.Marshal(map[string]string{ + "error": "unavailable", "message": "Can't answer right now. Try again in a moment, or ask another node.", + "outcome": "unknown", "stored_revision_id": stored, "node": node, "your_base": base, "hint": hint, + }) + return string(body) +} + +// TestUnconfirmedActivationIsTheEnginesUnknownOutcome pins which answers to a +// write UnconfirmedActivation reads as one whose activation the engine sent +// and could not confirm: the 503 naming the revision it stored, and no +// other. A lost race names a stored revision too, but the engine knows it +// did not activate, and a 503 naming none stored nothing to look for. +func TestUnconfirmedActivationIsTheEnginesUnknownOutcome(t *testing.T) { + for _, c := range []struct { + name, body string + status int + stored string + }{ + {name: "an activation whose answer was lost", status: http.StatusServiceUnavailable, stored: "rev-10", + body: unknownOutcome("rev-10", "node-0", "rev-9")}, + {name: "a write that lost the race", status: http.StatusConflict, + body: `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, + {name: "a seed another write beat", status: http.StatusPreconditionFailed, + body: `{"error":"already_configured","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, + {name: "an engine that could not store it", status: http.StatusServiceUnavailable, + body: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + f.handle("PUT /config", answer(c.status, c.body)) + _, err := apply(f, write) + stored, ok := engine.UnconfirmedActivation(err) + if ok != (c.stored != "") || stored != c.stored { + t.Errorf("UnconfirmedActivation(%v) = %q, %t; want %q, %t", err, stored, ok, c.stored, c.stored != "") + } + }) + } + if stored, ok := engine.UnconfirmedActivation(nil); ok || stored != "" { + t.Errorf("UnconfirmedActivation(nil) = %q, %t; want nothing", stored, ok) + } +} + // TestAReloadThatLosesItsRaceIsRevisionAdvanced pins that a reload, which // names no revision, is a write that races others: one that loses its // activation to another write is ErrRevisionAdvanced, as the engine answers it diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index aef218fa..dbf25d05 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -274,7 +274,8 @@ func TestEngineFleetNamesTheSeatsHolder(t *testing.T) { } // The fleet's activation names the revision GET /config answers, at its // epoch: the one read that says which epoch a revision the operator did - // not activate itself activated as. + // not activate itself activated as, and that settles a write the engine + // could not confirm. active, err := client.ActiveConfig(ctx) if err != nil { t.Fatal(err) diff --git a/internal/engine/errors.go b/internal/engine/errors.go index df787ef3..df5f7459 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -150,8 +150,10 @@ type APIError struct { // named one (revision_advanced, already_configured). CurrentRevisionID string - // StoredRevisionID is the revision a write that lost the race stored and - // left inert, when the engine kept one. + // StoredRevisionID is the revision a write stored, when the engine kept + // one and could not activate it: a write that lost the race, which left + // it inert, or one whose activation the engine sent and could not + // confirm (UnconfirmedActivation), which it may run or may not. StoredRevisionID string // RevisionID is the revision a refusal is about, when the engine named @@ -236,6 +238,24 @@ func (e *APIError) Is(target error) bool { return false } +// UnconfirmedActivation is the revision a config write stored whose activation +// the engine sent to its fleet and could not confirm, and true, when err is +// that answer: a 503 naming the revision it stored (ErrUnavailable with a +// StoredRevisionID). A lost race names a stored revision too, but it is no +// 503, and the engine knows that one did not activate. +// +// The fleet may run the revision or may not, and only a read of the fleet's +// activation (Fleet.Activation) says which: a caller that wrote it keeps it +// until a read does, and sends the write again only once one says it did +// not land. +func UnconfirmedActivation(err error) (string, bool) { + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusServiceUnavailable || apiErr.StoredRevisionID == "" { + return "", false + } + return apiErr.StoredRevisionID, true +} + // NotReadyError is an engine that answered but will not take work now. type NotReadyError struct { // Reason is the engine's own word for why: draining, unconfigured, shed diff --git a/internal/engine/types.go b/internal/engine/types.go index bfd0b65c..18ed1519 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -491,6 +491,9 @@ type Fleet struct { // activate itself, such as a reload of its document, activated as: // GET /config names the revision and not its epoch, and /health the // epoch a node has applied, which may be another revision's. + // And it is the one answer to whether a write the engine could not + // confirm activated (ErrUnavailable naming a StoredRevisionID): the + // pointer names that revision or it does not. Activation *Activation } diff --git a/internal/secretsync/state.go b/internal/secretsync/state.go index 69d817c8..24b38977 100644 --- a/internal/secretsync/state.go +++ b/internal/secretsync/state.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "maps" @@ -45,7 +46,7 @@ const ( // a space, by the activation epoch it waits for an activation above, // where a read of the fleet's activation pointer has followed it // (State.Observed). Like fingerprintKeyField each has a dash, so Owns - // never accepts it, and so have the three after them. + // never accepts it, and so have the five after them. withdrawnField = "withdrawn-names" writtenField = "written-names" @@ -57,6 +58,17 @@ const ( revisionField = "activated-revision" letGoField = "optional-keys" retainedField = "retained-names" + + // pendingField is the data key of the state Secret that holds the + // readings of the documents written whose activation a read is still to + // confirm (State.Pending), as one JSON object keyed by the hash the + // caller gave each. + pendingField = "pending-activations" + + // awaitField is the data key of the state Secret that lists the + // revisions the changes of the store await (State.Awaiting): one change + // and one revision per line, the name, a space and the revision. + awaitField = "awaited-revisions" ) // State is what the operator last pushed into one engine's secret store: a @@ -66,8 +78,12 @@ const ( // store after, with the activation epoch each waits past (Observed, Ran); one // revision of the engine's document whose reading of the store it knows, with // the Secret keys that revision lets go without (Activated, Adopted, LetGo); -// and the names it last kept for the running revision although their Secret -// or key is gone (Retained). +// the names it last kept for the running revision although their Secret or +// key is gone (Retained); the reading of each document the caller wrote +// whose revision a later read may find running without the caller having +// recorded it (Pending, Confirmed); and the revisions the engine stored for +// activations asked for after a change and could not confirm, any of which +// found running settles it (Awaiting, Found). // // It records nothing of what an engine node runs or starts on, which the // caller reads from the engine's fleet alone: a node applies the fleet's @@ -94,12 +110,57 @@ type State struct { // epoch came after, and unread where no read has followed it yet. withdrawn map[string]int64 written map[string]int64 + // awaits are the revisions Awaiting recorded for those changes, by + // name: each a revision the engine stored for an activation asked for + // after the change, sorted. An entry for a name the State records no + // change of is none: a change settled otherwise leaves nothing to await. + awaits map[string][]string + // readings is how the documents the State knows of read the store, nil + // for none (readingRecord). + readings *readingRecord + retained map[string]struct{} +} + +// readingRecord is how the documents a State knows of read the store: the +// revision whose reading it records, and the documents the caller wrote whose +// revision a read may yet find running (Pending), whose reading becomes the +// revision's once one does (Confirmed). Behind one pointer, so that a State, +// which callers pass by value, stays inside the size a value parameter is +// held to; and never changed once made: a State that changes it makes another +// (withReadings). +type readingRecord struct { // revision is the revision whose reading the State records, empty for // none, and letGo the Secret keys its document lets go without, by // store name. revision string letGo map[string]secretref.Key - retained map[string]struct{} + // pending are the readings Pending recorded: the Secret keys each + // document lets go without, by store name, by the hash the caller gave + // the document. + pending map[string]map[string]secretref.Key +} + +// reading is what s records of how documents read the store, the zero record +// for nothing. +func (s State) reading() readingRecord { + if s.readings == nil { + return readingRecord{} + } + return *s.readings +} + +// withReadings is s recording r, which it does not change afterwards. +func (s State) withReadings(r readingRecord) State { + s.readings = nil + if r.revision != "" || len(r.letGo) > 0 || len(r.pending) > 0 { + s.readings = &r + } + return s +} + +// pending is what s records under Pending, nil for nothing. +func (s State) pending() map[string]map[string]secretref.Key { + return s.reading().pending } // Withdrawn are the names Sync or Withdraw removed from the store because the @@ -146,7 +207,7 @@ func (s State) Written() []string { // each withdrawn, so both are settled. The revision it records stays // recorded. func (s State) Reloaded() State { - s.withdrawn, s.written = nil, nil + s.withdrawn, s.written, s.awaits = nil, nil, nil return s } @@ -156,6 +217,7 @@ func (s State) Reloaded() State { // The revision it records stays recorded. func (s State) Reactivated() State { s.withdrawn = nil + s.awaits = s.awaited() return s } @@ -196,14 +258,79 @@ func (s State) Observed(epoch int64) State { func (s State) Ran(epoch int64) State { s.withdrawn = ran(s.withdrawn, epoch) s.written = ran(s.written, epoch) + s.awaits = s.awaited() + return s +} + +// Awaiting is s once the caller has asked the engine for an activation after +// every change of the store s records, and the engine stored revision for it +// and could not confirm activating it: should a read find revision running, +// it activated after every one of those changes and every node read the +// store when it applied it, so it settles them (Found), whatever the fleet's +// activation pointer says. A change a later read of the pointer has followed +// is settled by any activation above it (Ran), and needs no such record; one +// no read has followed (Observed) needs it, since no epoch the pointer names +// later can settle it, and the pointer is likeliest to be unreadable when the +// engine loses an activation's answer. A later change of a name waits for an +// activation of its own. +func (s State) Awaiting(revision string) State { + if revision == "" { + return s + } + awaits := s.awaited() + for _, marks := range []map[string]int64{s.withdrawn, s.written} { + for name := range marks { + if revisions := awaits[name]; !slices.Contains(revisions, revision) { + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = slices.Sorted(slices.Values(append(slices.Clone(revisions), revision))) + } + } + } + s.awaits = awaits + return s +} + +// Found is s once the caller has read the engine running revision, as a +// revision its nodes can open: every change that awaits it (Awaiting) is +// settled, as Ran settles one an activation above its epoch came after. +func (s State) Found(revision string) State { + awaits := s.awaits + if revision == "" || len(awaits) == 0 { + return s + } + read := func(name string, _ int64) bool { return slices.Contains(awaits[name], revision) } + s.withdrawn = without(s.withdrawn, read) + s.written = without(s.written, read) + s.awaits = s.awaited() return s } // SameMarks reports whether s and o record the same changes of the store, -// each with the same mark: whether a save of one leaves what a save of the -// other did. +// each with the same mark and awaiting the same revisions: whether a save of +// one leaves what a save of the other did. func (s State) SameMarks(o State) bool { - return maps.Equal(s.withdrawn, o.withdrawn) && maps.Equal(s.written, o.written) + return maps.Equal(s.withdrawn, o.withdrawn) && maps.Equal(s.written, o.written) && + maps.EqualFunc(s.awaited(), o.awaited(), slices.Equal) +} + +// awaited is what s awaits (Awaiting) for the changes it records, in a map of +// its own, nil for none: an entry for a name settled otherwise is dropped. +func (s State) awaited() map[string][]string { + var awaits map[string][]string + for name, revisions := range s.awaits { + _, withdrawn := s.withdrawn[name] + _, written := s.written[name] + if len(revisions) == 0 || !withdrawn && !written { + continue + } + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = revisions + } + return awaits } // Unobserved reports whether s records a change of the store no read of the @@ -235,9 +362,15 @@ func observed(marks map[string]int64, epoch int64) map[string]int64 { // ran is marks less every one an activation at epoch is above, in a map of // its own, nil for none. func ran(marks map[string]int64, epoch int64) map[string]int64 { + return without(marks, func(_ string, mark int64) bool { return epoch > mark }) +} + +// without is marks less every one settled reports, in a map of its own, nil +// for none. +func without(marks map[string]int64, settled func(name string, mark int64) bool) map[string]int64 { var left map[string]int64 for name, mark := range marks { - if epoch > mark { + if settled(name, mark) { continue } if left == nil { @@ -268,23 +401,89 @@ func (s State) Activated(revision string, optional []secretref.Key) State { // optional, but did not activate it after the withdrawals it owes, so the // engine may have read the store with them. func (s State) Adopted(revision string, optional []secretref.Key) State { - s.revision = revision - s.letGo = make(map[string]secretref.Key, len(optional)) + r := s.reading() + r.revision = revision + r.letGo = make(map[string]secretref.Key, len(optional)) for _, k := range optional { - s.letGo[StoreName(k)] = k + r.letGo[StoreName(k)] = k } - return s + return s.withReadings(r) +} + +// Pending is s recording the reading of a document the caller is about to +// write, which lets go without the Secret keys optional, by hash, the hash the +// caller gives that document, which must tell it from every other document it +// writes: should a read find the revision the engine stored for the write +// running before the caller has recorded its activation (its answer lost, or +// the caller's record of it), Confirmed records that revision's reading from +// it. The caller records it, and saves s, before the write is sent, since a +// write whose answer is lost leaves nothing else to say how its document reads +// the store. +func (s State) Pending(hash string, optional []secretref.Key) State { + letGo := make(map[string]secretref.Key, len(optional)) + for _, k := range optional { + letGo[StoreName(k)] = k + } + pending := maps.Clone(s.pending()) + if pending == nil { + pending = map[string]map[string]secretref.Key{} + } + pending[hash] = letGo + r := s.reading() + r.pending = pending + return s.withReadings(r) +} + +// Pends reports whether s records the reading of the document hash names +// (Pending). +func (s State) Pends(hash string) bool { + _, ok := s.pending()[hash] + return ok +} + +// Confirmed is s recording revision, which the caller found running the +// document whose reading Pending recorded under hash, as reading the store +// that way, as Adopted records a revision of a document the caller renders: +// without settling a withdrawal or a write, which only a read of the epoch the +// revision activated as settles (Ran). ok is false where s records no reading +// under hash, and s is returned unchanged. +func (s State) Confirmed(revision, hash string) (next State, ok bool) { + letGo, ok := s.pending()[hash] + if !ok || revision == "" { + return s, false + } + r := s.reading() + r.revision, r.letGo = revision, letGo + return s.withReadings(r), true +} + +// KeepPending is s recording only the readings Pending recorded under hashes: +// the caller's record of the rest says no revision it has not recorded can be +// that write. +func (s State) KeepPending(hashes []string) State { + var kept map[string]map[string]secretref.Key + for _, hash := range hashes { + if letGo, ok := s.pending()[hash]; ok { + if kept == nil { + kept = map[string]map[string]secretref.Key{} + } + kept[hash] = letGo + } + } + r := s.reading() + r.pending = kept + return s.withReadings(r) } // Recorded is the revision whose reading s records, empty for none. func (s State) Recorded() string { - return s.revision + return s.reading().revision } // Records reports whether s records how revision reads the store: whether // LetGo answers for it. func (s State) Records(revision string) bool { - return revision != "" && revision == s.revision + return revision != "" && revision == s.reading().revision } // LetGo are the store names the document of revision lets go without, every @@ -304,7 +503,7 @@ func (s State) LetGo(revision string) []string { if !s.Records(revision) { return nil } - return slices.Sorted(maps.Keys(s.letGo)) + return slices.Sorted(maps.Keys(s.reading().letGo)) } // LetGoKeys are the Secret keys LetGo names the store entries of, sorted as @@ -315,13 +514,13 @@ func (s State) LetGoKeys(revision string) []secretref.Key { if !s.Records(revision) { return nil } - return slices.SortedFunc(maps.Values(s.letGo), secretref.Compare) + return slices.SortedFunc(maps.Values(s.reading().letGo), secretref.Compare) } // letsGo reports whether the document of revision lets go without name, as // LetGo says. func (s State) letsGo(revision, name string) bool { - _, ok := s.letGo[name] + _, ok := s.reading().letGo[name] return ok && s.Records(revision) } @@ -343,6 +542,7 @@ func (s *State) withdraw(name string) { s.withdrawn = map[string]int64{} } s.withdrawn[name] = unread + s.changedAgain(name) } // write records a value written under name in s, which it changes in place, @@ -352,6 +552,20 @@ func (s *State) write(name string) { s.written = map[string]int64{} } s.written[name] = unread + s.changedAgain(name) +} + +// changedAgain records in s, which it changes in place, that name changed +// after every activation Awaiting recorded for it: none of them read it now. +func (s *State) changedAgain(name string) { + if _, ok := s.awaits[name]; ok { + awaits := maps.Clone(s.awaits) + delete(awaits, name) + if len(awaits) == 0 { + awaits = nil + } + s.awaits = awaits + } } // deleted records name removed from the store in s, which it changes in @@ -363,13 +577,13 @@ func (s *State) deleted(name string) { } // carried is a State that starts from s: its key, its record of withdrawals, -// of writes and of the activated revision, and what it last retained, in -// maps of its own where a Sync or a Withdraw changes them in place, and -// fingerprints. +// of writes and the revisions they await, and of how documents read the +// store, and what it last retained, in maps of its own where a Sync or a +// Withdraw changes them in place, and fingerprints. func (s State) carried(fingerprints map[string][]byte) State { return State{ key: s.key, fingerprints: fingerprints, withdrawn: maps.Clone(s.withdrawn), written: maps.Clone(s.written), - revision: s.revision, letGo: s.letGo, retained: s.retained, + awaits: s.awaits, readings: s.readings, retained: s.retained, } } @@ -487,10 +701,74 @@ func LoadState(ctx context.Context, r client.Reader, ref types.NamespacedName, o // let go or retained. s.withdrawn = markLines(secret.Data[withdrawnField]) s.written = markLines(secret.Data[writtenField]) - s.revision = string(secret.Data[revisionField]) - s.letGo = keyLines(secret.Data[letGoField]) + s.awaits = awaitLines(secret.Data[awaitField]) + s.awaits = s.awaited() s.retained = ownedLines(secret.Data[retainedField]) - return s, nil + return s.withReadings(readingRecord{ + revision: string(secret.Data[revisionField]), letGo: keyLines(secret.Data[letGoField]), + pending: pendingOf(secret.Data[pendingField]), + }), nil +} + +// pendingRecord is one reading State.Pending recorded, as the state Secret +// holds it: Secret keys as secretref.Key prints them. +type pendingRecord struct { + Optional []string `json:"optional"` +} + +// pendingOf reads the readings State.Pending recorded from data, nil for none. +// What does not decode is dropped, as a malformed line of the other fields +// is: only a hand edit writes it, and the cost is the one State.Confirmed +// would have saved, a revision read as requiring every name it references. +func pendingOf(data []byte) map[string]map[string]secretref.Key { + var records map[string]pendingRecord + if json.Unmarshal(data, &records) != nil { + return nil + } + var pending map[string]map[string]secretref.Key + for hash, r := range records { + if hash == "" { + continue + } + letGo := map[string]secretref.Key{} + for _, line := range r.Optional { + if k, ok := parseKey(line); ok { + letGo[StoreName(k)] = k + } + } + if pending == nil { + pending = map[string]map[string]secretref.Key{} + } + pending[hash] = letGo + } + return pending +} + +// pendingData is s's pending readings as the state Secret holds them, keyed +// by hash, and nothing for none, as the other lists are; encoding/json writes +// the keys sorted, so a State saves the same bytes whatever order its maps +// hold them in. +func (s State) pendingData() []byte { + if len(s.pending()) == 0 { + return []byte{} + } + records := make(map[string]pendingRecord, len(s.pending())) + for hash, letGo := range s.pending() { + keys := slices.SortedFunc(maps.Values(letGo), secretref.Compare) + optional := make([]string, len(keys)) + for i, k := range keys { + optional[i] = k.String() + } + records[hash] = pendingRecord{Optional: optional} + } + data, err := json.Marshal(records) + if err != nil { + // A map of strings to a struct of strings always encodes; were it + // ever not to, nothing pending is the State's zero, which costs a + // revision read as requiring every name and nothing else. + return []byte{} + } + return data } // keyLines are the lines of data that are Secret keys as secretref.Key prints @@ -569,6 +847,42 @@ func marked(marks map[string]int64) []byte { return out } +// awaitLines are the lines of data that are a name Owns accepts and a +// revision, the revisions of each name sorted, nil for none. A line that is +// not one is dropped: only a hand edit writes it, and the cost is the one a +// lost record of an unconfirmed activation costs, an activation asked for +// again. +func awaitLines(data []byte) map[string][]string { + var awaits map[string][]string + for line := range strings.Lines(string(data)) { + name, revision, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + if !ok || !Owns(name) || revision == "" || strings.ContainsAny(revision, " \t") || + slices.Contains(awaits[name], revision) { + continue + } + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = append(awaits[name], revision) + } + for name := range awaits { + slices.Sort(awaits[name]) + } + return awaits +} + +// awaitedLines is awaits one change and one revision per line, sorted by +// name and then revision, never nil. +func awaitedLines(awaits map[string][]string) []byte { + out := []byte{} + for _, name := range slices.Sorted(maps.Keys(awaits)) { + for _, revision := range awaits[name] { + out = append(out, name+" "+revision+"\n"...) + } + } + return out +} + // ownedLines are the lines of data that are names Owns accepts, as a set, // nil for none. func ownedLines(data []byte) map[string]struct{} { @@ -619,7 +933,7 @@ func SaveState(ctx context.Context, c client.Client, ref types.NamespacedName, o if err != nil { return fmt.Errorf("finding the kind of the owner of the secret sync state %s: %w", ref, err) } - data := make(map[string][]byte, len(s.fingerprints)+6) + data := make(map[string][]byte, len(s.fingerprints)+7) maps.Copy(data, s.fingerprints) if len(s.key) > 0 { data[fingerprintKeyField] = []byte(s.key) @@ -631,14 +945,17 @@ func SaveState(ctx context.Context, c client.Client, ref types.NamespacedName, o // that lets a name go. data[withdrawnField] = marked(s.withdrawn) data[writtenField] = marked(s.written) - data[revisionField] = []byte(s.revision) - keys := slices.SortedFunc(maps.Values(s.letGo), secretref.Compare) + data[awaitField] = awaitedLines(s.awaited()) + reading := s.reading() + data[revisionField] = []byte(reading.revision) + keys := slices.SortedFunc(maps.Values(reading.letGo), secretref.Compare) printed := make([]string, len(keys)) for i, k := range keys { printed[i] = k.String() } data[letGoField] = lines(printed) data[retainedField] = lines(s.Retained()) + data[pendingField] = s.pendingData() secret := corev1ac.Secret(ref.Name, ref.Namespace). WithLabels(map[string]string{ "app.kubernetes.io/name": "crewlet", diff --git a/internal/secretsync/state_test.go b/internal/secretsync/state_test.go index f63c510e..fd843792 100644 --- a/internal/secretsync/state_test.go +++ b/internal/secretsync/state_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/hmac" "crypto/sha256" + "encoding/json" "errors" "fmt" "maps" @@ -196,6 +197,91 @@ func TestSaveStateKeepsTheWrittenNames(t *testing.T) { } } +// TestARevisionStoredForAnActivationSettlesTheChangesItWasAskedAfter: the +// engine stored a revision for an activation the operator asked for after a +// withdrawal and a write and could not confirm activating it, and no read of +// the fleet's pointer followed either change, so no epoch the pointer names +// later settles them (Ran). Finding that revision running settles both +// (Found), and finding any other settles neither; a change made after the +// activation was asked for waits for one of its own; what is awaited survives +// a save and a load, a line a hand edit wrote is dropped, and nothing of it +// is kept once the changes are settled otherwise. Without the record, the +// operator activated the same document again for changes that activation had +// already read. +func TestARevisionStoredForAnActivationSettlesTheChangesItWasAskedAfter(t *testing.T) { + ctx := context.Background() + c := newStateClient(t) + state := State{key: newFingerprintKey()} + state.withdraw(nameA) + state.write(nameB) + awaiting := state.Awaiting(revision2) + if awaiting.SameMarks(state) { + t.Error("SameMarks() = true for a state that awaits a revision beside one that awaits none") + } + if got := awaiting.Ran(9); !slices.Equal(got.Withdrawn(), []string{nameA}) || !slices.Equal(got.Written(), []string{nameB}) { + t.Fatalf("Ran(9) settled %v and %v of changes no read followed; want both owed", got.Withdrawn(), got.Written()) + } + if got := awaiting.Found(revision1); len(got.Withdrawn()) != 1 || len(got.Written()) != 1 { + t.Errorf("Found(%s) left %v and %v, want both owed: another revision is no proof the request landed", + revision1, got.Withdrawn(), got.Written()) + } + if got := awaiting.Found(revision2); len(got.Withdrawn()) != 0 || len(got.Written()) != 0 || len(got.awaited()) != 0 { + t.Errorf("Found(%s) left %v, %v and awaits %v, want both settled and nothing awaited", + revision2, got.Withdrawn(), got.Written(), got.awaited()) + } + + later := awaiting.carried(nil) + later.write(nameB) + if got := later.Found(revision2); len(got.Withdrawn()) != 0 || !slices.Equal(got.Written(), []string{nameB}) { + t.Errorf("after %s was written again, Found(%s) left %v and %v; want the later write owed alone", + nameB, revision2, got.Withdrawn(), got.Written()) + } + + if err := SaveState(ctx, c, stateRef, company(), awaiting.Awaiting(revision1)); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + if want := nameA + " " + revision1 + "\n" + nameA + " " + revision2 + "\n" + nameB + " " + revision1 + "\n" + nameB + + " " + revision2 + "\n"; string(secret.Data[awaitField]) != want { + t.Fatalf("%s = %q, want %q", awaitField, secret.Data[awaitField], want) + } + secret.Data[awaitField] = append(secret.Data[awaitField], []byte("not-a-store-name r3\n"+nameA+"\n"+nameC+" r3\n")...) + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + want := map[string][]string{nameA: {revision1, revision2}, nameB: {revision1, revision2}} + if !maps.EqualFunc(loaded.awaited(), want, slices.Equal) { + t.Fatalf("awaits after a load = %v, want %v: a malformed line, and one for a change the state does not record, "+ + "dropped", loaded.awaited(), want) + } + if got := loaded.Found(revision1); len(got.Withdrawn()) != 0 || len(got.Written()) != 0 { + t.Errorf("Found(%s) after a load left %v and %v, want both settled", revision1, got.Withdrawn(), got.Written()) + } + + for name, settled := range map[string]State{ + "Reloaded": loaded.Reloaded(), + "Reactivated": loaded.Reactivated(), + "Ran": loaded.Observed(4).Ran(5), + } { + if err := SaveState(ctx, c, stateRef, company(), settled); err != nil { + t.Fatalf("SaveState: %v", err) + } + saved := awaitLines(stateSecret(t, c).Data[awaitField]) + for awaited := range saved { + if !slices.Contains(settled.Withdrawn(), awaited) && !slices.Contains(settled.Written(), awaited) { + t.Errorf("after %s the state Secret holds %s for %s, a change it settled", name, awaitField, awaited) + } + } + if name == "Reactivated" && !slices.Equal(saved[nameB], []string{revision1, revision2}) { + t.Errorf("after Reactivated %s awaits %v, want the write still awaiting both revisions", nameB, saved[nameB]) + } + } +} + // TestActivationIsRecordedForItsRevisionAlone: what Activated records of the // operator's last activation answers for that revision and for no other, an // empty one included, the Secret keys it lets go as well as their names, @@ -264,7 +350,7 @@ func TestActivationIsRecordedForItsRevisionAlone(t *testing.T) { if err := SaveState(ctx, c, stateRef, company(), State{key: next.key}); err != nil { t.Fatalf("SaveState: %v", err) } - for _, field := range []string{revisionField, letGoField, retainedField} { + for _, field := range []string{revisionField, letGoField, retainedField, pendingField, awaitField} { if got, ok := stateSecret(t, c).Data[field]; !ok || len(got) != 0 { t.Errorf("the state Secret holds %s = %q (present: %t), want it empty, and owned", field, got, ok) } @@ -296,6 +382,129 @@ func TestAdoptionRecordsARevisionWithoutSettlingAWithdrawal(t *testing.T) { } } +// TestAPendingReadingRecordsTheRevisionFoundRunningIt: the reading of a +// document about to be written is recorded under the hash the caller gives it +// (Pending), and once the caller finds a revision running that document +// (Confirmed) the State records that revision's reading from it, as Adopted +// records a revision of a document the caller renders: LetGo answers the +// document's optional keys for it, and no withdrawal or write is settled, +// since only a read of the epoch the revision activated as says it came after +// them. The readings stay for the caller to drop, a hash never pending +// confirms nothing, KeepPending keeps only the hashes it names, and a Sync +// carries what is pending. +func TestAPendingReadingRecordsTheRevisionFoundRunningIt(t *testing.T) { + const hash2, hash3 = "hash-of-revision-2", "hash-of-revision-3" + base := State{key: newFingerprintKey()}.Activated(revision1, nil) + base.withdraw(nameA) + base.write(nameB) + pending := base.Pending(hash2, []secretref.Key{keyC}).Pending(hash3, nil) + if len(base.pending()) != 0 || base.Pends(hash2) { + t.Fatalf("Pending changed the State it was made from: %v", base.pending()) + } + if got := slices.Sorted(maps.Keys(pending.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Fatalf("pending = %v, want [%s %s]", got, hash2, hash3) + } + if !pending.Pends(hash2) || !pending.Pends(hash3) || pending.Pends(revision2) { + t.Errorf("Pends(%s), Pends(%s), Pends(%s) = %t, %t, %t; want true, true, false", hash2, hash3, revision2, + pending.Pends(hash2), pending.Pends(hash3), pending.Pends(revision2)) + } + + confirmed, ok := pending.Confirmed(revision2, hash2) + if !ok { + t.Fatalf("Confirmed(%s, %s) found no pending reading", revision2, hash2) + } + if confirmed.Recorded() != revision2 || !slices.Equal(confirmed.LetGo(revision2), []string{nameC}) { + t.Errorf("after the confirmation Recorded() = %s and LetGo(%s) = %v, want %s and [%s]", + confirmed.Recorded(), revision2, confirmed.LetGo(revision2), revision2, nameC) + } + if !slices.Equal(confirmed.Withdrawn(), []string{nameA}) || !slices.Equal(confirmed.Written(), []string{nameB}) { + t.Errorf("after the confirmation Withdrawn() = %v and Written() = %v, want [%s] and [%s]: a confirmation "+ + "settles nothing", confirmed.Withdrawn(), confirmed.Written(), nameA, nameB) + } + if got := slices.Sorted(maps.Keys(confirmed.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Errorf("the confirmation left %v pending, want [%s %s]: dropping them is the caller's", got, hash2, hash3) + } + if pending.Recorded() != revision1 { + t.Errorf("Confirmed changed the State it was made from: Recorded() = %s", pending.Recorded()) + } + for _, c := range []struct{ revision, hash string }{{revision2, "no-such-hash"}, {"", hash2}} { + if same, ok := pending.Confirmed(c.revision, c.hash); ok || same.Recorded() != revision1 { + t.Errorf("Confirmed(%q, %q) = %t, recorded %s; want nothing changed", c.revision, c.hash, ok, same.Recorded()) + } + } + kept := pending.KeepPending([]string{hash3, "no-such-hash"}) + if got := slices.Sorted(maps.Keys(kept.pending())); !slices.Equal(got, []string{hash3}) { + t.Errorf("KeepPending([%s no-such-hash]) keeps %v pending, want [%s]", hash3, got, hash3) + } + if len(pending.pending()) != 2 { + t.Errorf("KeepPending changed the State it was made from: %v", pending.pending()) + } + + eng := newFakeEngine() + synced, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), base.Pending(hash2, []secretref.Key{keyC})) + if after, ok := synced.Confirmed(revision2, hash2); !ok || !slices.Equal(after.LetGo(revision2), []string{nameC}) { + t.Errorf("Confirmed(%s, %s) after a Sync = %t, let go %v; want the reading carried", revision2, hash2, ok, + after.LetGo(revision2)) + } +} + +// TestPendingReadingsSurviveASaveAndALoad: what Pending records survives a +// save and a load, what only a hand edit writes (a line that is no Secret key, +// a reading under no hash) is dropped, and a record that is not JSON at all is +// nothing pending. +func TestPendingReadingsSurviveASaveAndALoad(t *testing.T) { + ctx := context.Background() + const hash2, hash3 = "hash-of-revision-2", "hash-of-revision-3" + pending := State{key: newFingerprintKey()}.Activated(revision1, nil). + Pending(hash2, []secretref.Key{keyC}).Pending(hash3, nil) + c := newStateClient(t) + if err := SaveState(ctx, c, stateRef, company(), pending); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + var records map[string]map[string]any + if err := json.Unmarshal(secret.Data[pendingField], &records); err != nil { + t.Fatalf("the state Secret holds %s = %q, not JSON: %v", pendingField, secret.Data[pendingField], err) + } + records[hash2]["optional"] = append(records[hash2]["optional"].([]any), "not-a-key") + records[""] = map[string]any{"optional": []any{keyC.String()}} + var err error + if secret.Data[pendingField], err = json.Marshal(records); err != nil { + t.Fatal(err) + } + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if got := slices.Sorted(maps.Keys(loaded.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Fatalf("pending after a load = %v, want [%s %s]", got, hash2, hash3) + } + if got := slices.Sorted(maps.Keys(loaded.pending()[hash2])); !slices.Equal(got, []string{nameC}) { + t.Errorf("the reading pending under %s after a load lets go %v, want [%s]", hash2, got, nameC) + } + if len(loaded.pending()[hash3]) != 0 { + t.Errorf("the reading pending under %s after a load lets go %v, want nothing", hash3, loaded.pending()[hash3]) + } + again, _ := loaded.Confirmed(revision2, hash2) + want, _ := pending.Confirmed(revision2, hash2) + if !slices.Equal(again.LetGoKeys(revision2), want.LetGoKeys(revision2)) { + t.Errorf("Confirmed(%s, %s) after a load lets go %v, want %v", revision2, hash2, again.LetGoKeys(revision2), + want.LetGoKeys(revision2)) + } + + secret = stateSecret(t, c) + secret.Data[pendingField] = []byte("{not json") + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + if loaded, err = LoadState(ctx, c, stateRef, company()); err != nil || len(loaded.pending()) != 0 { + t.Errorf("LoadState over a pending record that is not JSON = %v, %v; want nothing pending", loaded.pending(), err) + } +} + // TestSaveStateForgetsWhatTheStateDropped: an entry the state no longer has // must leave the Secret too, or a later load would vouch for a name the // store no longer holds. @@ -312,7 +521,8 @@ func TestSaveStateForgetsWhatTheStateDropped(t *testing.T) { t.Fatalf("SaveState: %v", err) } got := slices.Sorted(maps.Keys(stateSecret(t, c).Data)) - want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, writtenField, revisionField, letGoField, retainedField})) + want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, writtenField, revisionField, + letGoField, retainedField, pendingField, awaitField})) if !slices.Equal(got, want) { t.Fatalf("state Secret keys = %v, want %v", got, want) } From 3b579bef804afe5037b44c235e560529c3c3a849 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 20:31:31 +0000 Subject: [PATCH 19/36] fix(controller): hold a company whose volume is under the stream floor The EngineClass floors stop a new class asking for less than 11Gi, or 14Gi with the files in the stream's own bucket, but a company keeps the claim it was made with (dataVolumeSize reads the StatefulSet's own claim template), and the API server's validation ratcheting lets a class stored at an older floor go on updating, and stamping its claims on new companies. Under crewlet#142 such a volume's stream cannot hold what a fresh node reserves (its nine state logs at their floors, 8120172544 bytes, and with the bucket 10826896725), and Crewlet refuses to boot on it wherever it has a log to create: a fresh node's every log, and the chat log #142 adds to a stream made before it, which a main-era 9Gi or 12Gi volume cannot fit. Nothing named the volume: the operator rolled the pod onto that engine and reported PodsNotReady over a restart loop. config.StoreMaxBytesFloor stays 6 GiB, so the document itself is valid and crewlet validate only advises; what refuses is the broker. The render already refuses, and the controller already holds the pods back on, the start-time refusals it can see coming (an *AddressError, an unacknowledged password-only sign-in). This follows that idiom: Bootstrap refuses a volume whose share is under what a fresh node of the class reserves (freshNodeFloor: the logs alone with the files in S3, the bucket beside them otherwise) with a *VolumeError (ErrVolumeTooSmall) carrying the volume, its stream, that reservation and the least whole GiB volume whose share holds it (leastDataVolume: 11Gi and 14Gi, which TestContractStreamFloor holds to the CRD's floors). The two figures move from the floor test into the render, where they are now read, and the floor test renders its below-floor case at the floor with the smaller stream set by hand, since the render refuses the smaller volume. The controller reports RuntimeReady=False with the new reason DataVolumeTooSmall, keeping the pods and their bootstrap Secret as they are. The message names the volume, the size it needs and what to grow (volumeTooSmallMessage). A company that has a data claim, or a StatefulSet that makes one, is told the claim to grow, and growing it clears the hold once the expansion lands. A new company made from a class stored under an old floor has neither: its volume would be the class's, so it is told to set the class's spec.data.storage.size, at which the operator then creates its volume, since there is no claim to grow. The hold cannot keep running an engine whose image already moved to one that cannot reserve its logs; what it changes is that the cause is named. TestBootstrapRefusesAVolumeUnderTheStreamFloor covers, for each place a class keeps its files, the main-era floors (12Gi and 9Gi), a volume a GiB under the new one, a byte under the least volume whose share holds the reservation, and zero, and that the least such volume and the floors render; it goes red with the render's check off and with the S3 class held to the bucket's figure. A Company runtime envtest spec holds a company whose StatefulSet was made with a 12Gi template back with the reason and its message, its template and bootstrap Secret untouched, then runs it on the 14Gi its claim was grown to. TestAVolumeUnderTheFloorNamesWhatToGrow holds the message to the claim where one exists and to the class where none does, and fails with the class's remedy dropped. The reason is in the troubleshooting table, by both remedies, and the runtime page's list of holds, and the troubleshooting bullet for the broker's refusal now says the operator holds such a company. TestBootstrapBoundsTheStreamByTheVolume's smallest case moves from 12Gi, which the render now refuses, to 14Gi. The reason's comment quotes both figures, so TestStreamFloorFiguresAreTheEngines lists the API's common_types.go with them, under the mirror table's constant for that path (commonTypesGo). Signed-off-by: Ali Yousefian --- api/v1alpha1/common_types.go | 22 ++++++ docs/concepts/runtime.md | 19 ++++- docs/guides/troubleshooting.md | 5 +- internal/controller/company_runtime.go | 9 +++ internal/controller/company_runtime_test.go | 72 ++++++++++++++++++ internal/controller/company_workload.go | 36 +++++++++ internal/controller/company_workload_test.go | 65 ++++++++++++++++ internal/render/bootstrap.go | 78 ++++++++++++++++++++ internal/render/bootstrap_test.go | 62 +++++++++++++++- internal/render/streamfloor_test.go | 51 +++++++++---- 10 files changed, 400 insertions(+), 19 deletions(-) diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 2b55f584..8f8c3bc0 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -483,6 +483,28 @@ const ( // what to change. ReasonAddressRefused = "AddressRefused" + // ReasonDataVolumeTooSmall is RuntimeReady=False when the company's data + // volume gives the engine an event stream (three quarters of it, Crewlet's + // stream.store_max_bytes) under what a fresh node of Crewlet reserves on + // it: its nine state logs at their floors, 8120172544 bytes, and with the + // files in the stream's own bucket that bucket beside them, 10826896725 + // bytes in all. The EngineClass API admits no data.storage.size under + // 11Gi, or 14Gi with the files in the bucket, so what reaches this is a + // volume claimed under an older floor: a running company's, whose claim + // template cannot change, or one a class stored before the floor rose + // stamps on a new company. Crewlet refuses to boot on such a stream + // wherever it has a state log to create, a fresh node's every log and the + // chat log an engine adds to a stream made before it, so the operator + // holds the pods back as they are rather than roll an engine that cannot + // boot; an engine whose image has already moved still cannot run on the + // volume, and what the hold changes is that the cause is named. The + // message names the volume, the stream it gives, what a fresh node + // reserves and the least volume that holds it, and what clears it: + // growing the claim (docs/concepts/runtime.md, "Growing a company's data + // volume"), or, for a new company with no claim yet, raising its class's + // data.storage.size, at which the operator then creates its volume. + ReasonDataVolumeTooSmall = "DataVolumeTooSmall" + // ReasonNoAlarms is EngineHealthy=True: the engine raises no alarm. ReasonNoAlarms = "NoAlarms" diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index 99f4ddbf..5e970159 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -521,8 +521,18 @@ A company's event stream is three quarters of its data volume, and the engine refuses to boot on a stream smaller than what a fresh node reserves ([the class's floor](company-document.md)). A volume made under an older floor, or before an engine that keeps more on its stream, can fall short, and -a company that keeps more than its volume holds needs room too. Grow the -claim, not the class: +a company that keeps more than its volume holds needs room too. The operator +holds the pods of a company whose volume falls short as they are, with +`RuntimeReady=False`, reason `DataVolumeTooSmall`, naming the volume and its +claim, rather than roll them onto an engine that cannot reserve its state +logs; a pod whose image has already moved to such an engine restarts all the +same, and the condition says why. A new company made from a class stored +before the floor rose, which the API server lets keep a size it no longer +admits until the size itself is changed, has no claim yet: the condition +names the class instead, and setting the class's `data.storage.size` to at +least its floor (`14Gi`, or `11Gi` with the files in S3) clears it, since the +operator then creates the company's volume at that size. A company that has +its claim grows the claim, not the class: 1. The claim's StorageClass must allow it (`allowVolumeExpansion: true`); the API server refuses an expansion otherwise. @@ -599,7 +609,10 @@ engine would be told it is reached at that Crewlet refuses to start on class states it, so this is in practice the class's API route made of a Company's namespace or name, whose last label is a number as `{{name}}.{{namespace}}`'s is in a namespace named `4711` or `0x1f` ([Reach -the API from elsewhere](../guides/exposure.md#reach-the-api-from-elsewhere)). +the API from elsewhere](../guides/exposure.md#reach-the-api-from-elsewhere)), +or a data volume whose stream is under what a fresh node reserves on it +(`DataVolumeTooSmall`, [Growing a company's data +volume](#growing-a-companys-data-volume)). Suspending still scales the pod to zero meanwhile. --- diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 1e027250..4b5beb40 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -79,6 +79,7 @@ with your last change yet. | | False | `APITokensInvalid` | `-api-tokens`, as a platform supplied it, lacks a token or holds one shorter than 43 characters or outside base64, base64url and hex. Repair it, or delete it to have the operator generate it. The pod is held as it is. | | | False | `InsecureSignInNotAcknowledged` | The class lets people sign in with a password alone (`spec.signIn.secondFactor: Optional`), and the Company is reached at an address Crewlet does not read as loopback, which the message names: Crewlet refuses to start so without the class's `spec.signIn.acceptInsecure: true`. The platform acknowledges it on the class or requires a second factor again ([Two-step verification](people.md#two-step-verification)). The pod is held as it is. | | | False | `AddressRefused` | An address the engine would be told it is reached at is one Crewlet refuses to start on; the message names it, its key in `crewlet.yaml` and where it came from. In practice it is the class's API route: its `api.hostname` template's last label holds `{{namespace}}` or `{{name}}`, and the Company's namespace is a number, such as `4711` or `0x1f`, or its namespace or name makes one with the characters beside it (`0x{{name}}` for a Company named `cafe`), so the host's last label is a number, which a browser reads as an IPv4 address. The platform ends the template in a domain of its own (`{{name}}.{{namespace}}.crewlet.example.com`), or the Company runs in a namespace of another name ([Reach the API from elsewhere](exposure.md#reach-the-api-from-elsewhere)). An address the Company states is refused by the API when it is written instead, unless it was admitted before the API refused it: the message then says to correct the Company's field. An artifact origin that meets the dashboard's is refused here too: give artifacts a domain of their own. The pod is held as it is. | +| | False | `DataVolumeTooSmall` | The data volume gives the engine an event stream, three quarters of it, under what a fresh node reserves on it, its state logs at their floors and, with the files in the stream's own bucket, that bucket beside them. The class's `data.storage.size` admits no less than `14Gi` (`11Gi` with the files in S3), so this is a volume sized under an older floor, and the message names what to grow. A company that has its claim (one that kept it, or one created again over the claim a company deleted before kept) is told the volume and the claim: [grow the claim](../concepts/runtime.md#growing-a-companys-data-volume) to at least the size the message names, and the pod runs once the expansion lands. Until then it is held as it is, which keeps an engine that still starts on the volume running, though not one whose image has already moved to an engine that cannot reserve its state logs there. A new company made from a class stored before the floor rose has no claim yet and is told the class: set its `spec.data.storage.size` to at least that size, and the operator creates the company's volume at that size. | | `Exposed` | True | `RouteAccepted` | Every route the class asks for was accepted. When the message adds that the engine is told people reach it at `http://localhost:8080`, every link it gives a person, an invitation's included, leads there through a port-forward: set the Company's `spec.exposure.dashboardBaseURL`, or route the API ([Where people reach the dashboard](exposure.md#where-people-reach-the-dashboard)). | | | True | `NoRoutes` | The class routes nothing (webhooks `None` or `External`, API `ClusterInternal`, artifacts `None`). The message says the same about the dashboard's address as under `RouteAccepted`. | | | False | `RouteNotAccepted` | A Gateway has not attached the HTTPRoute, or the Ingress controller has not published an address. The message carries the controller's own reason; a Gateway listener's `allowedRoutes` that does not admit the tenant's namespace is the usual cause. | @@ -209,7 +210,9 @@ kubectl logs acme-data-0 -n acme | grep -E 'api_bound|engine_recovered|api_liste message names the figure it needed and breaks it down. The class's `data.storage.size` admits no less than `14Gi` (`11Gi` with the files in S3), so this is a company whose volume was made smaller: under an older - floor, or before an engine that keeps more on its stream. + floor, or before an engine that keeps more on its stream. The operator holds such a company's pods as they + are (`RuntimeReady=False DataVolumeTooSmall`, above), which cannot stop a + pod whose image moved under it from restarting. [Grow its volume](../concepts/runtime.md#growing-a-companys-data-volume). - **`config_apply_failed`**: the revision was stored but this node could not start it; the message says why. The engine keeps running the previous diff --git a/internal/controller/company_runtime.go b/internal/controller/company_runtime.go index f5f3c292..08585a0a 100644 --- a/internal/controller/company_runtime.go +++ b/internal/controller/company_runtime.go @@ -234,6 +234,15 @@ func (r *CompanyReconciler) runWorkload( // pods keep the one they run, which named an address it took. return holdBack(crewletaiv1alpha1.ReasonAddressRefused, addressRefusalMessage(c, class, refused)) } + if small, ok := errors.AsType[*render.VolumeError](err); ok { + // The engine could not reserve its state logs on this stream at a + // start with a log to create, so the pods keep the one they run: an + // engine whose logs the stream already holds runs on, and one that + // already refuses it at least has its cause named here rather than + // in a restart loop's previous log. + return holdBack(crewletaiv1alpha1.ReasonDataVolumeTooSmall, + volumeTooSmallMessage(c, class, existing != nil || liveClaim(claims), small)) + } if errors.Is(err, render.ErrSignInNotAcknowledged) { // The engine would refuse this crewlet.yaml at every start, so the // pods keep the one they run, which requires a second factor or diff --git a/internal/controller/company_runtime_test.go b/internal/controller/company_runtime_test.go index 94855c8f..d066b321 100644 --- a/internal/controller/company_runtime_test.go +++ b/internal/controller/company_runtime_test.go @@ -1128,6 +1128,78 @@ var _ = Describe("Company runtime", func() { "the startup window covers the recovery of the volume the pod has") }) + It("holds the pods back on a data volume under what a fresh node reserves, and runs them once its claim grows", func() { + class := newClass() + company := reconciled(newCompany(class.Name)) + // A company claimed under an older floor: its StatefulSet as the + // operator made it then, with a 12Gi claim template. A template + // cannot change, so the StatefulSet is made again around one, as + // the operator would have made it. (envtest runs no garbage + // collector, so deleting it leaves nothing else to go with it.) + made := statefulSet() + old := made.DeepCopy() + old.ObjectMeta = metav1.ObjectMeta{ + Namespace: made.Namespace, Name: made.Name, Labels: made.Labels, Annotations: made.Annotations, + OwnerReferences: made.OwnerReferences, + } + old.Status = appsv1.StatefulSetStatus{} + old.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("12Gi") + Expect(k8sClient.Delete(ctx, made)).To(Succeed()) + Eventually(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(made), &appsv1.StatefulSet{})) + }).Should(BeTrue()) + Expect(k8sClient.Create(ctx, old)).To(Succeed()) + running := statefulSet() + bootstrap := &corev1.Secret{} + get(bootstrap, bootstrapName) + tierA := bootstrap.Data[configFile] + + By("holding the pods as they are, naming the volume and what clears it") + company = reconciled(company) + runtimeReady := condition(company, crewletaiv1alpha1.ConditionRuntimeReady) + Expect(runtimeReady.Status).To(Equal(metav1.ConditionFalse)) + Expect(runtimeReady.Reason).To(Equal(crewletaiv1alpha1.ReasonDataVolumeTooSmall)) + Expect(runtimeReady.Message).To(ContainSubstring("The data volume is 12Gi")) + Expect(runtimeReady.Message).To(ContainSubstring("is 9663676416 bytes")) + Expect(runtimeReady.Message).To(ContainSubstring("reserves 10826896725 bytes")) + Expect(runtimeReady.Message).To(ContainSubstring("Grow the claim " + dataClaimName + " to at least 14Gi")) + Expect(statefulSet().Spec.Template).To(Equal(running.Spec.Template), "the pods rolled onto a crewlet.yaml the engine refuses") + get(bootstrap, bootstrapName) + Expect(bootstrap.Data[configFile]).To(Equal(tierA), "the bootstrap Secret was rewritten") + + By("running them on the stream of the claim its owner grew, once the expansion lands") + expandable := &storagev1.StorageClass{ + Name: "expandable-" + namespace, + Provisioner: "example.com/expandable", + AllowVolumeExpansion: new(true), + } + Expect(k8sClient.Create(ctx, expandable)).To(Succeed()) + DeferCleanup(func() { Expect(k8sClient.Delete(ctx, expandable)).To(Succeed()) }) + claim := &corev1.PersistentVolumeClaim{ + Namespace: namespace, + Name: dataClaimName, + Labels: labelsFor(company, componentData), + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &expandable.Name, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("14Gi")}, + }, + }, + } + Expect(k8sClient.Create(ctx, claim)).To(Succeed()) + claim.Status.Phase = corev1.ClaimBound + claim.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("14Gi")} + Expect(k8sClient.Status().Update(ctx, claim)).To(Succeed()) + company = reconciled(company) + Expect(condition(company, crewletaiv1alpha1.ConditionRuntimeReady).Reason).To(Equal(crewletaiv1alpha1.ReasonPodsNotReady)) + get(bootstrap, bootstrapName) + Expect(string(bootstrap.Data[configFile])).To(ContainSubstring("store_max_bytes: 11274289152"), + "the event stream is three quarters of the 14Gi the claim now has") + Expect(statefulSet().Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().String()).To(Equal("12Gi"), + "the template stays as it was made") + }) + It("rolls the pods when a value they read at start changes, and only then", func() { class := newClass() reconciled(newCompany(class.Name)) diff --git a/internal/controller/company_workload.go b/internal/controller/company_workload.go index 33c45588..8b402b7a 100644 --- a/internal/controller/company_workload.go +++ b/internal/controller/company_workload.go @@ -931,6 +931,42 @@ func applyConfigOf[T any](v any) (*T, error) { return out, nil } +// volumeTooSmallMessage is RuntimeReady's account of a data volume whose +// share, the engine's event stream, is under what a fresh node of Crewlet +// reserves on it (small), with what clears it, which depends on where the volume's size +// comes from (dataVolumeSize). Where the company has a claim, or a data +// StatefulSet that makes one from a template that never changes (claimed), +// the claim is what is grown. Where it has neither, the volume is the one its +// class would give a new company, which a class stored before the floor rose +// still can, since the API server's ratcheting keeps a stored value it no +// longer admits: there is no claim to grow, so the class's size is what is +// raised. +func volumeTooSmallMessage( + c *crewletaiv1alpha1.Company, class *crewletaiv1alpha1.EngineClass, claimed bool, small *render.VolumeError, +) string { + sizes := fmt.Sprintf("three quarters of it, the engine's event stream, is %d bytes: a fresh node of Crewlet "+ + "reserves %d bytes of it for its state logs, and refuses to boot without them, so a data volume needs at "+ + "least %s.", small.Stream, small.Floor, small.Least.String()) + if !claimed { + return fmt.Sprintf("EngineClass %s gives a new company a data volume of %s, and %s Set the class's "+ + "spec.data.storage.size to at least %s, and the operator creates the company's data volume at that size. "+ + "Nothing of the company's data pods is created until then.", + class.Name, small.Volume.String(), sizes, small.Least.String()) + } + return fmt.Sprintf("The data volume is %s, and %s Grow the claim %s to at least %s (its StorageClass must allow "+ + "volume expansion; see \"Growing a company's data volume\" in the operator's docs/concepts/runtime.md), and the "+ + "operator sizes the stream from it once the expansion lands. The pods are held as they are.", + small.Volume.String(), sizes, dataVolume+"-"+dataStatefulSetName(c)+"-0", small.Least.String()) +} + +// liveClaim reports whether claims holds a data volume claim not being +// deleted: one a pod runs on, or will, whatever made it. +func liveClaim(claims []corev1.PersistentVolumeClaim) bool { + return slices.ContainsFunc(claims, func(claim corev1.PersistentVolumeClaim) bool { + return claim.DeletionTimestamp.IsZero() + }) +} + // existingStatefulSet returns the data StatefulSet, or nil when there is none. // The cache holds only the StatefulSets the operator manages (CacheOptions), // so when it has none the API server is asked: a StatefulSet of that name diff --git a/internal/controller/company_workload_test.go b/internal/controller/company_workload_test.go index d74fe289..a59d9b57 100644 --- a/internal/controller/company_workload_test.go +++ b/internal/controller/company_workload_test.go @@ -194,6 +194,71 @@ func TestDataVolumeSize(t *testing.T) { } } +// TestAVolumeUnderTheFloorNamesWhatToGrow holds DataVolumeTooSmall's message +// to the remedy that exists. A company with a data claim, or a StatefulSet +// that makes one, is told to grow that claim. One with neither has its volume +// from its class, a class stored before the floor rose (which the API +// server's ratcheting keeps), and is told to raise the class's size: there is +// no claim to grow, and naming one sent its owner looking for a claim that +// does not exist. +func TestAVolumeUnderTheFloorNamesWhatToGrow(t *testing.T) { + t.Parallel() + company := &crewletaiv1alpha1.Company{Namespace: "tenant-a", Name: companyName} + class := &crewletaiv1alpha1.EngineClass{Name: "small"} + small := &render.VolumeError{Volume: resource.MustParse("12Gi"), Stream: 9663676416, Floor: 10826896725, + Least: resource.MustParse("14Gi")} + for _, c := range []struct { + name string + claimed bool + says, saysNot []string + }{ + { + name: "a company with a claim", claimed: true, + says: []string{"The data volume is 12Gi", "Grow the claim data-" + companyName + "-data-0 to at least 14Gi", "held as they are"}, + saysNot: []string{"EngineClass", "spec.data.storage.size"}, + }, + { + name: "a new company of a class under the floor", + says: []string{"EngineClass small gives a new company a data volume of 12Gi", + "spec.data.storage.size to at least 14Gi", "is 9663676416 bytes", "reserves 10826896725 bytes"}, + saysNot: []string{"Grow the claim", "data-" + companyName + "-data-0"}, + }, + } { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + message := volumeTooSmallMessage(company, class, c.claimed, small) + for _, s := range c.says { + if !strings.Contains(message, s) { + t.Errorf("the message does not say %q: %s", s, message) + } + } + for _, s := range c.saysNot { + if strings.Contains(message, s) { + t.Errorf("the message says %q: %s", s, message) + } + } + if !endsOnce(message) { + t.Errorf("the message does not end with one full stop: %s", message) + } + }) + } + deleting := claimRequesting("12Gi") + deleting.DeletionTimestamp = new(metav1.Now()) + for _, c := range []struct { + name string + claims []corev1.PersistentVolumeClaim + live bool + }{ + {"no claim", nil, false}, + {"a claim being deleted", []corev1.PersistentVolumeClaim{deleting}, false}, + {"a claim", []corev1.PersistentVolumeClaim{deleting, claimRequesting("12Gi")}, true}, + } { + if got := liveClaim(c.claims); got != c.live { + t.Errorf("liveClaim of %s = %t, want %t", c.name, got, c.live) + } + } +} + // claimRequesting is a data volume claim that asks for size, not bound. func claimRequesting(size string) corev1.PersistentVolumeClaim { return corev1.PersistentVolumeClaim{ diff --git a/internal/render/bootstrap.go b/internal/render/bootstrap.go index 4e700ec8..1e11f7bb 100644 --- a/internal/render/bootstrap.go +++ b/internal/render/bootstrap.go @@ -190,6 +190,43 @@ var ErrNoAddress = errors.New("a data pod's crewlet.yaml needs the addresses the // caller holds the pods back on it rather than start one that crash-loops. var ErrSignInNotAcknowledged = errors.New("password-only sign-in off loopback needs the class's acknowledgement") +// ErrVolumeTooSmall is a data pod's Tier A asked for with a data volume whose +// share, the stream, is under what a fresh node of Crewlet reserves on it +// (freshNodeFloor). The EngineClass API admits no such size, so what reaches +// it is a volume claimed under an older, smaller floor, kept by a running +// company (its claims cannot change but by an expansion) or stamped by a +// class stored before the floor rose, which the API server's ratcheting still +// lets update. Crewlet refuses to boot on it wherever it has a state log to +// create, and an engine that keeps a log the stream does not hold yet has +// one at every boot, so the caller holds the pods back on it, naming the +// volume, rather than roll an engine that cannot boot; growing the claim is +// what clears it. A *VolumeError carries the sizes. +var ErrVolumeTooSmall = errors.New("crewlet cannot reserve its state logs on the stream a data volume this small gives it") + +// VolumeError is ErrVolumeTooSmall for one volume: its size, the stream three +// quarters of it would give the engine, what a fresh node reserves on it, and +// the least volume that holds that. +type VolumeError struct { + // Volume is the data volume's size, the claim the pods run on. + Volume resource.Quantity + // Stream is the stream.store_max_bytes Volume gives. + Stream int64 + // Floor is what a fresh node of the class reserves on its stream. + Floor int64 + // Least is the least volume, in whole gibibytes, whose stream holds + // Floor. + Least resource.Quantity +} + +func (e *VolumeError) Error() string { + return fmt.Sprintf("%v: a data volume of %s gives the engine a stream of %d bytes, under the %d a fresh node "+ + "reserves on it; a volume needs at least %s", ErrVolumeTooSmall, e.Volume.String(), e.Stream, e.Floor, + e.Least.String()) +} + +// Unwrap makes a VolumeError ErrVolumeTooSmall for errors.Is. +func (e *VolumeError) Unwrap() error { return ErrVolumeTooSmall } + // ErrKeyringInvalid is a Tier A asked for with a keyring the engine could not // use: no key, an active key it does not hold, an id that is not a valid key // id, or material that is not base64 of 32 bytes. The keyring is a Secret's, @@ -375,6 +412,42 @@ const ( streamShareDenominator = 4 ) +// What a fresh node of Crewlet reserves on its event stream before it boots +// (its engine.BrokerFloorFor): its nine state logs' ceilings at their +// floors, and, with the objects in the stream's own bucket, that bucket +// beside them at a quarter of the stream. Crewlet refuses to boot a node on +// a stream that cannot hold them where it has a log to create: one starting +// on an empty stream directory, or one whose engine keeps a log its stream +// does not hold yet. TestContractStreamFloor holds both to the figure +// `crewlet validate` names, and TestStreamFloorFiguresAreTheEngines holds +// every hand-written comment and page that quotes one to these. +const ( + freshNodeLogsFloor int64 = 8120172544 + freshNodeBucketFloor int64 = 10826896725 +) + +// freshNodeFloor is what a fresh node of class reserves on its stream: its +// state logs at their floors where the class keeps its files in S3, and the +// object store's bucket beside them where it keeps them in the stream's own +// bucket, the default. +func freshNodeFloor(class *v1alpha1.EngineClass) int64 { + if class.Spec.Objects.Backend == v1alpha1.ObjectBackendS3 { + return freshNodeLogsFloor + } + return freshNodeBucketFloor +} + +// leastDataVolume is the least data volume, in whole gibibytes, whose share +// (streamShare) holds floor: four quarters of the floor's three, a quarter +// rounded up since the share is taken in whole quarters, then the gibibyte +// rounded up. For freshNodeLogsFloor it is 11Gi and for freshNodeBucketFloor +// 14Gi, the EngineClass's floors on data.storage.size, which +// TestContractStreamFloor holds to it. +func leastDataVolume(floor int64) resource.Quantity { + volume := (floor + streamShareNumerator - 1) / streamShareNumerator * streamShareDenominator + return *resource.NewQuantity((volume+1<<30-1)>>30<<30, resource.BinarySI) +} + // storeMaxBytesCeiling is the largest stream.store_max_bytes Crewlet starts // on (config.StoreMaxBytesCeiling, 64 TiB), a guard against a unit mistake // rather than a policy: it is more than an order of magnitude above what the @@ -456,6 +529,8 @@ const keyringKeyBytes = 32 // // It fails when an address is missing (ErrNoAddress); when an address is one // Crewlet refuses to start on (an *AddressError, ErrAddressRefused); when the +// data volume gives a stream under what a fresh node reserves on it (a +// *VolumeError, ErrVolumeTooSmall); when the // class lets people sign in with a password alone at an ExternalURL that is // not on loopback without acknowledging it (ErrSignInNotAcknowledged); and // when the keyring cannot be used (ErrKeyringInvalid): no keys, an active key @@ -468,6 +543,9 @@ func Bootstrap(in *BootstrapInput) (*BootstrapDocument, error) { if err := refusedAddress(in.ExternalURL, in.PublicURL, in.ArtifactsURL, in.AllowedOrigins); err != nil { return nil, err } + if stream, floor := streamShare(in.VolumeSize), freshNodeFloor(in.Class); stream < floor { + return nil, &VolumeError{Volume: in.VolumeSize, Stream: stream, Floor: floor, Least: leastDataVolume(floor)} + } signIn := in.Class.Spec.SignIn acknowledged := signIn.AcceptInsecure != nil && *signIn.AcceptInsecure if signIn.SecondFactor == v1alpha1.SecondFactorOptional && !acknowledged && !loopback(in.ExternalURL) { diff --git a/internal/render/bootstrap_test.go b/internal/render/bootstrap_test.go index e12c24cc..99d79cc8 100644 --- a/internal/render/bootstrap_test.go +++ b/internal/render/bootstrap_test.go @@ -653,6 +653,16 @@ func TestSeatPodsCarryNoSignIn(t *testing.T) { } } +// The least whole-GiB data volumes whose stream share holds what a fresh node +// reserves, by where a class keeps its files: the broker's bucket beside the +// logs, or S3 (leastDataVolume of freshNodeBucketFloor and +// freshNodeLogsFloor), which the EngineClass's floors on data.storage.size +// are. +const ( + leastBucketVolume = "14Gi" + leastLogsVolume = "11Gi" +) + // TestBootstrapBoundsTheStreamByTheVolume pins the event stream's share of // the volume: three quarters of the claim the pod runs on, whatever the // class says now, since the claim is what the pod was given, and never more @@ -663,7 +673,7 @@ func TestBootstrapBoundsTheStreamByTheVolume(t *testing.T) { size string want int64 }{ - {size: "12Gi", want: 9 << 30}, + {size: leastBucketVolume, want: 14 << 30 / 4 * 3}, {size: "85Ti", want: 85 << 40 / 4 * 3}, {size: "93824992236884", want: 70368744177663}, {size: "93824992236888", want: storeMaxBytesCeiling}, @@ -694,6 +704,56 @@ func TestBootstrapBoundsTheStreamByTheVolume(t *testing.T) { } } +// TestBootstrapRefusesAVolumeUnderTheStreamFloor holds the render to what a +// fresh node of Crewlet reserves on its stream, for each place a class keeps +// its files: a volume whose share is under it, which a company keeps from an +// older floor, is refused as a *VolumeError naming the volume, the stream it +// gives, what a fresh node reserves and the least whole-GiB volume that holds +// it, rather than rendered into a crewlet.yaml no node with a state log to +// create boots on; the least such volume renders. +func TestBootstrapRefusesAVolumeUnderTheStreamFloor(t *testing.T) { + for _, c := range []struct { + bootstrap, size, least string + stream, floor int64 + }{ + {bootstrap: caseNATS, size: "12Gi", stream: 12 << 30 / 4 * 3, floor: freshNodeBucketFloor, least: leastBucketVolume}, + {bootstrap: caseNATS, size: "13Gi", stream: 13 << 30 / 4 * 3, floor: freshNodeBucketFloor, least: leastBucketVolume}, + // A byte under the least volume whose share holds the reservation. + {bootstrap: caseNATS, size: "14435862299", stream: 14435862299 / 4 * 3, floor: freshNodeBucketFloor, + least: leastBucketVolume}, + {bootstrap: caseNATS, size: "0", stream: 0, floor: freshNodeBucketFloor, least: leastBucketVolume}, + {bootstrap: "s3", size: "9Gi", stream: 9 << 30 / 4 * 3, floor: freshNodeLogsFloor, least: leastLogsVolume}, + {bootstrap: "s3", size: "10826896727", stream: 10826896727 / 4 * 3, floor: freshNodeLogsFloor, least: leastLogsVolume}, + } { + t.Run(c.bootstrap+"/"+c.size, func(t *testing.T) { + in := bootstrapCaseNamed(t, c.bootstrap).input(t) + in.VolumeSize = resource.MustParse(c.size) + doc, err := Bootstrap(in) + refused, ok := errors.AsType[*VolumeError](err) + if !ok || !errors.Is(err, ErrVolumeTooSmall) || doc != nil { + t.Fatalf("Bootstrap of a %s volume = %v, %v; want a *VolumeError", c.size, doc, err) + } + if !refused.Volume.Equal(in.VolumeSize) || refused.Stream != c.stream || refused.Floor != c.floor || + refused.Least.String() != c.least { + t.Errorf("the refusal of a %s volume names %s, a stream of %d under %d and a least volume of %s; "+ + "want %s, %d under %d and %s", c.size, refused.Volume.String(), refused.Stream, refused.Floor, + refused.Least.String(), c.size, c.stream, c.floor, c.least) + } + }) + } + for _, c := range []struct{ bootstrap, size string }{ + {bootstrap: caseNATS, size: "14435862300"}, {bootstrap: caseNATS, size: leastBucketVolume}, + {bootstrap: "s3", size: "10826896728"}, {bootstrap: "s3", size: leastLogsVolume}, + } { + in := bootstrapCaseNamed(t, c.bootstrap).input(t) + in.VolumeSize = resource.MustParse(c.size) + if _, err := Bootstrap(in); err != nil { + t.Errorf("Bootstrap refuses a %s volume with files in %s, whose share holds a fresh node: %v", c.size, + c.bootstrap, err) + } + } +} + // TestDataPodsKeepTheirLeafListenerForPinnedSeats holds the data pods' leaf // listener open while the revision the engine runs pins a seat to a seat pod, // whatever the class says now: the engine keeps running that revision while diff --git a/internal/render/streamfloor_test.go b/internal/render/streamfloor_test.go index e38e719e..ea98c72c 100644 --- a/internal/render/streamfloor_test.go +++ b/internal/render/streamfloor_test.go @@ -1,6 +1,7 @@ package render import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -14,17 +15,6 @@ import ( "sigs.k8s.io/yaml" ) -// What a fresh node of Crewlet reserves on its event stream before it boots -// (its engine.BrokerFloorFor): its nine state logs' ceilings at their -// floors, and, with the objects in the stream's own bucket, that bucket -// beside them at a quarter of the stream. TestContractStreamFloor holds both -// to the figure `crewlet validate` names, and TestStreamFloorFiguresAreTheEngines -// holds every hand-written comment and page that quotes one to these. -const ( - freshNodeLogsFloor int64 = 8120172544 - freshNodeBucketFloor int64 = 10826896725 -) - // The EngineClass rules that floor the data volume, as controller-gen writes // them into the CRD: the size's own rule, the least any class boots on, and // the class's rule for a class whose objects are in the stream's bucket. @@ -44,7 +34,10 @@ var ( // as an advisory at stream.store_max_bytes rather than a refusal: the setting // is valid (above config.StoreMaxBytesFloor), and what refuses is the broker, // when the node reserves its logs and its object bucket on an empty stream -// directory. +// directory. And each floor is the least volume, in whole gibibytes, whose +// share holds what a fresh node reserves (leastDataVolume), which is where +// the render refuses a volume as one no node boots on (Bootstrap's +// *VolumeError). func TestContractStreamFloor(t *testing.T) { s3Floor, natsFloor := engineClassFloors(t) for _, c := range []struct { @@ -68,16 +61,45 @@ func TestContractStreamFloor(t *testing.T) { }}}, } { t.Run(c.bootstrap+"/"+v.size.String(), func(t *testing.T) { + // Rendered at the floor, its stream then set to v.size's share: + // the render refuses a volume under the floor (a *VolumeError), + // and what this holds is the engine's word on such a stream. in := bootstrapCaseNamed(t, c.bootstrap).input(t) - in.VolumeSize = v.size + in.VolumeSize = c.floor doc, err := Bootstrap(in) if err != nil { t.Fatal(err) } - validate(t, bootstrapEnv(doc), v.warns, "-tier", "bootstrap", write(t, "crewlet.yaml", doc.YAML)) + var tierA map[string]any + if err := yaml.Unmarshal(doc.YAML, &tierA); err != nil { + t.Fatal(err) + } + tierA["stream"].(map[string]any)["store_max_bytes"] = streamShare(v.size) + encoded, err := json.Marshal(tierA) + if err != nil { + t.Fatal(err) + } + written, err := yamlOf(encoded) + if err != nil { + t.Fatal(err) + } + validate(t, bootstrapEnv(doc), v.warns, "-tier", "bootstrap", write(t, "crewlet.yaml", written)) }) } } + for _, c := range []struct { + bootstrap string + floor resource.Quantity + reserves int64 + }{ + {bootstrap: caseNATS, floor: natsFloor, reserves: freshNodeBucketFloor}, + {bootstrap: "s3", floor: s3Floor, reserves: freshNodeLogsFloor}, + } { + if least := leastDataVolume(c.reserves); c.floor.Cmp(least) != 0 { + t.Errorf("the EngineClass's %s floor of %s is not leastDataVolume of %d, %s, where the render refuses "+ + "a volume", c.bootstrap, c.floor.String(), c.reserves, least.String()) + } + } } // engineClassFloors reads the two floors on an EngineClass's data volume out @@ -133,6 +155,7 @@ func bootstrapCaseNamed(t *testing.T, name string) *bootstrapCase { // the engine, so this is what keeps them the engine's. var streamFloorQuotes = map[string][]int64{ engineClassGo: {freshNodeLogsFloor, freshNodeBucketFloor}, + commonTypesGo: {freshNodeLogsFloor, freshNodeBucketFloor}, "docs/concepts/company-document.md": {freshNodeLogsFloor, freshNodeBucketFloor}, troubleshootingMd: {freshNodeLogsFloor, freshNodeBucketFloor}, "internal/engine/enginetest/enginetest.go": {freshNodeBucketFloor}, From ccadc14557174d84f9691c744405cb2c26e37469 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 31 Oct 2026 00:11:29 +0100 Subject: [PATCH 20/36] docs(api): say a unit's channel is one unit's room on Chatlet Unit.channel's doc says what the field holds and how Crewlet spells it, and since crewlet#142 that on Chatlet each channel is a room whose members follow the org chart. It did not say that such a room belongs to one unit: Crewlet refuses a second unit naming the same channel, so a child unit that should share its parent's room names none, which is the one way to share it. The rule over the field itself, and its test against the engine (TestUnitChannelFollowsTheEngine), came with #51. Signed-off-by: Ali Yousefian --- api/v1alpha1/unit_types.go | 4 +++- config/crd/bases/crewlet.ai_units.yaml | 4 +++- dist/chart/templates/crd/units.crewlet.ai.yaml | 4 +++- docs/reference/api.md | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/unit_types.go b/api/v1alpha1/unit_types.go index bdd3afcb..68f48d06 100644 --- a/api/v1alpha1/unit_types.go +++ b/api/v1alpha1/unit_types.go @@ -156,7 +156,9 @@ type UnitSpec struct { // whose members follow the org chart. Crewlet takes lowercase letters, // digits, - and _, starting with a letter or a digit, up to 64 // characters: the name itself, never #platform or a vendor's channel id. - // Crewlet's `channel`. + // On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + // naming the same channel; a child unit that should share its parent's + // room names none. Crewlet's `channel`. // +optional // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=64 diff --git a/config/crd/bases/crewlet.ai_units.yaml b/config/crd/bases/crewlet.ai_units.yaml index 18735e6e..c2b86a09 100644 --- a/config/crd/bases/crewlet.ai_units.yaml +++ b/config/crd/bases/crewlet.ai_units.yaml @@ -75,7 +75,9 @@ spec: whose members follow the org chart. Crewlet takes lowercase letters, digits, - and _, starting with a letter or a digit, up to 64 characters: the name itself, never #platform or a vendor's channel id. - Crewlet's `channel`. + On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + naming the same channel; a child unit that should share its parent's + room names none. Crewlet's `channel`. maxLength: 64 minLength: 1 pattern: ^[a-z0-9][a-z0-9_-]{0,63}$ diff --git a/dist/chart/templates/crd/units.crewlet.ai.yaml b/dist/chart/templates/crd/units.crewlet.ai.yaml index 746f79bb..2b06b20b 100644 --- a/dist/chart/templates/crd/units.crewlet.ai.yaml +++ b/dist/chart/templates/crd/units.crewlet.ai.yaml @@ -78,7 +78,9 @@ spec: whose members follow the org chart. Crewlet takes lowercase letters, digits, - and _, starting with a letter or a digit, up to 64 characters: the name itself, never #platform or a vendor's channel id. - Crewlet's `channel`. + On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + naming the same channel; a child unit that should share its parent's + room names none. Crewlet's `channel`. maxLength: 64 minLength: 1 pattern: ^[a-z0-9][a-z0-9_-]{0,63}$ diff --git a/docs/reference/api.md b/docs/reference/api.md index 2c1331c7..96ea0be3 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -4177,7 +4177,7 @@ _Appears in:_ | `leadRef` _[LocalObjectReference](#localobjectreference)_ | leadRef names the Seat that leads the unit. Unset inherits the
parent's lead. Crewlet's `lead`. | | Optional: \{\}
| | `purpose` _string_ | purpose is what the unit is for. Crewlet's `purpose`. | | MaxLength: 2048
MinLength: 1
Optional: \{\}
| | `goals` _string array_ | goals are what the unit is working toward. Crewlet's `goals`. | | MaxItems: 32
items:MaxLength: 1024
items:MinLength: 1
Optional: \{\}
| -| `channel` _string_ | channel is the unit's channel on the company's chat surface, by name,
such as platform; child units without one inherit it. On Chatlet, the
engine's own chat and the surface of a company that declares no
Mattermost or Slack Integration, each channel a unit names is a room
whose members follow the org chart. Crewlet takes lowercase letters,
digits, - and _, starting with a letter or a digit, up to 64
characters: the name itself, never #platform or a vendor's channel id.
Crewlet's `channel`. | | MaxLength: 64
MinLength: 1
Pattern: `^[a-z0-9][a-z0-9_-]\{0,63\}$`
Optional: \{\}
| +| `channel` _string_ | channel is the unit's channel on the company's chat surface, by name,
such as platform; child units without one inherit it. On Chatlet, the
engine's own chat and the surface of a company that declares no
Mattermost or Slack Integration, each channel a unit names is a room
whose members follow the org chart. Crewlet takes lowercase letters,
digits, - and _, starting with a letter or a digit, up to 64
characters: the name itself, never #platform or a vendor's channel id.
On Chatlet a room belongs to one unit, so Crewlet refuses a second unit
naming the same channel; a child unit that should share its parent's
room names none. Crewlet's `channel`. | | MaxLength: 64
MinLength: 1
Pattern: `^[a-z0-9][a-z0-9_-]\{0,63\}$`
Optional: \{\}
| | `knowledge` _string array_ | knowledge are free-text references to knowledge the unit relies on.
They are not a read scope. Crewlet's `knowledge`. | | MaxItems: 32
items:MaxLength: 1024
items:MinLength: 1
Optional: \{\}
| | `project` _string_ | project is the tracker project the unit files work in, by its key.
Crewlet requires a container key to be an upper-case letter followed
by 1 to 9 upper-case letters or digits, such as PLAT. Crewlet's
`project`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| | `space` _string_ | space is the knowledge container the unit writes pages in, by its
key, which follows the same rule as project. Crewlet's `space`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| From f0111ba057a97fc6ccf50b530f6e8fca4913942a Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 20:39:01 +0000 Subject: [PATCH 21/36] fix(render): expect what the engine advises a company on Chatlet Crewlet now runs a chat of its own, Chatlet: a company that declares no Slack or Mattermost (no integrations block for either, and no seat's Slack app) derives chat.backend chatlet, which is every company the operator renders without such an Integration. Two of its advisories moved with it. The warning of a Human seat with no chat contact is given only on a vendor surface, judged per surface: a seat with no account on the Slack or Mattermost the company talks on, a GitHub, GitLab or Atlassian id not counting. And a company on Chatlet whose units declare no channel is advised at units[0].channel that no team has a room the chart keeps. #51 already expects what the engine says of the guide's company and of the optional-keys case, which names no channel and is advised at units[0].channel, and the guide names its unit's channel. That left nothing holding the narrowed contact advisory, which the Seat API and two guides describe, so a new contract case, TestContractContactAdvisoryFollowsTheChatSurface, holds it on the Slack case: a Human seat with a GitHub login alone is warned at its contact of having no account on Slack, and one with a Slack member id is not. It goes red with either expectation flipped. The Seat contact doc, integrations.md and people.md now say the warning is given only on Slack or Mattermost, which accounts count, and that on Chatlet a person reads the rooms on the dashboard and nothing is warned; the troubleshooting page's list of what the engine says names both advisories, and two test comments stop calling a GitHub login a chat contact. Signed-off-by: Ali Yousefian --- api/v1alpha1/seat_types.go | 21 ++++--- config/crd/bases/crewlet.ai_seats.yaml | 21 ++++--- .../chart/templates/crd/seats.crewlet.ai.yaml | 21 ++++--- docs/guides/people.md | 6 +- docs/guides/troubleshooting.md | 6 +- docs/reference/api.md | 2 +- internal/render/contract_test.go | 62 +++++++++++++++++++ test/crd/cliagent_engine_test.go | 11 ++-- test/crd/orgchart_test.go | 3 +- 9 files changed, 118 insertions(+), 35 deletions(-) diff --git a/api/v1alpha1/seat_types.go b/api/v1alpha1/seat_types.go index 6525dc18..30fa3aa0 100644 --- a/api/v1alpha1/seat_types.go +++ b/api/v1alpha1/seat_types.go @@ -973,14 +973,19 @@ type SeatSpec struct { // +kubebuilder:validation:Pattern=`^[A-Z][A-Z0-9]{1,9}$` Space string `json:"space,omitempty"` - // contact is how agents reach the person in a Human seat on a chat - // surface, and how what the person does there reaches the engine as - // theirs. It applies to Human seats only, and is optional: a person who - // works only through the engine's dashboard has no account to name, and - // signs in on the seat through the engine's own identity directory. The - // engine warns of a Human seat without one, since nothing addressed to it - // on a chat surface reaches anybody, and the operator records the warning - // as an EngineWarnings event on the Company. + // contact is how agents reach the person in a Human seat on a vendor's + // chat surface or tool, and how what the person does there reaches the + // engine as theirs. It applies to Human seats only, and is optional: a + // person who works only through the engine's dashboard has no account to + // name, and signs in on the seat through the engine's own identity + // directory. On a company that talks on Slack or Mattermost (an + // Integration of either, or a seat's Slack app) the engine warns of a + // Human seat with no account there, a slackUserID or a + // mattermostUsername, since nothing addressed to it there reaches + // anybody; a GitHub, GitLab or Atlassian account does not count, and the + // operator records the warning as an EngineWarnings event on the + // Company. On Chatlet, which a company with neither talks on, the person + // reads the rooms on the dashboard as their seat, and nothing is warned. // +optional Contact SeatContact `json:"contact,omitzero"` diff --git a/config/crd/bases/crewlet.ai_seats.yaml b/config/crd/bases/crewlet.ai_seats.yaml index 96afad5a..a4248a67 100644 --- a/config/crd/bases/crewlet.ai_seats.yaml +++ b/config/crd/bases/crewlet.ai_seats.yaml @@ -124,14 +124,19 @@ spec: rule: self == oldSelf contact: description: |- - contact is how agents reach the person in a Human seat on a chat - surface, and how what the person does there reaches the engine as - theirs. It applies to Human seats only, and is optional: a person who - works only through the engine's dashboard has no account to name, and - signs in on the seat through the engine's own identity directory. The - engine warns of a Human seat without one, since nothing addressed to it - on a chat surface reaches anybody, and the operator records the warning - as an EngineWarnings event on the Company. + contact is how agents reach the person in a Human seat on a vendor's + chat surface or tool, and how what the person does there reaches the + engine as theirs. It applies to Human seats only, and is optional: a + person who works only through the engine's dashboard has no account to + name, and signs in on the seat through the engine's own identity + directory. On a company that talks on Slack or Mattermost (an + Integration of either, or a seat's Slack app) the engine warns of a + Human seat with no account there, a slackUserID or a + mattermostUsername, since nothing addressed to it there reaches + anybody; a GitHub, GitLab or Atlassian account does not count, and the + operator records the warning as an EngineWarnings event on the + Company. On Chatlet, which a company with neither talks on, the person + reads the rooms on the dashboard as their seat, and nothing is warned. minProperties: 1 properties: atlassianAccount: diff --git a/dist/chart/templates/crd/seats.crewlet.ai.yaml b/dist/chart/templates/crd/seats.crewlet.ai.yaml index 28cd51e0..08cd2ade 100644 --- a/dist/chart/templates/crd/seats.crewlet.ai.yaml +++ b/dist/chart/templates/crd/seats.crewlet.ai.yaml @@ -127,14 +127,19 @@ spec: rule: self == oldSelf contact: description: |- - contact is how agents reach the person in a Human seat on a chat - surface, and how what the person does there reaches the engine as - theirs. It applies to Human seats only, and is optional: a person who - works only through the engine's dashboard has no account to name, and - signs in on the seat through the engine's own identity directory. The - engine warns of a Human seat without one, since nothing addressed to it - on a chat surface reaches anybody, and the operator records the warning - as an EngineWarnings event on the Company. + contact is how agents reach the person in a Human seat on a vendor's + chat surface or tool, and how what the person does there reaches the + engine as theirs. It applies to Human seats only, and is optional: a + person who works only through the engine's dashboard has no account to + name, and signs in on the seat through the engine's own identity + directory. On a company that talks on Slack or Mattermost (an + Integration of either, or a seat's Slack app) the engine warns of a + Human seat with no account there, a slackUserID or a + mattermostUsername, since nothing addressed to it there reaches + anybody; a GitHub, GitLab or Atlassian account does not count, and the + operator records the warning as an EngineWarnings event on the + Company. On Chatlet, which a company with neither talks on, the person + reads the rooms on the dashboard as their seat, and nothing is warned. minProperties: 1 properties: atlassianAccount: diff --git a/docs/guides/people.md b/docs/guides/people.md index cf31ea7e..a39d987b 100644 --- a/docs/guides/people.md +++ b/docs/guides/people.md @@ -44,8 +44,10 @@ spec: - `contact` is optional. It names the person's chat accounts, so that an agent can mention or message them on Slack or Mattermost; a person who - works only through the dashboard needs none. Crewlet warns of a Human seat - without one, which the operator reports as an `EngineWarnings` event + works only through the dashboard needs none. On a company that talks on + either, Crewlet warns of a Human seat with no account there, which the + operator reports as an `EngineWarnings` event; on Chatlet, which a company + with neither talks on, it warns of nothing ([A person's accounts](integrations.md#a-persons-accounts)). - `email` is optional too, and unique: two Seats of one company may not declare one address, compared as Crewlet compares them (case aside, and diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 4b5beb40..f7f0664e 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -820,8 +820,10 @@ Some settings the engine runs are still worth knowing about. Plain `http` on a routable address sends passwords unencrypted. An `https` address with no proxy listed shows every caller as the proxy. A class may let people sign in with a password alone at a routable address, which it acknowledged. A company -may have no human seat anyone can sign in to, or a token ceiling another -leaves idle. None of these stops the engine. It says so in two places: +may have no human seat anyone can sign in to, a token ceiling another leaves +idle, a person on Slack or Mattermost nobody can mention there, or, on +Chatlet, no unit with a `channel`, and so no team with a room the chart +keeps. None of these stops the engine. It says so in two places: - **The Company's events**: the engine's answer to each revision the operator writes, as an `EngineWarnings` event (`The engine runs revision … with diff --git a/docs/reference/api.md b/docs/reference/api.md index 96ea0be3..8b5bd604 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -3712,7 +3712,7 @@ _Appears in:_ | `integrations` _[SeatIntegrations](#seatintegrations)_ | integrations is the seat's own identity on the company's
integrations. | | MinProperties: 1
Optional: \{\}
| | `project` _string_ | project is the tracker project a seat outside every unit owns, by its
key; a seat in a unit takes its unit's. Crewlet requires a container
key to be an upper-case letter followed by 1 to 9 upper-case letters or
digits, such as LEAD. Crewlet's `project`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| | `space` _string_ | space is the knowledge container a seat outside every unit owns, by
its key, which follows the same rule as project; a seat in a unit
takes its unit's. Crewlet's `space`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| -| `contact` _[SeatContact](#seatcontact)_ | contact is how agents reach the person in a Human seat on a chat
surface, and how what the person does there reaches the engine as
theirs. It applies to Human seats only, and is optional: a person who
works only through the engine's dashboard has no account to name, and
signs in on the seat through the engine's own identity directory. The
engine warns of a Human seat without one, since nothing addressed to it
on a chat surface reaches anybody, and the operator records the warning
as an EngineWarnings event on the Company. | | MinProperties: 1
Optional: \{\}
| +| `contact` _[SeatContact](#seatcontact)_ | contact is how agents reach the person in a Human seat on a vendor's
chat surface or tool, and how what the person does there reaches the
engine as theirs. It applies to Human seats only, and is optional: a
person who works only through the engine's dashboard has no account to
name, and signs in on the seat through the engine's own identity
directory. On a company that talks on Slack or Mattermost (an
Integration of either, or a seat's Slack app) the engine warns of a
Human seat with no account there, a slackUserID or a
mattermostUsername, since nothing addressed to it there reaches
anybody; a GitHub, GitLab or Atlassian account does not count, and the
operator records the warning as an EngineWarnings event on the
Company. On Chatlet, which a company with neither talks on, the person
reads the rooms on the dashboard as their seat, and nothing is warned. | | MinProperties: 1
Optional: \{\}
| | `availability` _string_ | availability tells colleagues of a Human seat when to expect an
answer, such as "CET business hours". Crewlet's `availability`. | | MaxLength: 256
MinLength: 1
Optional: \{\}
| diff --git a/internal/render/contract_test.go b/internal/render/contract_test.go index 5447e8de..f3dfc9bc 100644 --- a/internal/render/contract_test.go +++ b/internal/render/contract_test.go @@ -1215,6 +1215,68 @@ func TestContractGettingStarted(t *testing.T) { "-config", write(t, "crewlet.yaml", tierA.YAML), "-company", write(t, "company.yaml", doc.YAML)) } +// TestContractContactAdvisoryFollowsTheChatSurface holds what the Seat API +// and the guides say of a Human seat with no chat account to the engine: on a +// company that talks on Slack, here through its Slack Integration, Crewlet +// warns at the seat's contact of a Human seat that names no Slack account, a +// GitHub login not counting, since it says nothing of whether a Slack mention +// reaches the person; and it says nothing of one that names its Slack member +// id. On Chatlet it warns of neither, which TestContractGettingStarted +// holds. +func TestContractContactAdvisoryFollowsTheChatSurface(t *testing.T) { + tierA, err := Bootstrap(bootstrapCases()[0].input(t)) + if err != nil { + t.Fatal(err) + } + config := write(t, "crewlet.yaml", tierA.YAML) + for _, c := range []struct { + name string + contact v1alpha1.SeatContact + warns bool + }{ + {name: "a GitHub login alone", contact: v1alpha1.SeatContact{GitHubLogin: "jane-doe"}, warns: true}, + {name: "a Slack member id", contact: v1alpha1.SeatContact{SlackUserID: "U0JANEDOE", GitHubLogin: "jane-doe"}}, + } { + t.Run(c.name, func(t *testing.T) { + in := companyCaseNamed(t, "slack").input(t) + person := v1alpha1.Seat{ + Namespace: in.Company.Namespace, Name: "jane", + Spec: v1alpha1.SeatSpec{ + CompanyRef: v1alpha1.CompanyReference{Name: in.Company.Name}, + Kind: v1alpha1.SeatKindHuman, + Contact: c.contact, + }, + } + in.Seats = append(in.Seats, person) + doc, err := Company(in) + if err != nil { + t.Fatal(err) + } + i := slices.IndexFunc(doc.Sources, func(s Source) bool { + return s.Object == ObjectRef{Kind: KindSeat, Name: person.Name} + }) + if i < 0 { + t.Fatalf("the document renders Seat %s nowhere", person.Name) + } + var want []advisory + if c.warns { + want = append(want, advisory{ + path: doc.Sources[i].Path.String() + ".contact", + says: person.Name + " is a human seat with no account on Slack", + }) + } + env := bootstrapEnv(tierA) + for _, k := range doc.Refs { + env[secretsync.StoreName(k)] = dummyCredential(k) + } + for _, name := range doc.Provisioned { + env[name] = "minted-" + name + } + validate(t, env, want, "-config", config, "-company", write(t, "company.yaml", doc.YAML)) + }) + } +} + // TestContractScheduleRefusalsReachTheSchedule has the engine's dry run // refuse each of a schedule's mistakes no API rule can see, and checks that // the document attributes every problem to the Schedule, which is what the diff --git a/test/crd/cliagent_engine_test.go b/test/crd/cliagent_engine_test.go index 1cd6f919..f88c9846 100644 --- a/test/crd/cliagent_engine_test.go +++ b/test/crd/cliagent_engine_test.go @@ -1249,11 +1249,12 @@ func askEngine(t *testing.T, providers []v1.LLMProvider, list bool) engineReport } in.Seats = append(in.Seats, *s) } - // A person on a seat of their own, reached on a chat surface. Without - // one, Crewlet advises that nobody can be invited into the company, - // which is a verdict on the chart rather than on any provider; a - // tolerance for it here would hide that advisory on a provider's shape - // too, so the probe gives the engine nothing about the chart to say. + // A person on a seat of their own, on the company's own chat, where + // Crewlet asks no account of them. Without the seat, Crewlet advises + // that nobody can be invited into the company, which is a verdict on + // the chart rather than on any provider; a tolerance for it here would + // hide that advisory on a provider's shape too, so the probe gives the + // engine nothing about the chart to say. founder := seat("probe", "founder") founder.Spec.Kind = v1.SeatKindHuman founder.Spec.Contact = v1.SeatContact{GitHubLogin: "probe-founder"} diff --git a/test/crd/orgchart_test.go b/test/crd/orgchart_test.go index c105f2be..bc8a8061 100644 --- a/test/crd/orgchart_test.go +++ b/test/crd/orgchart_test.go @@ -182,7 +182,8 @@ func TestSeatValidation(t *testing.T) { {name: "a human seat with contact is accepted", mutate: asHuman(nothing)}, { // A person who works only through the dashboard has no chat - // account to name; Crewlet advises rather than refuses. + // account to name; on Slack or Mattermost Crewlet advises + // rather than refuses, and on Chatlet says nothing. name: "a human seat without contact is accepted", mutate: func(s *v1.Seat) { s.Spec.Kind = v1.SeatKindHuman }, }, From 660e027d6a986c780a4f87b68a3848f227583bd6 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 20:51:43 +0000 Subject: [PATCH 22/36] docs(api): say a company with no chat Integration runs Chatlet Crewlet now runs a chat of its own, Chatlet: a company whose document declares no Slack or Mattermost derives chat.backend chatlet, so every company the operator renders without such an Integration gets a room per unit channel, its seats and people talking there, and Chatlet's workspace on the dashboard, its messages kept on the chat state log (CREWLET_CHAT_LOG) every data node runs whatever its company's chat. No operator page said so; the integrations guide now does, linking the engine's Chatlet page, and says connecting Slack or Mattermost moves the company onto that vendor's chat. The engineConfig doc's list of the tuning blocks it carries also left out artifacts, which it passes through like the others and which holds no credential; the list names it. Signed-off-by: Ali Yousefian --- api/v1alpha1/company_types.go | 2 +- config/crd/bases/crewlet.ai_companies.yaml | 2 +- dist/chart/templates/crd/companies.crewlet.ai.yaml | 2 +- docs/guides/integrations.md | 7 +++++++ docs/reference/api.md | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index 05c8280f..c76cd6da 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -1237,7 +1237,7 @@ type CompanySpec struct { // engineConfig is passed through into the company document as written, // for the engine's tuning blocks that have no field here: turn_engine, - // learning, scheduling, workers, skill_variables and the + // learning, scheduling, workers, skill_variables, artifacts and the // notification_* settings, spelled as Crewlet spells them. Each entry is // one top-level key of the document and its value. The operator does not // interpret it; the engine validates it in the dry run before every diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index c23fa782..893fcead 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -409,7 +409,7 @@ spec: description: |- engineConfig is passed through into the company document as written, for the engine's tuning blocks that have no field here: turn_engine, - learning, scheduling, workers, skill_variables and the + learning, scheduling, workers, skill_variables, artifacts and the notification_* settings, spelled as Crewlet spells them. Each entry is one top-level key of the document and its value. The operator does not interpret it; the engine validates it in the dry run before every diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index a306fd09..67541b9b 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -412,7 +412,7 @@ spec: description: |- engineConfig is passed through into the company document as written, for the engine's tuning blocks that have no field here: turn_engine, - learning, scheduling, workers, skill_variables and the + learning, scheduling, workers, skill_variables, artifacts and the notification_* settings, spelled as Crewlet spells them. Each entry is one top-level key of the document and its value. The operator does not interpret it; the engine validates it in the dry run before every diff --git a/docs/guides/integrations.md b/docs/guides/integrations.md index aee5d999..84983050 100644 --- a/docs/guides/integrations.md +++ b/docs/guides/integrations.md @@ -15,6 +15,13 @@ Every credential is a Secret reference ([Supplying credentials](credentials.md)) [Credential Positions](../concepts/credentials.md) lists every position and the field that fills it. +A company with no Slack or Mattermost Integration talks on Chatlet instead, +Crewlet's own chat, which needs no Integration: the engine opens a room for +each unit's `channel`, whose members follow the org chart, and its seats and +people talk there, a person on the dashboard as their seat +([Chatlet](https://github.com/crewlet/crewlet/blob/main/docs/concepts/chatlet.md)). +Connecting Slack or Mattermost moves the company onto that vendor's chat. + An Integration connects the engine to the vendor. An agent acts there, its chat replies included, through a tool server: an `MCPServer` its Seat gives its own credentials. Most of those servers are programs the engine starts in diff --git a/docs/reference/api.md b/docs/reference/api.md index 8b5bd604..12c782a1 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -673,7 +673,7 @@ _Appears in:_ | `embeddings` _[EmbeddingSettings](#embeddingsettings)_ | embeddings is the provider vectors are computed with. Unset leaves
search keyword-only. Crewlet's providers.embeddings. | | Optional: \{\}
| | `sandbox` _[SandboxSettings](#sandboxsettings)_ | sandbox is where the company's agent seats' executors run. Required
of a company with an Agent seat and a model provider, which Crewlet
refuses without it (SandboxRequired). Crewlet's providers.sandbox. | | Optional: \{\}
| | `integrations` _[CompanyIntegrationSettings](#companyintegrationsettings)_ | integrations is company-wide integration behaviour. | | MinProperties: 1
Optional: \{\}
| -| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, token_budget,
assistant, providers, integrations, mcp_servers, roles, units or
seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's
spec.integrations. Every
position Crewlet holds a credential in lives under one of those keys, so
engineConfig can never carry a credential: the API server refuses a
Company whose engineConfig names one of them, before anything is
stored. | | MaxItems: 64
Optional: \{\}
| +| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables, artifacts and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, token_budget,
assistant, providers, integrations, mcp_servers, roles, units or
seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's
spec.integrations. Every
position Crewlet holds a credential in lives under one of those keys, so
engineConfig can never carry a credential: the API server refuses a
Company whose engineConfig names one of them, before anything is
stored. | | MaxItems: 64
Optional: \{\}
| | `config` _[ConfigSettings](#configsettings)_ | config is how the operator manages the company's document. | \{ managementPolicy:Enforce \} | MinProperties: 1
Optional: \{\}
| | `topology` _[CompanyTopology](#companytopology)_ | topology is how the engine is laid out in pods. | \{ data:map[replicas:1] \} | MinProperties: 1
Optional: \{\}
| | `exposure` _[CompanyExposure](#companyexposure)_ | exposure overrides how the company is reached. | | MinProperties: 1
Optional: \{\}
| From 3c136c78188f1d844b38ee7e889da87d90138df3 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 20:58:08 +0000 Subject: [PATCH 23/36] feat(api): type the Company's chat backend and Chatlet policy Crewlet's company document gains a chat block: backend chatlet, vendor or none, and under chat.chatlet the policy of Chatlet, its own chat: a message horizon, whether a room is private unless its creator says otherwise, and when a seat shows it is working. The tracker and the knowledge base, the two other surfaces a company picks between the engine's own and a vendor's, are typed fields of the Company; chat reached the document only as an engineConfig entry, which nothing validated before the dry run and which the API reference did not describe. spec.chat.backend is Chatlet, Vendor or None with no default: unset, the engine derives it (Vendor beside a Slack or Mattermost Integration or a seat's Slack app, Chatlet otherwise), and a defaulted Chatlet would render beside such an Integration, which the engine refuses. spec.chat.chatlet takes messageRetentionDays (0 for ever, or 30 to 3650), a defaultChannelPrivate and typingStatus (the WorkingStatus the vendor blocks already use), and is refused beside Vendor or None; with no backend it applies wherever the engine derives Chatlet, as Crewlet takes it. Chatlet beside a chat Integration, Vendor without one and a chatlet block with no backend beside one turn on other objects, so they are left to the engine's dry run, as a tracker of Jira without a Jira Integration is; TestContractChatSettingsFollowTheEngine holds that the engine refuses each where it says (chat.backend, chat.backend and chat.chatlet), that the document attributes each to the Company, and that it takes a chatlet block with no backend on a company with no chat Integration, and goes red when any of them is answered otherwise. chat joins the keys engineConfig may not set, beside main's assistant. The new chat company case renders every setting, which TestCompanyGolden, TestContractCompany and TestDiffContract hold. A new test, TestEngineConfigRefusesEveryTypedKey, holds typedKeys to the keys the document type renders and to the CRD's engineConfig rule, which nothing compared, and fails with chat taken out of either. It found the assistant missing from typedKeys, which main's assistant field renders and the CRD refuses in engineConfig: Diff took an assistant block only the engine had for a tuning block the Company leaves to the engine's defaults, rather than a change somebody made there, so typedKeys gains it too. The CRD suite covers each rule's edges. Signed-off-by: Ali Yousefian --- api/v1alpha1/company_types.go | 93 +++++++++++++++++-- api/v1alpha1/zz_generated.deepcopy.go | 42 +++++++++ config/crd/bases/crewlet.ai_companies.yaml | 85 +++++++++++++++-- .../templates/crd/companies.crewlet.ai.yaml | 85 +++++++++++++++-- docs/concepts/company-document.md | 2 +- docs/concepts/overview.md | 2 +- docs/guides/integrations.md | 11 ++- docs/reference/api.md | 63 ++++++++++++- internal/render/company.go | 8 ++ internal/render/company_test.go | 5 +- internal/render/contract_test.go | 71 +++++++++++++- internal/render/credentials_test.go | 54 ++++++++--- internal/render/diff.go | 4 +- internal/render/document.go | 12 +++ internal/render/enums.go | 12 +++ internal/render/helpers_test.go | 11 +++ .../render/testdata/company/chat.input.yaml | 39 ++++++++ internal/render/testdata/company/chat.yaml | 22 +++++ test/crd/company_test.go | 82 +++++++++++++++- test/crd/fixtures_test.go | 2 + 20 files changed, 651 insertions(+), 54 deletions(-) create mode 100644 internal/render/testdata/company/chat.input.yaml create mode 100644 internal/render/testdata/company/chat.yaml diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index c76cd6da..e6a6e674 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -111,6 +111,75 @@ type KnowledgeSettings struct { Vectors *bool `json:"vectors,omitempty"` } +// ChatBackend is which chat surface a company runs. It maps onto Crewlet's +// chat.backend (Chatlet is `chatlet`, Vendor is `vendor`, None is `none`). +// +kubebuilder:validation:Enum=Chatlet;Vendor;None +type ChatBackend string + +// The chat backends. +const ( + // ChatChatlet is Chatlet, Crewlet's own chat: rooms the engine keeps, a + // room for each unit's channel, read on the dashboard. + ChatChatlet ChatBackend = "Chatlet" + // ChatVendor is the Slack or Mattermost the company's Integrations (or + // a seat's Slack app) connect it to. + ChatVendor ChatBackend = "Vendor" + // ChatNone is a company with no chat. + ChatNone ChatBackend = "None" +) + +// ChatletSettings is the policy of Chatlet. It maps onto Crewlet's +// chat.chatlet. +// +kubebuilder:validation:MinProperties=1 +type ChatletSettings struct { + // messageRetentionDays is how long a message lives before the engine + // prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + // days, since under a month somebody away could miss a decision before + // reading it. Unset takes Crewlet's default of 365. Crewlet's + // `message_retention_days`. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=3650 + // +kubebuilder:validation:XValidation:rule="self == 0 || self >= 30",message="messageRetentionDays must be 0, which keeps messages for ever, or 30 to 3650" + MessageRetentionDays *int32 `json:"messageRetentionDays,omitempty"` + + // defaultChannelPrivate makes a room created without a stated + // visibility private; unset, such a room is public, readable by the + // company's seats. A room created with a visibility keeps it. Crewlet's + // `default_channel_private`. + // +optional + DefaultChannelPrivate *bool `json:"defaultChannelPrivate,omitempty"` + + // typingStatus is when a seat shows in a room that it is working on a + // message. Unset takes Crewlet's default, Always. Crewlet's + // `typing_status`. + // +optional + TypingStatus WorkingStatus `json:"typingStatus,omitempty"` +} + +// ChatSettings is which chat surface a company runs and the policy of +// Chatlet. It maps onto Crewlet's chat block. +// +kubebuilder:validation:XValidation:rule="!has(self.chatlet) || !has(self.backend) || self.backend == 'Chatlet'",message="chatlet applies to backend Chatlet only, or to no backend, which Crewlet derives Chatlet for a company with no Slack or Mattermost" +// +kubebuilder:validation:MinProperties=1 +type ChatSettings struct { + // backend is which chat surface the company runs. Unset derives it, as + // Crewlet does: Vendor for a company with a Slack or Mattermost + // Integration or a seat with a Slack app, Chatlet for every other. So + // there is no default here: a Chatlet written beside such an + // Integration, or a Vendor without one, is refused by the engine's dry + // run, reported on the Company, since a company's conversations cannot + // live in two places. + // +optional + Backend ChatBackend `json:"backend,omitempty"` + + // chatlet is the policy of Chatlet: for backend Chatlet, or for no + // backend on a company Crewlet derives Chatlet for. On a company whose + // chat is a vendor's nothing would read it, so the engine's dry run + // refuses it there, reported on the Company. + // +optional + Chatlet ChatletSettings `json:"chatlet,omitzero"` +} + // LLMProviderKind is the kind of model provider a reference names. // +kubebuilder:validation:Enum=LLMProvider;PlatformLLMProvider type LLMProviderKind string @@ -1201,6 +1270,11 @@ type CompanySpec struct { // +optional Knowledge KnowledgeSettings `json:"knowledge,omitzero"` + // chat is which chat surface the company runs: Chatlet, Crewlet's own, + // unless it is connected to Slack or Mattermost. Crewlet's `chat`. + // +optional + Chat ChatSettings `json:"chat,omitzero"` + // tokenBudget caps the tokens all seats together may spend per window. // Each window binds every agent seat, so Crewlet refuses one below // turn_engine.executor_min_budget_tokens (25000 unless spec.engineConfig @@ -1244,20 +1318,19 @@ type CompanySpec struct { // apply, and a refusal is reported on the Company. // // It may not set what another field or object of this API owns: name, - // mission, vision, policies, timezone, tracker, knowledge, token_budget, - // assistant, providers, integrations, mcp_servers, roles, units or - // seat_access_migrated, which the operator writes with every - // integration kind so the engine acts on each Seat's - // spec.integrations. Every - // position Crewlet holds a credential in lives under one of those keys, so - // engineConfig can never carry a credential: the API server refuses a - // Company whose engineConfig names one of them, before anything is - // stored. + // mission, vision, policies, timezone, tracker, knowledge, chat, + // token_budget, assistant, providers, integrations, mcp_servers, roles, + // units or seat_access_migrated, which the operator writes with every + // integration kind so the engine acts on each Seat's spec.integrations. + // Every position Crewlet holds a credential in lives under one of those + // keys, so engineConfig can never carry a credential: the API server + // refuses a Company whose engineConfig names one of them, before anything + // is stored. // +optional // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=64 - // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated']))",message="engineConfig may not set name, mission, vision, policies, timezone, tracker, knowledge, token_budget, assistant, providers, integrations, mcp_servers, roles, units or seat_access_migrated: those are fields and objects of this API, among them every block that holds a credential" + // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated']))",message="engineConfig may not set name, mission, vision, policies, timezone, tracker, knowledge, token_budget, assistant, providers, integrations, mcp_servers, roles, units or seat_access_migrated: those are fields and objects of this API, among them every block that holds a credential" EngineConfig []EngineConfigEntry `json:"engineConfig,omitempty"` // config is how the operator manages the company's document. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 54f96596..ca8f87b3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -131,6 +131,47 @@ func (in *CLIAgentAuth) DeepCopy() *CLIAgentAuth { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChatSettings) DeepCopyInto(out *ChatSettings) { + *out = *in + in.Chatlet.DeepCopyInto(&out.Chatlet) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChatSettings. +func (in *ChatSettings) DeepCopy() *ChatSettings { + if in == nil { + return nil + } + out := new(ChatSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChatletSettings) DeepCopyInto(out *ChatletSettings) { + *out = *in + if in.MessageRetentionDays != nil { + in, out := &in.MessageRetentionDays, &out.MessageRetentionDays + *out = new(int32) + **out = **in + } + if in.DefaultChannelPrivate != nil { + in, out := &in.DefaultChannelPrivate, &out.DefaultChannelPrivate + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChatletSettings. +func (in *ChatletSettings) DeepCopy() *ChatletSettings { + if in == nil { + return nil + } + out := new(ChatletSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClassEnvVar) DeepCopyInto(out *ClassEnvVar) { *out = *in @@ -387,6 +428,7 @@ func (in *CompanySpec) DeepCopyInto(out *CompanySpec) { } out.Tracker = in.Tracker in.Knowledge.DeepCopyInto(&out.Knowledge) + in.Chat.DeepCopyInto(&out.Chat) out.TokenBudget = in.TokenBudget in.LLM.DeepCopyInto(&out.LLM) in.Assistant.DeepCopyInto(&out.Assistant) diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index 893fcead..56e48968 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -240,6 +240,72 @@ spec: it runs on, one whose webTools is Denied' rule: '!has(self.enabled) || !self.enabled || (has(self.llm) && size(self.llm) > 0)' + chat: + description: |- + chat is which chat surface the company runs: Chatlet, Crewlet's own, + unless it is connected to Slack or Mattermost. Crewlet's `chat`. + minProperties: 1 + properties: + backend: + description: |- + backend is which chat surface the company runs. Unset derives it, as + Crewlet does: Vendor for a company with a Slack or Mattermost + Integration or a seat with a Slack app, Chatlet for every other. So + there is no default here: a Chatlet written beside such an + Integration, or a Vendor without one, is refused by the engine's dry + run, reported on the Company, since a company's conversations cannot + live in two places. + enum: + - Chatlet + - Vendor + - None + type: string + chatlet: + description: |- + chatlet is the policy of Chatlet: for backend Chatlet, or for no + backend on a company Crewlet derives Chatlet for. On a company whose + chat is a vendor's nothing would read it, so the engine's dry run + refuses it there, reported on the Company. + minProperties: 1 + properties: + defaultChannelPrivate: + description: |- + defaultChannelPrivate makes a room created without a stated + visibility private; unset, such a room is public, readable by the + company's seats. A room created with a visibility keeps it. Crewlet's + `default_channel_private`. + type: boolean + messageRetentionDays: + description: |- + messageRetentionDays is how long a message lives before the engine + prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + days, since under a month somebody away could miss a decision before + reading it. Unset takes Crewlet's default of 365. Crewlet's + `message_retention_days`. + format: int32 + maximum: 3650 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: messageRetentionDays must be 0, which keeps messages + for ever, or 30 to 3650 + rule: self == 0 || self >= 30 + typingStatus: + description: |- + typingStatus is when a seat shows in a room that it is working on a + message. Unset takes Crewlet's default, Always. Crewlet's + `typing_status`. + enum: + - Always + - Addressed + type: string + type: object + type: object + x-kubernetes-validations: + - message: chatlet applies to backend Chatlet only, or to no backend, + which Crewlet derives Chatlet for a company with no Slack or Mattermost + rule: '!has(self.chatlet) || !has(self.backend) || self.backend + == ''Chatlet''' companyName: description: |- companyName is the company's name. Crewlet derives every seat's @@ -416,15 +482,14 @@ spec: apply, and a refusal is reported on the Company. It may not set what another field or object of this API owns: name, - mission, vision, policies, timezone, tracker, knowledge, token_budget, - assistant, providers, integrations, mcp_servers, roles, units or - seat_access_migrated, which the operator writes with every - integration kind so the engine acts on each Seat's - spec.integrations. Every - position Crewlet holds a credential in lives under one of those keys, so - engineConfig can never carry a credential: the API server refuses a - Company whose engineConfig names one of them, before anything is - stored. + mission, vision, policies, timezone, tracker, knowledge, chat, + token_budget, assistant, providers, integrations, mcp_servers, roles, + units or seat_access_migrated, which the operator writes with every + integration kind so the engine acts on each Seat's spec.integrations. + Every position Crewlet holds a credential in lives under one of those + keys, so engineConfig can never carry a credential: the API server + refuses a Company whose engineConfig names one of them, before anything + is stored. items: description: |- EngineConfigEntry is one top-level key of the company document that @@ -458,7 +523,7 @@ spec: those are fields and objects of this API, among them every block that holds a credential' rule: self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', - 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', + 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated'])) exposure: diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index 67541b9b..c695dd66 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -243,6 +243,72 @@ spec: it runs on, one whose webTools is Denied' rule: '!has(self.enabled) || !self.enabled || (has(self.llm) && size(self.llm) > 0)' + chat: + description: |- + chat is which chat surface the company runs: Chatlet, Crewlet's own, + unless it is connected to Slack or Mattermost. Crewlet's `chat`. + minProperties: 1 + properties: + backend: + description: |- + backend is which chat surface the company runs. Unset derives it, as + Crewlet does: Vendor for a company with a Slack or Mattermost + Integration or a seat with a Slack app, Chatlet for every other. So + there is no default here: a Chatlet written beside such an + Integration, or a Vendor without one, is refused by the engine's dry + run, reported on the Company, since a company's conversations cannot + live in two places. + enum: + - Chatlet + - Vendor + - None + type: string + chatlet: + description: |- + chatlet is the policy of Chatlet: for backend Chatlet, or for no + backend on a company Crewlet derives Chatlet for. On a company whose + chat is a vendor's nothing would read it, so the engine's dry run + refuses it there, reported on the Company. + minProperties: 1 + properties: + defaultChannelPrivate: + description: |- + defaultChannelPrivate makes a room created without a stated + visibility private; unset, such a room is public, readable by the + company's seats. A room created with a visibility keeps it. Crewlet's + `default_channel_private`. + type: boolean + messageRetentionDays: + description: |- + messageRetentionDays is how long a message lives before the engine + prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + days, since under a month somebody away could miss a decision before + reading it. Unset takes Crewlet's default of 365. Crewlet's + `message_retention_days`. + format: int32 + maximum: 3650 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: messageRetentionDays must be 0, which keeps messages + for ever, or 30 to 3650 + rule: self == 0 || self >= 30 + typingStatus: + description: |- + typingStatus is when a seat shows in a room that it is working on a + message. Unset takes Crewlet's default, Always. Crewlet's + `typing_status`. + enum: + - Always + - Addressed + type: string + type: object + type: object + x-kubernetes-validations: + - message: chatlet applies to backend Chatlet only, or to no backend, + which Crewlet derives Chatlet for a company with no Slack or Mattermost + rule: '!has(self.chatlet) || !has(self.backend) || self.backend + == ''Chatlet''' companyName: description: |- companyName is the company's name. Crewlet derives every seat's @@ -419,15 +485,14 @@ spec: apply, and a refusal is reported on the Company. It may not set what another field or object of this API owns: name, - mission, vision, policies, timezone, tracker, knowledge, token_budget, - assistant, providers, integrations, mcp_servers, roles, units or - seat_access_migrated, which the operator writes with every - integration kind so the engine acts on each Seat's - spec.integrations. Every - position Crewlet holds a credential in lives under one of those keys, so - engineConfig can never carry a credential: the API server refuses a - Company whose engineConfig names one of them, before anything is - stored. + mission, vision, policies, timezone, tracker, knowledge, chat, + token_budget, assistant, providers, integrations, mcp_servers, roles, + units or seat_access_migrated, which the operator writes with every + integration kind so the engine acts on each Seat's spec.integrations. + Every position Crewlet holds a credential in lives under one of those + keys, so engineConfig can never carry a credential: the API server + refuses a Company whose engineConfig names one of them, before anything + is stored. items: description: |- EngineConfigEntry is one top-level key of the company document that @@ -461,7 +526,7 @@ spec: those are fields and objects of this API, among them every block that holds a credential' rule: self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', - 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', + 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated'])) exposure: diff --git a/docs/concepts/company-document.md b/docs/concepts/company-document.md index e2e32ac1..cad2135c 100644 --- a/docs/concepts/company-document.md +++ b/docs/concepts/company-document.md @@ -37,7 +37,7 @@ handle, and model providers and tool servers by name. | Object | Becomes | |---|---| -| `Company` | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `token_budget`, `providers.embeddings`, `providers.sandbox`, `providers.llm_order`, `integrations.check_interval_seconds`, `seat_access_migrated` (every integration kind, so the engine acts on each Seat's `spec.integrations`), and one top-level key per `spec.engineConfig` entry | +| `Company` | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `chat`, `token_budget`, `providers.embeddings`, `providers.sandbox`, `providers.llm_order`, `integrations.check_interval_seconds`, `seat_access_migrated` (every integration kind, so the engine acts on each Seat's `spec.integrations`), and one top-level key per `spec.engineConfig` entry | | `EngineClass` | `providers.llm.` for each of its `platformLLMProviders`, when the Company sets `spec.llm.usePlatformProviders`; its `state_dir` is `/var/lib/crewlet/llm-cli-platform/`, on the data volume | | `LLMProvider` | `providers.llm.`; its `state_dir` is `/var/lib/crewlet/llm-cli/`, on the data volume | | `MCPServer` | one entry of `mcp_servers`, named by its `metadata.name` | diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index c882dc83..ca673cfb 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -31,7 +31,7 @@ the [API reference](../reference/api.md). | Kind | Scope | What it is | Becomes in the company document | |---|---|---|---| | `EngineClass` | Cluster | A platform's defaults and guard rails: the engine image, the size of the data pod, how companies are exposed, limits, models the platform provides | Nothing directly; it shapes the pods, the routes and the limits of every Company that names it | -| `Company` | Namespace | The company itself, and the engine that runs it | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `token_budget`, `providers.embeddings`, `providers.sandbox`, and the engine tuning blocks in `engineConfig` | +| `Company` | Namespace | The company itself, and the engine that runs it | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `chat`, `token_budget`, `providers.embeddings`, `providers.sandbox`, and the engine tuning blocks in `engineConfig` | | `Unit` | Namespace | One node of the org chart: a division, a team, a squad | One entry of `units`, nested under its parent | | `Seat` | Namespace | One seat: an AI agent, or a person agents work with, tied to their own accounts ([A person's accounts](../guides/integrations.md#a-persons-accounts)) | One entry of `roles`, inside its unit | | `LLMProvider` | Namespace | A model a company's seats can run on: a coding CLI the engine drives ([Running seats on a coding CLI](../guides/cli-agents.md)) | One entry of `providers.llm` | diff --git a/docs/guides/integrations.md b/docs/guides/integrations.md index 84983050..86ff7127 100644 --- a/docs/guides/integrations.md +++ b/docs/guides/integrations.md @@ -20,7 +20,16 @@ Crewlet's own chat, which needs no Integration: the engine opens a room for each unit's `channel`, whose members follow the org chart, and its seats and people talk there, a person on the dashboard as their seat ([Chatlet](https://github.com/crewlet/crewlet/blob/main/docs/concepts/chatlet.md)). -Connecting Slack or Mattermost moves the company onto that vendor's chat. +Connecting Slack or Mattermost moves the company onto that vendor's chat. The +Company's `spec.chat` names the backend instead of leaving the engine to +derive it (`None` turns chat off), and sets the policy of Chatlet under +`chatlet`, beside `Chatlet` or no backend at all: how long a message is +kept (`messageRetentionDays`, 365 by default, `0` for ever), whether a room +is private unless its creator says otherwise (`defaultChannelPrivate`), and +when a seat shows it is working (`typingStatus`). The engine's dry run +refuses, on the Company, `Chatlet` beside a Slack or Mattermost Integration, +`Vendor` without one, and a `chatlet` block on a company whose chat is a +vendor's, which nothing would read. An Integration connects the engine to the vendor. An agent acts there, its chat replies included, through a tool server: an `MCPServer` its Seat gives diff --git a/docs/reference/api.md b/docs/reference/api.md index 12c782a1..34f2734a 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -366,6 +366,65 @@ _Appears in:_ | `Denied` | CLIAgentWebToolsDenied runs the CLI with its web tools off.
| +#### ChatBackend + +_Underlying type:_ _string_ + +ChatBackend is which chat surface a company runs. It maps onto Crewlet's +chat.backend (Chatlet is `chatlet`, Vendor is `vendor`, None is `none`). + +_Validation:_ +- Enum: [Chatlet Vendor None] + +_Appears in:_ +- [ChatSettings](#chatsettings) + +| Field | Description | +| --- | --- | +| `Chatlet` | ChatChatlet is Chatlet, Crewlet's own chat: rooms the engine keeps, a
room for each unit's channel, read on the dashboard.
| +| `Vendor` | ChatVendor is the Slack or Mattermost the company's Integrations (or
a seat's Slack app) connect it to.
| +| `None` | ChatNone is a company with no chat.
| + + +#### ChatSettings + + + +ChatSettings is which chat surface a company runs and the policy of +Chatlet. It maps onto Crewlet's chat block. + +_Validation:_ +- MinProperties: 1 + +_Appears in:_ +- [CompanySpec](#companyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `backend` _[ChatBackend](#chatbackend)_ | backend is which chat surface the company runs. Unset derives it, as
Crewlet does: Vendor for a company with a Slack or Mattermost
Integration or a seat with a Slack app, Chatlet for every other. So
there is no default here: a Chatlet written beside such an
Integration, or a Vendor without one, is refused by the engine's dry
run, reported on the Company, since a company's conversations cannot
live in two places. | | Enum: [Chatlet Vendor None]
Optional: \{\}
| +| `chatlet` _[ChatletSettings](#chatletsettings)_ | chatlet is the policy of Chatlet: for backend Chatlet, or for no
backend on a company Crewlet derives Chatlet for. On a company whose
chat is a vendor's nothing would read it, so the engine's dry run
refuses it there, reported on the Company. | | MinProperties: 1
Optional: \{\}
| + + +#### ChatletSettings + + + +ChatletSettings is the policy of Chatlet. It maps onto Crewlet's +chat.chatlet. + +_Validation:_ +- MinProperties: 1 + +_Appears in:_ +- [ChatSettings](#chatsettings) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `messageRetentionDays` _integer_ | messageRetentionDays is how long a message lives before the engine
prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650
days, since under a month somebody away could miss a decision before
reading it. Unset takes Crewlet's default of 365. Crewlet's
`message_retention_days`. | | Maximum: 3650
Minimum: 0
Optional: \{\}
| +| `defaultChannelPrivate` _boolean_ | defaultChannelPrivate makes a room created without a stated
visibility private; unset, such a room is public, readable by the
company's seats. A room created with a visibility keeps it. Crewlet's
`default_channel_private`. | | Optional: \{\}
| +| `typingStatus` _[WorkingStatus](#workingstatus)_ | typingStatus is when a seat shows in a room that it is working on a
message. Unset takes Crewlet's default, Always. Crewlet's
`typing_status`. | | Enum: [Always Addressed]
Optional: \{\}
| + + #### ClassEnvVar @@ -667,13 +726,14 @@ _Appears in:_ | `timeZone` _string_ | timeZone is the company's clock, an IANA time zone such as
Europe/Berlin: where its days begin for budgets, due dates and
schedules. Unset is UTC. Crewlet's `timezone`. | | MaxLength: 64
MinLength: 1
Pattern: `^[A-Za-z][A-Za-z0-9_+-]*(/[A-Za-z0-9_+-]+)*$`
Optional: \{\}
| | `tracker` _[TrackerSettings](#trackersettings)_ | tracker is which work tracker the company runs. | | MinProperties: 1
Optional: \{\}
| | `knowledge` _[KnowledgeSettings](#knowledgesettings)_ | knowledge is which knowledge base the company runs. | | MinProperties: 1
Optional: \{\}
| +| `chat` _[ChatSettings](#chatsettings)_ | chat is which chat surface the company runs: Chatlet, Crewlet's own,
unless it is connected to Slack or Mattermost. Crewlet's `chat`. | | MinProperties: 1
Optional: \{\}
| | `tokenBudget` _[TokenBudget](#tokenbudget)_ | tokenBudget caps the tokens all seats together may spend per window.
Each window binds every agent seat, so Crewlet refuses one below
turn_engine.executor_min_budget_tokens (25000 unless spec.engineConfig
sets turn_engine): an executor launch needs that much of every window
left, and a seat under a smaller ceiling would never launch one. The
refusal comes from the engine's dry run, Invalid on the Company.
Crewlet's `token_budget`. | | MinProperties: 1
Optional: \{\}
| | `llm` _[CompanyLLMSettings](#companyllmsettings)_ | llm is how the company chooses among its model providers. | | MinProperties: 1
Optional: \{\}
| | `assistant` _[CompanyAssistant](#companyassistant)_ | assistant configures Ask Crewlet. Unset leaves it off.
Crewlet's `assistant`. | | MinProperties: 1
Optional: \{\}
| | `embeddings` _[EmbeddingSettings](#embeddingsettings)_ | embeddings is the provider vectors are computed with. Unset leaves
search keyword-only. Crewlet's providers.embeddings. | | Optional: \{\}
| | `sandbox` _[SandboxSettings](#sandboxsettings)_ | sandbox is where the company's agent seats' executors run. Required
of a company with an Agent seat and a model provider, which Crewlet
refuses without it (SandboxRequired). Crewlet's providers.sandbox. | | Optional: \{\}
| | `integrations` _[CompanyIntegrationSettings](#companyintegrationsettings)_ | integrations is company-wide integration behaviour. | | MinProperties: 1
Optional: \{\}
| -| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables, artifacts and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, token_budget,
assistant, providers, integrations, mcp_servers, roles, units or
seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's
spec.integrations. Every
position Crewlet holds a credential in lives under one of those keys, so
engineConfig can never carry a credential: the API server refuses a
Company whose engineConfig names one of them, before anything is
stored. | | MaxItems: 64
Optional: \{\}
| +| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables, artifacts and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, chat,
token_budget, assistant, providers, integrations, mcp_servers, roles,
units or seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's spec.integrations.
Every position Crewlet holds a credential in lives under one of those
keys, so engineConfig can never carry a credential: the API server
refuses a Company whose engineConfig names one of them, before anything
is stored. | | MaxItems: 64
Optional: \{\}
| | `config` _[ConfigSettings](#configsettings)_ | config is how the operator manages the company's document. | \{ managementPolicy:Enforce \} | MinProperties: 1
Optional: \{\}
| | `topology` _[CompanyTopology](#companytopology)_ | topology is how the engine is laid out in pods. | \{ data:map[replicas:1] \} | MinProperties: 1
Optional: \{\}
| | `exposure` _[CompanyExposure](#companyexposure)_ | exposure overrides how the company is reached. | | MinProperties: 1
Optional: \{\}
| @@ -4293,6 +4353,7 @@ _Validation:_ - Enum: [Always Addressed] _Appears in:_ +- [ChatletSettings](#chatletsettings) - [MattermostIntegration](#mattermostintegration) - [SlackIntegration](#slackintegration) diff --git a/internal/render/company.go b/internal/render/company.go index 410c7185..ca70df22 100644 --- a/internal/render/company.go +++ b/internal/render/company.go @@ -485,11 +485,19 @@ func (r *renderer) company() companyDoc { RootSpace: spec.Knowledge.RootSpace, Vectors: spec.Knowledge.Vectors, }, + Chat: chatDoc{Backend: chatBackend(spec.Chat.Backend)}, TokenBudget: tokenBudget(spec.TokenBudget), } if days := spec.Tracker.Worklet.InboxRetentionDays; days != 0 { doc.Tracker.Worklet = &trackerWorkletDoc{InboxRetentionDays: days} } + if chatlet := spec.Chat.Chatlet; chatlet != (v1alpha1.ChatletSettings{}) { + doc.Chat.Chatlet = &chatletDoc{ + MessageRetentionDays: chatlet.MessageRetentionDays, + DefaultChannelPrivate: chatlet.DefaultChannelPrivate, + TypingStatus: workingStatus(chatlet.TypingStatus), + } + } doc.Providers = r.providersDoc() doc.Assistant = r.assistant() doc.Integrations = r.integrations() diff --git a/internal/render/company_test.go b/internal/render/company_test.go index 6b5f07eb..b99f48cd 100644 --- a/internal/render/company_test.go +++ b/internal/render/company_test.go @@ -35,10 +35,10 @@ func companyCases() []companyCase { return []companyCase{ // The example company every page of the docs uses. {name: sampleCompany, exposed: true}, - {name: "minimal", advisories: agentsAlone}, + {name: caseMinimal, advisories: agentsAlone}, {name: "minimal-agent", advisories: agentsAlone}, {name: "org-chart"}, - {name: "slack", exposed: true, advisories: agentsAlone}, + {name: caseSlack, exposed: true, advisories: agentsAlone}, {name: "slack-phrases", exposed: true, advisories: agentsAlone}, {name: "github-enterprise", exposed: true, advisories: agentsAlone}, {name: "github-minimal", exposed: true, advisories: agentsAlone}, @@ -51,6 +51,7 @@ func companyCases() []companyCase { {name: caseAtlassian, exposed: true}, {name: caseSeatPools}, {name: caseOptionalKeys, advisories: []advisory{advisoryNoUnitChannel}}, + {name: "chat"}, } } diff --git a/internal/render/contract_test.go b/internal/render/contract_test.go index f3dfc9bc..d4dc3cae 100644 --- a/internal/render/contract_test.go +++ b/internal/render/contract_test.go @@ -1238,7 +1238,7 @@ func TestContractContactAdvisoryFollowsTheChatSurface(t *testing.T) { {name: "a Slack member id", contact: v1alpha1.SeatContact{SlackUserID: "U0JANEDOE", GitHubLogin: "jane-doe"}}, } { t.Run(c.name, func(t *testing.T) { - in := companyCaseNamed(t, "slack").input(t) + in := companyCaseNamed(t, caseSlack).input(t) person := v1alpha1.Seat{ Namespace: in.Company.Namespace, Name: "jane", Spec: v1alpha1.SeatSpec{ @@ -1349,6 +1349,75 @@ func TestContractScheduleRefusalsReachTheSchedule(t *testing.T) { } } +// TestContractChatSettingsFollowTheEngine has the engine's dry run judge the +// chat settings no API rule can, because each turns on the company's +// Integrations and seats. Chatlet beside a Slack Integration would split the +// company's conversations between two surfaces, Vendor on a company with no +// chat Integration has nothing to talk through, and a chatlet block with no +// backend beside a Slack Integration sets a policy for a chat the company does +// not run: each is refused where Crewlet says, and the document attributes it +// to the Company, which is where the controller reports it. A chatlet block +// with no backend on a company with no chat Integration is Chatlet's policy, +// which the API admits as the engine does, and the engine takes it. +func TestContractChatSettingsFollowTheEngine(t *testing.T) { + client := enginetest.Start(t).Client() + chatlet := v1alpha1.ChatletSettings{DefaultChannelPrivate: new(true)} + for _, c := range []struct { + name string + from string + chat v1alpha1.ChatSettings + refusedAt string + }{ + { + name: "Chatlet beside a Slack Integration", from: caseSlack, + chat: v1alpha1.ChatSettings{Backend: v1alpha1.ChatChatlet}, refusedAt: "chat.backend", + }, + { + name: "Vendor with no chat Integration", from: caseMinimal, + chat: v1alpha1.ChatSettings{Backend: v1alpha1.ChatVendor}, refusedAt: "chat.backend", + }, + { + name: "a chatlet block with no backend beside a Slack Integration", from: caseSlack, + chat: v1alpha1.ChatSettings{Chatlet: chatlet}, refusedAt: "chat.chatlet", + }, + { + name: "a chatlet block with no backend and no chat Integration", from: caseMinimal, + chat: v1alpha1.ChatSettings{Chatlet: chatlet}, + }, + } { + t.Run(c.name, func(t *testing.T) { + in := companyCaseNamed(t, c.from).input(t) + in.Company.Spec.Chat = c.chat + doc, err := Company(in) + if err != nil { + t.Fatal(err) + } + _, err = client.ApplyConfig(t.Context(), engine.ApplyRequest{Document: doc.YAML, DryRun: true}) + if c.refusedAt == "" { + if err != nil { + t.Fatalf("the dry run refused a chat the API admits: %v", err) + } + return + } + verr, ok := errors.AsType[*engine.ValidationError](err) + if !ok { + t.Fatalf("the dry run answered %v, want a refusal", err) + } + if len(verr.Problems) == 0 { + t.Fatalf("the refusal names no problem: %v", verr) + } + want := ObjectRef{Kind: KindCompany, Name: in.Company.Name} + for _, p := range verr.Problems { + got, ok := doc.Attribute(PathOf(p.Segments)) + if path := PathOf(p.Segments).String(); path != c.refusedAt || !ok || got != want { + t.Errorf("the engine's problem at %q (%s) is attributed to %v, %v; want %s, on %v", + path, p.Message, got, ok, c.refusedAt, want) + } + } + }) + } +} + // TestContractExecutorRefusalsReachTheirObject has the engine's dry run // refuse the mistakes about a seat's executor that the render leaves to it, // as no object's own spec shows them, and checks that the document attributes diff --git a/internal/render/credentials_test.go b/internal/render/credentials_test.go index 6efa152f..094f3e7e 100644 --- a/internal/render/credentials_test.go +++ b/internal/render/credentials_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "slices" "strings" "testing" @@ -332,6 +333,12 @@ func engineConfigValidator(t *testing.T) func(entries ...any) field.ErrorList { } } +// engineConfigEntry is an engineConfig entry setting the top-level key of the +// company document to an empty block. +func engineConfigEntry(key string) map[string]any { + return map[string]any{keyName: key, keyValue: map[string]any{}} +} + // TestEngineConfigCannotNameACredentialBlock: engineConfig passes a // top-level block of the company document through as written, so it may // never name a block Crewlet holds a credential under, or a literal could @@ -365,22 +372,41 @@ func TestEngineConfigCannotNameACredentialBlock(t *testing.T) { } } -// engineConfigEntry is a Company.spec.engineConfig entry setting key to an -// empty block. -func engineConfigEntry(key string) map[string]any { - return map[string]any{keyName: key, "value": map[string]any{}} -} - -// TestEngineConfigCannotNameATypedKey holds the rule on -// Company.spec.engineConfig to typedKeys: an entry under a key the render -// writes from a field would be merged over it, or replace it. -func TestEngineConfigCannotNameATypedKey(t *testing.T) { +// TestEngineConfigRefusesEveryTypedKey: the render writes every top-level key +// a field or object of this API owns, and merges a Company's engineConfig +// entries over the result, so an entry naming such a key would replace what +// the render wrote; and Diff tells a key the render writes from one an +// engineConfig entry does by typedKeys. So typedKeys must be every top-level +// key the company document's type renders, and the CRD's rule on +// spec.engineConfig must refuse each of them, while a tuning block the API +// does not type is let through. A key typed without both fails here. +func TestEngineConfigRefusesEveryTypedKey(t *testing.T) { + doc := reflect.TypeFor[companyDoc]() + rendered := make([]string, 0, doc.NumField()) + for f := range doc.Fields() { + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + rendered = append(rendered, name) + } + slices.Sort(rendered) + typed := slices.Sorted(slices.Values(typedKeys)) + if !slices.Equal(rendered, typed) { + t.Errorf("the company document renders the top-level keys %v, and typedKeys is %v", rendered, typed) + } validate := engineConfigValidator(t) + refused := func(key string) bool { + return slices.ContainsFunc(validate(engineConfigEntry(key)), func(e *field.Error) bool { + return strings.HasPrefix(e.Detail, engineConfigRefusal) + }) + } for _, key := range typedKeys { - errs := validate(engineConfigEntry(key)) - if !slices.ContainsFunc(errs, func(e *field.Error) bool { return strings.HasPrefix(e.Detail, engineConfigRefusal) }) { - t.Errorf("the render writes %s from the API, and the CRD lets engineConfig set it: add it to the rule "+ - "on Company.spec.engineConfig", key) + if !refused(key) { + t.Errorf("the render writes %s itself, and the CRD lets engineConfig set it too: add it to the rule on "+ + "Company.spec.engineConfig", key) + } + } + for _, key := range []string{"turn_engine", "learning", "scheduling", "artifacts"} { + if refused(key) { + t.Errorf("engineConfig may not set %s, a tuning block the API does not type", key) } } } diff --git a/internal/render/diff.go b/internal/render/diff.go index 56499336..4167c437 100644 --- a/internal/render/diff.go +++ b/internal/render/diff.go @@ -14,8 +14,8 @@ import ( // objects of the API render, the same list the API refuses in a Company's // engineConfig. Every other top-level key is an engineConfig entry. var typedKeys = []string{ - "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "token_budget", - keyProviders, keyIntegrations, keyMCPServers, keyRoles, keyUnits, keySeatAccessMigrated, + "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "chat", "token_budget", + "assistant", keyProviders, keyIntegrations, keyMCPServers, keyRoles, keyUnits, keySeatAccessMigrated, } // keySeatAccessMigrated is the document's list of the integration kinds whose diff --git a/internal/render/document.go b/internal/render/document.go index 6e5b3a56..a43f8cff 100644 --- a/internal/render/document.go +++ b/internal/render/document.go @@ -14,6 +14,7 @@ type companyDoc struct { Timezone string `json:"timezone,omitempty"` Tracker trackerDoc `json:"tracker,omitzero"` Knowledge knowledgeDoc `json:"knowledge,omitzero"` + Chat chatDoc `json:"chat,omitzero"` TokenBudget tokenBudgetDoc `json:"token_budget,omitzero"` Assistant assistantDoc `json:"assistant,omitzero"` Providers providersDoc `json:"providers,omitzero"` @@ -43,6 +44,17 @@ type knowledgeDoc struct { Vectors *bool `json:"vectors,omitempty"` } +type chatDoc struct { + Backend string `json:"backend,omitempty"` + Chatlet *chatletDoc `json:"chatlet,omitempty"` +} + +type chatletDoc struct { + MessageRetentionDays *int32 `json:"message_retention_days,omitempty"` + DefaultChannelPrivate *bool `json:"default_channel_private,omitempty"` + TypingStatus string `json:"typing_status,omitempty"` +} + type tokenBudgetDoc struct { Day int64 `json:"day,omitempty"` Week int64 `json:"week,omitempty"` diff --git a/internal/render/enums.go b/internal/render/enums.go index 1818d4a5..a8e34afc 100644 --- a/internal/render/enums.go +++ b/internal/render/enums.go @@ -72,6 +72,18 @@ func knowledgeBackend(b v1alpha1.KnowledgeBackend) string { return "" } +func chatBackend(b v1alpha1.ChatBackend) string { + switch b { + case v1alpha1.ChatChatlet: + return "chatlet" + case v1alpha1.ChatVendor: + return "vendor" + case v1alpha1.ChatNone: + return crewletNone + } + return "" +} + func embeddingProviderType(t v1alpha1.EmbeddingProviderType) string { switch t { case v1alpha1.EmbeddingProviderOpenAI, "": diff --git a/internal/render/helpers_test.go b/internal/render/helpers_test.go index 4bb7e344..01a5ac2e 100644 --- a/internal/render/helpers_test.go +++ b/internal/render/helpers_test.go @@ -30,6 +30,14 @@ const ( testUID = types.UID("3f2a9c1e-6b1d-4e7a-9a52-0c8d5f1e2b47") ) +// caseSlack is the company case whose people talk on Slack, and caseMinimal +// the one that connects nothing, which the contract tests start from for a +// company on a vendor's chat and one on Chatlet. +const ( + caseSlack = "slack" + caseMinimal = "minimal" +) + // The names of the Nimbus sample objects the tests break and look up. const ( sampleCompany = "nimbus" @@ -80,6 +88,9 @@ const ( // a unit's, an engineConfig block's. const keyName = "name" +// keyValue is the key an engineConfig entry holds its block under. +const keyValue = "value" + // keyLLM is the key of a model chain in the company document: a seat's, and // the assistant's. const keyLLM = "llm" diff --git a/internal/render/testdata/company/chat.input.yaml b/internal/render/testdata/company/chat.input.yaml new file mode 100644 index 00000000..c5ee53c4 --- /dev/null +++ b/internal/render/testdata/company/chat.input.yaml @@ -0,0 +1,39 @@ +# A company on Chatlet, named rather than derived, with every setting of its +# policy: a horizon on its messages, rooms private unless their creator says +# otherwise, and a working indicator only where a person waits. Its one unit +# declares the room its team talks in, and its one person sits in that team. +apiVersion: crewlet.ai/v1alpha1 +kind: Company +metadata: + name: talky +spec: + engineClassName: basic + companyName: Talky + chat: + backend: Chatlet + chatlet: + messageRetentionDays: 90 + defaultChannelPrivate: true + typingStatus: Addressed +--- +apiVersion: crewlet.ai/v1alpha1 +kind: Unit +metadata: + name: team +spec: + companyRef: + name: talky + displayName: Team + channel: team +--- +apiVersion: crewlet.ai/v1alpha1 +kind: Seat +metadata: + name: founder +spec: + companyRef: + name: talky + kind: Human + displayName: Founder + unitRef: + name: team diff --git a/internal/render/testdata/company/chat.yaml b/internal/render/testdata/company/chat.yaml new file mode 100644 index 00000000..759ed795 --- /dev/null +++ b/internal/render/testdata/company/chat.yaml @@ -0,0 +1,22 @@ +chat: + backend: chatlet + chatlet: + default_channel_private: true + message_retention_days: 90 + typing_status: addressed +name: Talky +seat_access_migrated: +- atlassian +- datadog +- github +- gitlab +- mattermost +- slack +units: +- channel: team + id: team + name: Team + roles: + - handle: founder + kind: human + name: Founder diff --git a/test/crd/company_test.go b/test/crd/company_test.go index d577e38e..678b46b7 100644 --- a/test/crd/company_test.go +++ b/test/crd/company_test.go @@ -40,6 +40,10 @@ func TestCompanyDefaults(t *testing.T) { {"deletionPolicy", s.DeletionPolicy, v1.DeletionPolicyRetain}, {"tracker.backend", s.Tracker.Backend, v1.TrackerWorklet}, {"knowledge.backend", s.Knowledge.Backend, v1.KnowledgeDoclet}, + // No default: unset is a backend the engine derives from the + // company's Integrations, and a Chatlet written for it would be + // refused beside a Slack or Mattermost one. + {"chat.backend", s.Chat.Backend, v1.ChatBackend("")}, } { if !equal(d.got, d.want) { t.Errorf("%s defaulted to %v, want %v", d.field, d.got, d.want) @@ -63,6 +67,14 @@ func fullCompany(c *v1.Company) { RootSpace: "HQ", Vectors: new(true), } + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatChatlet, + Chatlet: v1.ChatletSettings{ + MessageRetentionDays: new(int32(90)), + DefaultChannelPrivate: new(true), + TypingStatus: v1.WorkingStatusAddressed, + }, + } c.Spec.TokenBudget = v1.TokenBudget{Day: 2_000_000} c.Spec.LLM.Order = []v1.LLMProviderReference{ {Name: providerName}, @@ -133,6 +145,67 @@ func TestCompanyValidation(t *testing.T) { }, want: "worklet applies to backend Worklet only", }, + { + name: "a chatlet block on backend Vendor is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatVendor, + Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}, + } + }, + want: msgChatlet, + }, + { + // As Crewlet takes it: with no backend the engine derives one, + // and a chatlet block applies wherever that is Chatlet. + name: "a chatlet block with no backend is accepted", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}} + }, + }, + { + name: "a chatlet block on backend None is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatNone, + Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}, + } + }, + want: msgChatlet, + }, + { + name: "a company with no chat is accepted", + mutate: func(c *v1.Company) { c.Spec.Chat = v1.ChatSettings{Backend: v1.ChatNone} }, + }, + {name: "a message horizon of 0, for ever, is accepted", mutate: chatRetention(0)}, + {name: "a message horizon of 29 days is refused", mutate: chatRetention(29), want: msgChatRetention}, + {name: "a message horizon of 30 days is accepted", mutate: chatRetention(30)}, + {name: "a message horizon of 3650 days is accepted", mutate: chatRetention(3650)}, + { + name: "a message horizon of 3651 days is refused", + mutate: chatRetention(3651), + want: "should be less than or equal to 3650", + }, + { + name: "a message horizon below 0 is refused", + mutate: chatRetention(-1), + want: "should be greater than or equal to 0", + }, + { + name: "a typing status the engine does not have is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatChatlet, + Chatlet: v1.ChatletSettings{TypingStatus: "Sometimes"}, + } + }, + want: "Unsupported value: \"Sometimes\"", + }, + { + name: "a chat backend the engine does not have is refused", + mutate: func(c *v1.Company) { c.Spec.Chat = v1.ChatSettings{Backend: "Slack"} }, + want: "Unsupported value: \"Slack\"", + }, { // The second reference names the same provider: kind defaults // to LLMProvider before the rule runs. @@ -807,7 +880,7 @@ func TestCompanyAssistantDeepWorkValidation(t *testing.T) { func TestCompanyEngineConfigReservedKeys(t *testing.T) { reserved := []string{ - "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", + "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "chat", "token_budget", "assistant", "providers", "integrations", "mcp_servers", "roles", "units", } const msgReservedKey = "engineConfig may not set name, mission" @@ -843,6 +916,13 @@ func TestCompanyEngineConfigReservedKeys(t *testing.T) { runCreateCases(t, company, cases) } +// chatRetention sets Chatlet's message horizon to days. +func chatRetention(days int32) func(*v1.Company) { + return func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{Backend: v1.ChatChatlet, Chatlet: v1.ChatletSettings{MessageRetentionDays: &days}} + } +} + func engineConfigOf(key string) []v1.EngineConfigEntry { return []v1.EngineConfigEntry{{Name: key, Value: &v1.EngineConfigValue{Raw: []byte(`{}`)}}} } diff --git a/test/crd/fixtures_test.go b/test/crd/fixtures_test.go index 638edb4b..d302237a 100644 --- a/test/crd/fixtures_test.go +++ b/test/crd/fixtures_test.go @@ -174,6 +174,8 @@ const ( msgS3EndpointHost = "endpoint's host must be a name, an IPv4 address or an IPv6 address in brackets" msgS3EndpointPort = "endpoint's port must be 1 to 65535" msgUnitID = "a Unit's name is its Crewlet unit id" + msgChatlet = "chatlet applies to backend Chatlet only, or to no backend" + msgChatRetention = "messageRetentionDays must be 0, which keeps messages for ever, or 30 to 3650" msgDuplicate = "Duplicate value" msgThreeReplicas = "Unsupported value: 3" msgStorageFloor = "size must be at least 11Gi" From 4a44fe390ce4e6348d39c6c25af16d6afd4c04d6 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 20:59:58 +0000 Subject: [PATCH 24/36] docs(api): state when an addressed working indicator shows Crewlet now answers "is a person waiting on this seat" with one rule per chat backend (notify.AddressRule), which its working indicator, its delivery check and its prompt all read: a direct message, a personal mention, or a thread the seat is in for a reason that owes an answer, never one it reached through a room-wide address or only by having spoken there. WorkingStatusAddressed still said "a thread the seat follows", which is the second of those exclusions; and since the typed chat block reuses WorkingStatus for Chatlet, it now also says what addresses a seat there: a reply in a thread it started, and a person's message in a unit's room that reached nobody else and so falls to the unit's lead. The enum's values are unchanged; only the API reference moves. Signed-off-by: Ali Yousefian --- api/v1alpha1/integration_types.go | 7 ++++++- docs/reference/api.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/integration_types.go b/api/v1alpha1/integration_types.go index 164558da..8acf2fb5 100644 --- a/api/v1alpha1/integration_types.go +++ b/api/v1alpha1/integration_types.go @@ -16,7 +16,12 @@ const ( // WorkingStatusAlways shows it on every turn a chat message started. WorkingStatusAlways WorkingStatus = "Always" // WorkingStatusAddressed shows it only when a person is plausibly - // waiting: a direct message, a mention, or a thread the seat follows. + // waiting on the seat: a direct message, a mention of it by name, and + // on Slack or Mattermost a thread it is in because it was named there, + // on Chatlet a reply in a thread it started or a person's message in a + // unit's room that reached nobody else and so falls to it as the unit's + // lead. Never a message it reached through a room-wide address, or only + // by having spoken in the thread before. WorkingStatusAddressed WorkingStatus = "Addressed" ) diff --git a/docs/reference/api.md b/docs/reference/api.md index 34f2734a..0b7d6993 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -4360,7 +4360,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `Always` | WorkingStatusAlways shows it on every turn a chat message started.
| -| `Addressed` | WorkingStatusAddressed shows it only when a person is plausibly
waiting: a direct message, a mention, or a thread the seat follows.
| +| `Addressed` | WorkingStatusAddressed shows it only when a person is plausibly
waiting on the seat: a direct message, a mention of it by name, and
on Slack or Mattermost a thread it is in because it was named there,
on Chatlet a reply in a thread it started or a person's message in a
unit's room that reached nobody else and so falls to it as the unit's
lead. Never a message it reached through a room-wide address, or only
by having spoken in the thread before.
| #### WorkletSettings From 835e6afc6ea209b6478e1e2604ef90da02bc2f8a Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 21:01:46 +0000 Subject: [PATCH 25/36] test(engine): answer the client's doubles as the engine now does Crewlet now answers a write whose If-Match names a revision another write has moved past with 412 revision_advanced, a failed precondition, whether the revision was stale when the request arrived or lost the activation's compare-and-set after it; only a write that named no revision loses with 409. The operator matches ErrRevisionAdvanced by its code alone, every write of the document it makes carries an If-Match or a create-only condition, and the one request of its that names no revision, its activation of the running revision again (POST /config/reload), still loses with 409, so it went on working; but the client's doubles answered their stale-revision case 409, to a write that sent no If-Match, which only a reload is. TestErrorMapping's cases may now carry the request they make: the stale case sends If-Match and is answered 412, and a lost race with no If-Match is answered 409, both ErrRevisionAdvanced. TestRevisionAdvancedNamesTheCurrentRevision reads both revisions off the 412. TestEngineStaleIfMatchIsRevisionAdvanced asserts the real engine's 412 too, and fails expecting 409. The doubles' refusal for a missing grant now carries the engine's own sentence for [admin] ("You don't have the access this needs (Admin). Ask an admin for it."), which no operator code reads. The client's comments say which status each write gets, the reload's 409 among them. Signed-off-by: Ali Yousefian --- internal/engine/client.go | 8 +++--- internal/engine/client_test.go | 51 +++++++++++++++++++++++++++------- internal/engine/engine_test.go | 6 ++++ internal/engine/errors.go | 13 +++++++-- internal/engine/types.go | 6 ++-- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/internal/engine/client.go b/internal/engine/client.go index 6e7e396c..b646f90f 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -362,10 +362,10 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // // A document the engine refuses fails with a *ValidationError locating each // problem; a stale req.IfMatch, or a write that lost a race with another -// writer, with ErrRevisionAdvanced; req.CreateOnly on a configured engine with -// ErrAlreadyConfigured; and a write while the engine's keyring does not open -// the fleet's revision, which the write reads first, with -// ErrRevisionUnreadable. +// writer, with ErrRevisionAdvanced (a 412 when req.IfMatch is set, a 409 when +// it is not); req.CreateOnly on a configured engine with ErrAlreadyConfigured; +// and a write while the engine's keyring does not open the fleet's revision, +// which the write reads first, with ErrRevisionUnreadable. func (c *Client) ApplyConfig(ctx context.Context, req ApplyRequest) (ApplyResult, error) { if req.IfMatch != "" && req.CreateOnly { return ApplyResult{}, errors.New("apply the company document: IfMatch and CreateOnly are mutually exclusive: " + diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 3d316920..4341b115 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -33,6 +33,10 @@ const ( baseRevision = "rev-1" + // revisionAdvanced is the code of a write or a reload that lost its race + // to another writer. + revisionAdvanced = "revision_advanced" + // poolLabel and dataPool are the placement label the operator gives a // data pod, and its value. poolLabel = "crewlet.ai/pool" @@ -397,18 +401,34 @@ func TestUnlocatedDocumentRefusalIsStillAValidationError(t *testing.T) { } func TestErrorMapping(t *testing.T) { + // staleWrite is the write the operator makes: conditioned on the + // revision it read, which another write has moved past. + staleWrite := write + staleWrite.IfMatch = "rev-1" cases := []struct { name string + req *engine.ApplyRequest handler http.HandlerFunc sentinel error wantCode string wantStatus int }{ { + // A failed precondition: the request named the revision it + // was built on, and it is not the active one. name: "stale if-match", - handler: answer(http.StatusConflict, + req: &staleWrite, + handler: answer(http.StatusPreconditionFailed, `{"error":"revision_advanced","current_revision_id":"rev-9","your_base":"\"rev-1\""}`), - sentinel: engine.ErrRevisionAdvanced, wantCode: "revision_advanced", wantStatus: http.StatusConflict, + sentinel: engine.ErrRevisionAdvanced, wantCode: revisionAdvanced, wantStatus: http.StatusPreconditionFailed, + }, + { + // A conflict: a write that named no revision lost the race + // to another. The code is the same, and so is the sentinel. + name: "a lost race with no if-match", + handler: answer(http.StatusConflict, + `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`), + sentinel: engine.ErrRevisionAdvanced, wantCode: revisionAdvanced, wantStatus: http.StatusConflict, }, { name: "create-only on a configured engine", @@ -434,7 +454,7 @@ func TestErrorMapping(t *testing.T) { { name: "a token without the grant", handler: answer(http.StatusForbidden, `{"error":"unauthorized","grants":["admin"],`+ - `"message":"You don't have the access this needs (Admin, Worklet, Doclet or Artifacts). Ask an admin for it.",`+ + `"message":"You don't have the access this needs (Admin). Ask an admin for it.",`+ `"reason":"no_grant"}`), sentinel: engine.ErrForbidden, wantCode: "unauthorized", wantStatus: http.StatusForbidden, }, @@ -498,7 +518,11 @@ func TestErrorMapping(t *testing.T) { t.Run(tc.name, func(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", tc.handler) - _, err := apply(f, write) + req := write + if tc.req != nil { + req = *tc.req + } + _, err := apply(f, req) var apiErr *engine.APIError if !errors.As(err, &apiErr) { t.Fatalf("ApplyConfig() = %v, want an *APIError", err) @@ -568,14 +592,21 @@ func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { } } +// TestRevisionAdvancedNamesTheCurrentRevision reads both revisions a lost +// compare-and-set names, from the answer the operator meets: its writes are +// conditioned on the revision they were built on, so the engine refuses one +// another write moved past with 412. func TestRevisionAdvancedNamesTheCurrentRevision(t *testing.T) { f := newFakeEngine(t) - f.handle("PUT /config", answer(http.StatusConflict, + f.handle("PUT /config", answer(http.StatusPreconditionFailed, `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`)) - _, err := apply(f, write) + req := write + req.IfMatch = "rev-1" + _, err := apply(f, req) var apiErr *engine.APIError - if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != "rev-9" || apiErr.StoredRevisionID != "rev-10" { - t.Fatalf("ApplyConfig() = %v, want the current and the stored revision named", err) + if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != "rev-9" || apiErr.StoredRevisionID != "rev-10" || + apiErr.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("ApplyConfig() = %v, want a 412 naming the current and the stored revision", err) } } @@ -619,7 +650,7 @@ func TestRefusalMessageIsWhatTheEngineSaid(t *testing.T) { func TestAuthorityRefusalNamesItsGrants(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", answer(http.StatusForbidden, `{"error":"unauthorized","grants":["admin"],`+ - `"message":"You don't have the access this needs (Admin, Worklet, Doclet or Artifacts). Ask an admin for it.",`+ + `"message":"You don't have the access this needs (Admin). Ask an admin for it.",`+ `"reason":"no_grant"}`)) _, err := apply(f, write) var apiErr *engine.APIError @@ -854,7 +885,7 @@ func TestPutSecretsSendsOneBody(t *testing.T) { "the re-activation lost a race": { reload: `{"error":"revision_advanced","detail":"another configuration write activated first", "stored_revision_id":"rev-3","node":"node-a"}`, - want: &engine.Reload{Error: "revision_advanced", Detail: "another configuration write activated first", + want: &engine.Reload{Error: revisionAdvanced, Detail: "another configuration write activated first", StoredRevisionID: "rev-3", Node: "node-a"}, }, } { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index dbf25d05..e640cae3 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "net/http" "slices" "strings" "testing" @@ -352,6 +353,11 @@ func TestEngineStaleIfMatchIsRevisionAdvanced(t *testing.T) { if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != next.RevisionID { t.Fatalf("ApplyConfig() with a stale If-Match = %v, want the current revision %q named", err, next.RevisionID) } + // A failed precondition, as HTTP says of a condition the request + // stated: only a write that named no revision loses with a 409. + if apiErr.StatusCode != http.StatusPreconditionFailed { + t.Errorf("ApplyConfig() with a stale If-Match answered %d, want 412", apiErr.StatusCode) + } } // TestEngineDryRunLocatesProblems pins what attribution is built on: the engine diff --git a/internal/engine/errors.go b/internal/engine/errors.go index df5f7459..0e5c41f9 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -24,9 +24,13 @@ var ( // ErrRevisionAdvanced is a compare-and-set the caller lost: the active // revision is not the one an If-Match named, or another write activated - // between this write's read and its activation. Re-read the active - // configuration and render again; the engine may have kept this write's - // revision as inert history. + // between this write's read and its activation. The engine answers it + // 412 to a write whose If-Match named a revision, as the operator's + // writes over a running document do, and 409 to one that named none, as + // its activation of the running revision again (ReloadConfig) does; a + // create-only write that loses its race is ErrAlreadyConfigured instead. + // Re-read the active configuration and render again; the engine may have + // kept this write's revision as inert history. ErrRevisionAdvanced = errors.New("the engine's active configuration revision has advanced") // ErrAlreadyConfigured is a create-only write (ApplyRequest.CreateOnly) @@ -218,6 +222,9 @@ func (e *APIError) Is(target error) bool { // same way. A draining engine's 503 is a *NotReadyError instead. return e.StatusCode == http.StatusServiceUnavailable case ErrRevisionAdvanced: + // By code alone: the engine writes it 412 for a write that named + // the revision it was built on and 409 for one that named none, + // and either way what moved is the company's revision. return e.Code == codeRevisionAdvanced case ErrAlreadyConfigured: return e.Code == codeAlreadyConfigured diff --git a/internal/engine/types.go b/internal/engine/types.go index 18ed1519..638020a8 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -236,8 +236,10 @@ type ApplyRequest struct { Summary string // IfMatch makes the write conditional on the active revision being this - // one; a stale one fails with ErrRevisionAdvanced before anything is - // stored. Empty is unconditional. Mutually exclusive with CreateOnly. + // one: a stale one fails with ErrRevisionAdvanced, a 412, before anything + // is stored; a write another activated past once it was stored fails + // with it too. Empty is unconditional. Mutually exclusive with + // CreateOnly. IfMatch string // CreateOnly makes the write land only on an engine, and a fleet, with no From 9986b355893797b2321f0901cb228b671cd2134c Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 08:52:31 +0000 Subject: [PATCH 26/36] docs(docs): name the node with -api in the backup command The runtime page's backup command named the node with -url, which crewlet#142 renames -api on every command that reaches a running node, with no alias, so against it the command stops at "flag provided but not defined: -url". It now names the node with -api, and the sentence before it says which flag that is. The token it pipes in is as it was. Signed-off-by: Ali Yousefian --- docs/concepts/runtime.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index 5e970159..f5037830 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -918,16 +918,16 @@ the start, which `EngineHealthy=False Alarms` shows. Take backups through the engine, which is the only process that can read its estate. `crewlet backup` writes a verified copy into a directory on the data pod's volume, without stopping the engine. Like every command that reaches a -running node it takes its token from `CREWLET_API_TOKEN` alone, never a flag, -so pipe in the access token the Company publishes -([Reaching the engine](#reaching-the-engine)); the directory is named on your -machine, so the timestamp is its clock's, in UTC: +running node it takes the node's address with `-api` and its token from +`CREWLET_API_TOKEN` alone, never a flag, so pipe in the access token the +Company publishes ([Reaching the engine](#reaching-the-engine)); the +directory is named on your machine, so the timestamp is its clock's, in UTC: ```bash dir=/var/lib/crewlet/backups/$(date -u +%Y-%m-%dT%H-%M) kubectl get secret acme-api-access -n acme -o jsonpath='{.data.token}' | base64 -d | kubectl exec -i -n acme acme-data-0 -- sh -c 'CREWLET_API_TOKEN=$(cat) exec crewlet backup \ - -config /etc/crewlet/crewlet.yaml -url http://127.0.0.1:8080 -dir "$1"' sh "$dir" + -config /etc/crewlet/crewlet.yaml -api http://127.0.0.1:8080 -dir "$1"' sh "$dir" ``` Run it at least daily. Copy each backup off the volume (`kubectl cp`), since From bc15be795ded1220bbf0316a35f115bcabd34456 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Wed, 21 Oct 2026 21:06:26 +0000 Subject: [PATCH 27/36] docs(docs): say the slack_ prefix arms Crewlet's chat skills on Slack Crewlet's bundled tool skills now trigger on slack-mcp-server's tool names under the slack_ prefix as well as on Mattermost's: the mentions, hand-off and directives skills on slack_conversations_add_message, and the channel skill on slack_channels_list, slack_channels_me and slack_users_search too. The guide's Slack MCPServer sets that prefix and said nothing about why, so another prefix would have looked harmless. Say it in the Slack section, linking the engine's own account of the pin. The Mattermost section named only the posting tool and the mentions skill, while the channel skill already triggered on three more of that server's tools and the mentions skill on its edit; name all five. Every name was read in the pinned releases, 1.3.0 and 0.6.1. Signed-off-by: Ali Yousefian --- docs/guides/tool-servers.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/guides/tool-servers.md b/docs/guides/tool-servers.md index 562d726e..ac116681 100644 --- a/docs/guides/tool-servers.md +++ b/docs/guides/tool-servers.md @@ -512,9 +512,15 @@ Site URL's path. `mcp-server-mattermost` sends every call to that address with `httpx`, which follows no redirect unless told to, and it never tells it, so at an address Mattermost redirects no agent can post. -The prefix makes its posting tool `mattermost_post_message`, the name -Crewlet's [mentions skill](https://github.com/crewlet/crewlet/blob/main/examples/tool-skills/platform-mentions.md) -triggers on. Each seat gives it `MATTERMOST_TOKEN`, the bot token of its +The prefix makes the server's tools `mattermost_post_message`, +`mattermost_update_message`, `mattermost_list_my_channels`, +`mattermost_list_public_channels` and `mattermost_search_users`, the names +Crewlet's bundled [tool skills](https://github.com/crewlet/crewlet/tree/main/examples/tool-skills) +trigger on: its [mentions skill](https://github.com/crewlet/crewlet/blob/main/examples/tool-skills/platform-mentions.md) +among them, which teaches a seat Mattermost's `@username` mentions and thread +replies before its first post. Under another prefix, or a release that +renames one of those tools, the skills go quiet without a word. Each seat +gives it `MATTERMOST_TOKEN`, the bot token of its `spec.integrations.mattermost` ([Mattermost](integrations.md#mattermost); Crewlet's [Mattermost](https://github.com/crewlet/crewlet/blob/main/docs/integrations/mattermost.md#configure-in-yaml)). @@ -577,6 +583,15 @@ spec: toolPrefix: slack_ ``` +The prefix does here what it does on Mattermost: it makes the server's tools +`slack_conversations_add_message`, `slack_channels_list`, +`slack_channels_me` and `slack_users_search`, the names the same bundled +skills trigger on, so a seat posting on Slack is taught Slack's +`<@member-id>` mentions and thread replies before its first post. Under +another prefix, or a release that renames one of those tools, they go quiet +without a word (Crewlet's +[Slack](https://github.com/crewlet/crewlet/blob/main/docs/integrations/slack.md#configure-in-yaml)). + `slack-mcp-server` registers `conversations_add_message`, its only tool that posts, when `SLACK_MCP_ADD_MESSAGE_TOOL` is set, or when `SLACK_MCP_ENABLED_TOOLS`, the list of tools it offers, names it; a list that From 4c19e94c151a1987513a9c1a454d0ea443a8ec68 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 04:31:41 +0000 Subject: [PATCH 28/36] docs(engine): describe the 503 of an activation the engine can't confirm Crewlet answers a PUT /config whose activation it sent to the fleet's coordination store, and never heard back about, with 503 unavailable, "outcome": "unknown", the stored_revision_id and the node that stored it, under the Retry-After of an undecided answer, with a hint saying how to look (RefuseActivationUnknown, as crewlet's main words it): the fleet may run that revision or may not, and only a read of the fleet's activation says which. #142 also answers a condition of the node on /config, /setup and /secrets (a coordination store that did not answer, its store busy with other writes or closed for a snapshot, a deadline) as 503 unavailable with the condition in words, and a Retry-After where waiting clears it. What the operator does with the unconfirmed activation is the earlier commit's, fix(controller): never write twice over an activation left unconfirmed: engine.UnconfirmedActivation reads the answer, the write is recorded in status.config.unconfirmed, and the next reconcile reads the fleet's activation before it writes anything, sending the write again on its first condition only while the fleet still runs the revision it was built on. That commit wrote the troubleshooting guide's account and the reconcile's. What the client still said was out of date: its package comment and ErrUnavailable called every 503 a request the engine asks to be sent again, and ApplyConfig named no such answer. Read that way, this answer says to send the same write again at once, which loses the race to its own first attempt once that one did activate. So ErrUnavailable says what a 503 is and that UnconfirmedActivation tells this one from the rest, ApplyConfig names it and the read that settles it, StoredRevisionID names both answers that carry a stored revision, and the package comment calls ErrUnavailable a request the engine could not answer now. TestUnconfirmedActivationIsTheEnginesUnknownOutcome also decodes the engine's answer: ErrUnavailable and not ErrRevisionAdvanced, the undecided answer's two-second wait, and the engine's hint, which names the node whose history alone holds the revision. It fails with the Retry-After or the hint no longer read. Signed-off-by: Ali Yousefian --- internal/engine/client.go | 12 +++++++++--- internal/engine/client_test.go | 31 ++++++++++++++++++++++++++++++- internal/engine/errors.go | 30 ++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/internal/engine/client.go b/internal/engine/client.go index b646f90f..a0ade88b 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -14,7 +14,7 @@ // matches ErrRevisionAdvanced, a refused token ErrUnauthorized, a token the // engine will not let do this ErrForbidden, a write a managed document // refuses this token ErrConfigManaged, a revision the engine's keyring does -// not open ErrRevisionUnreadable, a request the engine asks to be sent again +// not open ErrRevisionUnreadable, a request the engine could not answer now // ErrUnavailable, a draining or unconfigured engine ErrNotReady, and anything // else an *APIError with the engine's code and message. // @@ -364,8 +364,14 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // problem; a stale req.IfMatch, or a write that lost a race with another // writer, with ErrRevisionAdvanced (a 412 when req.IfMatch is set, a 409 when // it is not); req.CreateOnly on a configured engine with ErrAlreadyConfigured; -// and a write while the engine's keyring does not open the fleet's revision, -// which the write reads first, with ErrRevisionUnreadable. +// a write while the engine's keyring does not open the fleet's revision, which +// the write reads first, with ErrRevisionUnreadable; and a write whose +// activation the engine sent and could not confirm with ErrUnavailable naming +// the revision it stored, which UnconfirmedActivation reads. The fleet may run +// that revision or may not: the caller keeps it until a read of the fleet's +// activation (Fleet.Activation) says which, and sends the write again, on the +// condition it was first sent on, only once that read shows the fleet still +// runs the revision the write was built on. func (c *Client) ApplyConfig(ctx context.Context, req ApplyRequest) (ApplyResult, error) { if req.IfMatch != "" && req.CreateOnly { return ApplyResult{}, errors.New("apply the company document: IfMatch and CreateOnly are mutually exclusive: " + diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 4341b115..ce01cd0b 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -1440,6 +1440,14 @@ func unknownOutcome(stored, node, base string) string { // and could not confirm: the 503 naming the revision it stored, and no // other. A lost race names a stored revision too, but the engine knows it // did not activate, and a 503 naming none stored nothing to look for. +// +// It also pins how that answer decodes: ErrUnavailable and not +// ErrRevisionAdvanced, the Retry-After of an undecided answer, and the +// engine's hint, which names the node whose history alone holds the revision. +// Read as a lost race it would send the operator to render again over a +// revision that may be its own; the wait is when a read may tell, never a +// cue to send the same write again, which from the same base loses to its +// own first attempt. func TestUnconfirmedActivationIsTheEnginesUnknownOutcome(t *testing.T) { for _, c := range []struct { name, body string @@ -1457,12 +1465,33 @@ func TestUnconfirmedActivationIsTheEnginesUnknownOutcome(t *testing.T) { } { t.Run(c.name, func(t *testing.T) { f := newFakeEngine(t) - f.handle("PUT /config", answer(c.status, c.body)) + f.handle("PUT /config", func(w http.ResponseWriter, r *http.Request) { + if c.stored != "" { + w.Header().Set("Retry-After", "2") + } + answer(c.status, c.body)(w, r) + }) _, err := apply(f, write) stored, ok := engine.UnconfirmedActivation(err) if ok != (c.stored != "") || stored != c.stored { t.Errorf("UnconfirmedActivation(%v) = %q, %t; want %q, %t", err, stored, ok, c.stored, c.stored != "") } + if c.stored == "" { + return + } + var apiErr *engine.APIError + if !errors.As(err, &apiErr) || !errors.Is(err, engine.ErrUnavailable) || + errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("ApplyConfig() = %v, want an *APIError matching ErrUnavailable and not ErrRevisionAdvanced", err) + } + var want struct{ Hint string } + if err := json.Unmarshal([]byte(c.body), &want); err != nil { + t.Fatal(err) + } + if apiErr.RetryAfter != 2*time.Second || apiErr.Hint != want.Hint || !strings.Contains(apiErr.Hint, "node-0") { + t.Errorf("APIError = {retry after %s, hint %q}, want {2s, the engine's hint naming node-0}", + apiErr.RetryAfter, apiErr.Hint) + } }) } if stored, ok := engine.UnconfirmedActivation(nil); ok || stored != "" { diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 0e5c41f9..eac74663 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -61,11 +61,23 @@ var ( // changes nothing. ErrForbidden = errors.New("the engine does not let the API token do this") - // ErrUnavailable is a request the engine could not answer now and asks - // to be sent again (503): it could not decide who is asking, or could - // not read what the request needs, such as who holds a seat a write - // takes away. The *APIError behind it carries RetryAfter when the - // engine said when. + // ErrUnavailable is a request the engine could not answer now (503): it + // could not decide who is asking, or could not read what the request + // needs, such as who holds a seat a write takes away or a store of its + // fleet's that did not answer. The *APIError behind it carries + // RetryAfter where the engine said when to ask again. + // + // A config write can be one too after the engine has stored its + // revision: the activation was sent to the fleet's coordination store, + // whose answer never came back, so the engine may run the revision or + // may not. That 503 carries the Retry-After of an undecided answer, when + // a read may tell, and names the revision in StoredRevisionID and the + // node that stored it in its hint: UnconfirmedActivation tells it from + // every other 503. What settles it is a read of the fleet's + // activation (Fleet.Activation), before the write is sent again: once + // the first did activate, the same write from the same base loses the + // race to it and is refused ErrRevisionAdvanced, and a write built on + // the revision it stored activates the same document a second time. ErrUnavailable = errors.New("the engine could not answer now") // ErrRevisionUnreadable is a revision of the company document that the @@ -155,9 +167,11 @@ type APIError struct { CurrentRevisionID string // StoredRevisionID is the revision a write stored, when the engine kept - // one and could not activate it: a write that lost the race, which left - // it inert, or one whose activation the engine sent and could not - // confirm (UnconfirmedActivation), which it may run or may not. + // one and did not confirm activating it: inert history beside a lost + // race (ErrRevisionAdvanced, ErrAlreadyConfigured), or, on a 503 + // (ErrUnavailable, read by UnconfirmedActivation), the revision whose + // activation the engine sent and could not confirm, which the fleet may + // run or may not. StoredRevisionID string // RevisionID is the revision a refusal is about, when the engine named From 486e3b89eea74156311386cf69dbb2a2f1c234de Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 08:56:44 +0000 Subject: [PATCH 29/36] test(engine): answer the client's 503 doubles as the engine words them crewlet#142 now words a 503 by its Retry-After. Under one, unavailable and identity_unavailable keep the code's own sentence ("Try again in a moment"). Under none, which the engine writes for a condition no wait clears (a record a node can't read yet, a node the fleet evicted), they say "This can't be done here, and waiting won't change that." and "Can't check who you are here, and waiting won't change that.". A write whose outcome is unknown keeps the code's sentence, so the unconfirmed activation's double is still the engine's answer. And a condition of the node on /config, /setup or /secrets (its store busy with other writes, closed while it adopts a peer's snapshot, a deadline) is answered 503 unavailable with the condition in words as the detail, where crewlet's main answered 500, or, for an activation pointer it could not read, 503 with a reason field. The operator goes on doing the right thing: a 503 is ErrUnavailable whatever it says, and the reconcile asks again every ten seconds, or sooner where the engine said when, since nothing else tells it that somebody has cleared the condition (an unconfirmed activation, which UnconfirmedActivation tells apart, is settled by a read before anything is written). But five doubles answered a 503 the engine no longer writes, none answered the 503 a condition of the node now is, and the client's and the guide's account of a 503 had no word for one no wait clears. - TestErrorMapping's engine that could not decide answers as the authority table's unknown does, with a two-second Retry-After, and a new case answers the identity directory a write that removes a human seat reads, refused on this node until somebody acts: no Retry-After, and the no-wait sentence. - TestUnavailableCarriesTheEnginesRetryAfter gives each case the body the engine writes beside its header, the no-wait sentence where there is none. - TestFleetRefusals's unreadable lease table now carries the condition's own words as the detail, which #142 writes beside the code's sentence and the five seconds. - TestStartingWriteIsAskedAgainWhenTheEngineSays, and the starting 503 the controller's quoting tests end on, give the detail #142 writes for a node recovering what it holds, which names its store, its broker and its copy of the estate in parentheses where crewlet's main sets them off with dashes. - TestUnconfirmedActivationIsTheEnginesUnknownOutcome's engine that could not store the write answers as a coordination store that did not answer does: two seconds, and the condition's own words as the detail. With no Retry-After, its message was the one the engine no longer writes beside that body. - TestActiveConfigWhileTheFleetCannotBeRead's unreadable pointer answers as #142 answers a condition on a read: the five seconds of its health tick, and the condition's own words as the detail, which the client takes for the error's message, where it carried main's reason and a message the engine never wrote. - TestPutSecretErrorsNeverContainTheValue gains a store too busy to take the value, answered 503 with the condition in words, beside the fault of the engine's own it still answers 500. ErrUnavailable's comment says what a 503 with no RetryAfter means, and why the client retries it the same way. troubleshooting.md's EngineUnavailable row and its causes name the 503 no wait clears, and the 503 of a condition of the node that clears, which the message quotes, and say the operator keeps asking until it is cleared in the engine. TestErrorMapping's new case fails with ErrUnavailable matching every 503 but identity_unavailable, and TestUnavailableCarriesTheEnginesRetryAfter's "none" case fails with a missing Retry-After read as a second, and TestActiveConfigWhileTheFleetCannotBeRead with the engine's message taken over its detail. Signed-off-by: Ali Yousefian --- docs/guides/troubleshooting.md | 12 ++- internal/controller/sentence_test.go | 2 +- internal/engine/client_test.go | 124 ++++++++++++++++++--------- internal/engine/errors.go | 13 ++- 4 files changed, 104 insertions(+), 47 deletions(-) diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index f7f0664e..f5742c0a 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -120,7 +120,7 @@ with your last change yet. | | False | `Drifted` | Under `Observe`, the engine runs another document; the message names the places that differ. | | | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write, or after the operator found it running where it adopts a revision it did not write. | | | False | `ApplyFailed` | The engine's first data pod, the one the operator reaches, did not apply the revision: it keeps running the one it ran before, and starts on the one its volume names if it restarts, the same one unless the engine is still moving it (the message says which), and the engine's secret store keeps every value of either meanwhile. The message carries the engine's status, its posture (`isolated` on a single node, `shed` or `stuck` beside peers that applied it) and its own account of the failure; see [The engine did not apply a revision](#the-engine-did-not-apply-a-revision). | -| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine answered `503`: it asked to be asked again later (it is starting, or cannot read the coordination store that holds the fleet's revision), or it stored the operator's write and couldn't confirm activating it. Looked at again every ten seconds, or sooner when the engine says when, and a write it couldn't confirm is not sent again until a read shows it did not land; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | +| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine answered `503`: it asked to be asked again later (it is starting, cannot read the coordination store that holds the fleet's revision, or met a condition of its own that clears, such as a store busy with other writes), met something on its side that waiting doesn't clear, or stored the operator's write and couldn't confirm activating it. Looked at again every ten seconds, or sooner when the engine says when, and a write it couldn't confirm is not sent again until a read shows it did not land; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | @@ -372,6 +372,16 @@ requests to `http://-data-0.-data..svc:8080` fail: document it cannot vouch for, and the engine logs `config_request_unavailable`. It clears once the engine's broker is back; the data pod's log says why it is not. +- The engine met a condition of its own that clears: its store busy with + other writes, closed while the node adopts a peer's snapshot or shuts + down, or a request out of time. `/config` and `/secrets` answer + `503 unavailable` with a `Retry-After` the operator follows, and the + condition in the engine's words, which the message quotes. +- The engine met something on its side that waiting doesn't clear. It + answers `503` with no `Retry-After`, and the message says waiting won't + change that and, where the engine can tell, who has to act. The engine's + log names what it met. The operator keeps asking every ten seconds, so + `ConfigApplied` turns once that is cleared in the engine. - The engine stored the operator's write and couldn't confirm that it activated it: the message names the revision. The engine may run it or may not. Nothing needs doing: the operator keeps the write in diff --git a/internal/controller/sentence_test.go b/internal/controller/sentence_test.go index c6035294..190a8546 100644 --- a/internal/controller/sentence_test.go +++ b/internal/controller/sentence_test.go @@ -35,7 +35,7 @@ var ( quotedSentence = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "unavailable", Message: "Can't answer right now. Try again in a moment, or ask another node.", RetryAfter: 2 * time.Second} quotedHint = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "starting", - Message: "This node is recovering what it holds — its store, its broker and its copy of the estate — " + + Message: "This node is recovering what it holds (its store, its broker and its copy of the estate) " + "and serves nothing until it has started.", Hint: `Retry after the Retry-After: once the node serves, /health stops answering status "starting" ` + `and /ready stops naming reason "starting".`, diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index ce01cd0b..a65277fb 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -118,6 +118,17 @@ func answer(status int, body string) http.HandlerFunc { } } +// answerAfter writes body, JSON, with status and a Retry-After of seconds: +// a 503 the engine writes where a wait clears what it met. The engine words a +// 503 by that header (httpjson.UnavailableWith), so a double that leaves it +// off answers with a message the engine never writes beside that body. +func answerAfter(status int, seconds, body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", seconds) + answer(status, body)(w, r) + } +} + // page writes body as a proxy in front of the engine would: not JSON. func page(status int) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { @@ -492,13 +503,24 @@ func TestErrorMapping(t *testing.T) { wantStatus: http.StatusConflict, }, { + // The authority table's unknown: two seconds, and the code's + // own sentence, which invites the wait. name: "an engine that could not decide", - handler: answer(http.StatusServiceUnavailable, `{"error":"unavailable",`+ + handler: answerAfter(http.StatusServiceUnavailable, "2", `{"error":"unavailable",`+ `"message":"Can't answer right now. Try again in a moment, or ask another node."}`), sentinel: engine.ErrUnavailable, wantCode: "unavailable", wantStatus: http.StatusServiceUnavailable, }, { - // A 503 asks for the request again whoever wrote it. + // A condition no wait clears: the identity directory a write + // that removes a human seat reads, refused on this node until + // somebody acts. No Retry-After, and a message that says + // waiting won't change it. + name: "a condition no wait clears", + handler: answer(http.StatusServiceUnavailable, identityUnavailableNoWait), + sentinel: engine.ErrUnavailable, wantCode: "identity_unavailable", wantStatus: http.StatusServiceUnavailable, + }, + { + // A 503 is retried whoever wrote it. name: "a proxy's 503", handler: page(http.StatusServiceUnavailable), sentinel: engine.ErrUnavailable, wantStatus: http.StatusServiceUnavailable, }, @@ -662,25 +684,41 @@ func TestAuthorityRefusalNamesItsGrants(t *testing.T) { } } -// TestUnavailableCarriesTheEnginesRetryAfter reads when the engine asked to -// be sent a request again: a 503's Retry-After in whole seconds, which is -// what the engine writes. A date is not read, nor is the header on anything -// but a 503. -func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { - const identityUnavailable = `{"error":"identity_unavailable",` + +// The engine's refusal of a write that removes a human seat when the node +// could not read the identity directory to see whether anybody holds it, as +// it words it under a wait (identityUnavailableSoon, beside a two-second +// Retry-After) and where no wait clears it (identityUnavailableNoWait, with +// none). +const ( + identityUnavailableSoon = `{"error":"identity_unavailable",` + `"hint":"This write removes a human seat, and this node could not read the identity directory to see ` + `whether anybody holds it; nothing was written.",` + `"message":"Can't check who you are right now. Try again in a moment.","seats":["founder"]}` + identityUnavailableNoWait = `{"error":"identity_unavailable",` + + `"hint":"This write removes a human seat, and this node could not read the identity directory to see ` + + `whether anybody holds it; nothing was written.",` + + `"message":"Can't check who you are here, and waiting won't change that.","seats":["founder"]}` +) + +// TestUnavailableCarriesTheEnginesRetryAfter reads when the engine asked to +// be sent a request again: a 503's Retry-After in whole seconds, which is +// what the engine writes, and none where it wrote none, which it words as a +// wait that changes nothing. A date is not read, nor is the header on +// anything but a 503. +func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { for _, c := range []struct { name string status int header string + body string want time.Duration }{ - {name: "seconds", status: http.StatusServiceUnavailable, header: "2", want: 2 * time.Second}, - {name: "none", status: http.StatusServiceUnavailable}, - {name: "a date", status: http.StatusServiceUnavailable, header: "Fri, 16 Oct 2026 23:59:00 GMT"}, - {name: "not a 503", status: http.StatusConflict, header: "2"}, + {name: "seconds", status: http.StatusServiceUnavailable, header: "2", body: identityUnavailableSoon, + want: 2 * time.Second}, + {name: "none", status: http.StatusServiceUnavailable, body: identityUnavailableNoWait}, + {name: "a date", status: http.StatusServiceUnavailable, header: "Fri, 16 Oct 2026 23:59:00 GMT", + body: identityUnavailableSoon}, + {name: "not a 503", status: http.StatusConflict, header: "2", body: identityUnavailableSoon}, } { t.Run(c.name, func(t *testing.T) { f := newFakeEngine(t) @@ -688,7 +726,7 @@ func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { if c.header != "" { w.Header().Set("Retry-After", c.header) } - answer(c.status, identityUnavailable)(w, r) + answer(c.status, c.body)(w, r) }) _, err := apply(f, write) var apiErr *engine.APIError @@ -778,7 +816,7 @@ func TestStartingWriteIsAskedAgainWhenTheEngineSays(t *testing.T) { w.Header().Set("Retry-After", "10") answer(http.StatusServiceUnavailable, `{"error":"starting",`+ `"message":"This node is still starting and is not serving yet. Try again in a moment.",`+ - `"detail":"This node is recovering what it holds — its store, its broker and its copy of the estate — `+ + `"detail":"This node is recovering what it holds (its store, its broker and its copy of the estate) `+ `and serves nothing until it has started.",`+ `"hint":"Retry after the Retry-After: once the node serves, /health stops answering status \"starting\" `+ `and /ready stops naming reason \"starting\"."}`)(w, r) @@ -833,12 +871,17 @@ func TestPutSecretSendsTheRawValue(t *testing.T) { // TestPutSecretErrorsNeverContainTheValue holds the package's promise on every // way a write can fail: a refusal, an error answer, a proxy's page and a // transport failure. The engine's answers are as it words them: a value past -// what the route takes, and a store it could not write. +// what the route takes; a store too busy to take it, a condition of the node +// crewlet#142 answers 503 with, in words, where crewlet's main answered 500; +// and a fault of its own. func TestPutSecretErrorsNeverContainTheValue(t *testing.T) { const value = "sk-live-do-not-leak" cases := map[string]http.HandlerFunc{ "refused": answer(http.StatusRequestEntityTooLarge, `{"error":"body_too_large","message":"The request body is `+ `larger than this endpoint accepts. Send a smaller document, or split the change across more than one request."}`), + "a store too busy to take it": answerAfter(http.StatusServiceUnavailable, "2", `{"error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node.",`+ + `"detail":"This node was too busy to take this in time. Try again in a moment."}`), "internal error": answer(http.StatusInternalServerError, `{"error":"internal_error",`+ `"message":"Something went wrong inside the engine. The reason is in this node's log."}`), "a proxy's page": page(http.StatusBadGateway), @@ -1396,22 +1439,22 @@ func TestRevisionReadsTheStoredDocument(t *testing.T) { } // TestActiveConfigWhileTheFleetCannotBeRead pins what an engine that cannot -// read its activation pointer answers GET /config with: 503 with a -// Retry-After, never a guess, and the client's error is ErrUnavailable with -// that delay, which the config reconciler retries at rather than reporting a -// fault. +// read its activation pointer answers GET /config with, as crewlet#142 +// answers a condition of the node on a read: 503 unavailable with a +// Retry-After, the five seconds of its health tick, and the condition in words +// as the detail, never a guess. The client's error is ErrUnavailable with that +// delay and those words, which the config reconciler retries at, and quotes, +// rather than reporting a fault. func TestActiveConfigWhileTheFleetCannotBeRead(t *testing.T) { + const said = "This node couldn't reach the rest of the fleet. Try again in a moment." f := newFakeEngine(t) f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":3}`)) - f.handle("GET /config", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "2") - answer(http.StatusServiceUnavailable, `{"error":"unavailable","message":"try again shortly", -"reason":"the fleet's activation could not be read from the coordination store"}`)(w, r) - }) + f.handle("GET /config", answerAfter(http.StatusServiceUnavailable, "5", `{"error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node.","detail":"`+said+`"}`)) _, err := f.client().ActiveConfig(context.Background()) apiErr, ok := errors.AsType[*engine.APIError](err) - if !errors.Is(err, engine.ErrUnavailable) || !ok || apiErr.RetryAfter != 2*time.Second { - t.Fatalf("ActiveConfig() = %v, want ErrUnavailable asking again in 2s", err) + if !errors.Is(err, engine.ErrUnavailable) || !ok || apiErr.RetryAfter != 5*time.Second || apiErr.Message != said { + t.Fatalf("ActiveConfig() = %v, want ErrUnavailable asking again in 5s, in the engine's words", err) } } @@ -1452,25 +1495,26 @@ func TestUnconfirmedActivationIsTheEnginesUnknownOutcome(t *testing.T) { for _, c := range []struct { name, body string status int + retryAfter string stored string }{ - {name: "an activation whose answer was lost", status: http.StatusServiceUnavailable, stored: "rev-10", - body: unknownOutcome("rev-10", "node-0", "rev-9")}, + {name: "an activation whose answer was lost", status: http.StatusServiceUnavailable, retryAfter: "2", + stored: "rev-10", body: unknownOutcome("rev-10", "node-0", "rev-9")}, {name: "a write that lost the race", status: http.StatusConflict, body: `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, {name: "a seed another write beat", status: http.StatusPreconditionFailed, body: `{"error":"already_configured","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, - {name: "an engine that could not store it", status: http.StatusServiceUnavailable, - body: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`}, + {name: "an engine that could not store it", status: http.StatusServiceUnavailable, retryAfter: "2", + body: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node.",` + + `"detail":"This node couldn't reach the rest of the fleet. Try again in a moment."}`}, } { t.Run(c.name, func(t *testing.T) { f := newFakeEngine(t) - f.handle("PUT /config", func(w http.ResponseWriter, r *http.Request) { - if c.stored != "" { - w.Header().Set("Retry-After", "2") - } - answer(c.status, c.body)(w, r) - }) + if c.retryAfter != "" { + f.handle("PUT /config", answerAfter(c.status, c.retryAfter, c.body)) + } else { + f.handle("PUT /config", answer(c.status, c.body)) + } _, err := apply(f, write) stored, ok := engine.UnconfirmedActivation(err) if ok != (c.stored != "") || stored != c.stored { @@ -1533,11 +1577,9 @@ func TestFleetRefusals(t *testing.T) { unavailable := newFakeEngine(t) // The lease table is the coordination store's, and one that did not // answer clears by the health tick's five seconds. - unavailable.handle("GET /fleet", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "5") - answer(http.StatusServiceUnavailable, - `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`)(w, r) - }) + unavailable.handle("GET /fleet", answerAfter(http.StatusServiceUnavailable, "5", + `{"detail":"This node couldn't reach the rest of the fleet. Try again in a moment.","error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node."}`)) if got, err := unavailable.client().Fleet(context.Background()); err == nil { t.Errorf("Fleet() of an engine answering 503 = %+v, want an error", got) } else if apiErr, ok := errors.AsType[*engine.APIError](err); !ok || apiErr.StatusCode != http.StatusServiceUnavailable { diff --git a/internal/engine/errors.go b/internal/engine/errors.go index eac74663..3ee540b2 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -65,7 +65,10 @@ var ( // could not decide who is asking, or could not read what the request // needs, such as who holds a seat a write takes away or a store of its // fleet's that did not answer. The *APIError behind it carries - // RetryAfter where the engine said when to ask again. + // RetryAfter where the engine said when to ask again. Where it said + // nothing, no wait clears what it met, and its message says so: a + // condition of the node, such as a record it can't read yet, that + // somebody clears in the engine, whom the detail names where it can. // // A config write can be one too after the engine has stored its // revision: the activation was sent to the fleet's coordination store, @@ -230,10 +233,12 @@ func (e *APIError) Is(target error) bool { // says nothing about what the engine lets the token do. return e.StatusCode == http.StatusForbidden && slices.Contains(codesForbidden, e.Code) case ErrUnavailable: - // By status alone: a 503 is "ask again later" whoever wrote it, - // and the engine writes it under several codes (unavailable, + // By status alone: a 503 is "not now" whoever wrote it, and the + // engine writes it under several codes (unavailable, // identity_unavailable and the stores' own), each retried the - // same way. A draining engine's 503 is a *NotReadyError instead. + // same way, whether it said when or that no wait clears it: + // nothing else tells the caller that somebody has cleared it. A + // draining engine's 503 is a *NotReadyError instead. return e.StatusCode == http.StatusServiceUnavailable case ErrRevisionAdvanced: // By code alone: the engine writes it 412 for a write that named From 38c3870883ad47ec5db21720dcebeaad77837032 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 23 Oct 2026 09:01:39 +0000 Subject: [PATCH 30/36] test(engine): answer the seat-holder 503 under the engine's own code crewlet#142's engine now refuses a write that removes a human seat, when the node can't read who holds it, under a code of its own: 503 seat_holders_unavailable. It used to be identity_unavailable, whose sentence ("Can't check who you are") is about the caller. The answer also carries its words in `detail`, not `hint`: the identity log's own sentence where the log refused the read, and otherwise that nothing was written and to try again in a moment. Its seat_held hint lost its em dashes too. The client's doubles still answered with the old bodies, so they no longer described anything the engine writes: - TestErrorMapping's "a condition no wait clears" now gives the engine's seat_holders_unavailable body for a node the fleet evicted. It has no Retry-After, the code's no-wait message, and the eviction sentence as its detail. - TestUnavailableCarriesTheEnginesRetryAfter gives each case the body that matches its header, the two-second answer included. - TestSeatHeldNamesEachHolderButNotTheirLogin's hint is the engine's. The operator's behaviour holds. Any 503 is ErrUnavailable whatever its code, so the reconcile asks again every ten seconds, or sooner where the engine says when. ErrUnavailable's comment now names the new code among the 503s it covers. Against that engine the package passes. With ErrUnavailable refusing seat_holders_unavailable, TestErrorMapping's case and three of TestUnavailableCarriesTheEnginesRetryAfter's fail. Signed-off-by: Ali Yousefian --- internal/engine/client_test.go | 47 +++++++++++++++++----------------- internal/engine/errors.go | 9 ++++--- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index a65277fb..32e88ae6 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -511,13 +511,13 @@ func TestErrorMapping(t *testing.T) { sentinel: engine.ErrUnavailable, wantCode: "unavailable", wantStatus: http.StatusServiceUnavailable, }, { - // A condition no wait clears: the identity directory a write - // that removes a human seat reads, refused on this node until - // somebody acts. No Retry-After, and a message that says - // waiting won't change it. + // A condition no wait clears: who holds the human seats a + // write removes, read from the identity directory, refused on + // this node until somebody acts. No Retry-After, and a + // message that says waiting won't change it. name: "a condition no wait clears", - handler: answer(http.StatusServiceUnavailable, identityUnavailableNoWait), - sentinel: engine.ErrUnavailable, wantCode: "identity_unavailable", wantStatus: http.StatusServiceUnavailable, + handler: answer(http.StatusServiceUnavailable, seatHoldersUnavailableNoWait), + sentinel: engine.ErrUnavailable, wantCode: "seat_holders_unavailable", wantStatus: http.StatusServiceUnavailable, }, { // A 503 is retried whoever wrote it. @@ -685,19 +685,20 @@ func TestAuthorityRefusalNamesItsGrants(t *testing.T) { } // The engine's refusal of a write that removes a human seat when the node -// could not read the identity directory to see whether anybody holds it, as -// it words it under a wait (identityUnavailableSoon, beside a two-second -// Retry-After) and where no wait clears it (identityUnavailableNoWait, with -// none). +// could not read the identity directory to see who holds it, under a code of +// its own (seat_holders_unavailable), as it words it under a wait +// (seatHoldersUnavailableSoon, beside a two-second Retry-After) and where no +// wait clears it (seatHoldersUnavailableNoWait, with none: the node answering +// was evicted, and its detail says who acts). const ( - identityUnavailableSoon = `{"error":"identity_unavailable",` + - `"hint":"This write removes a human seat, and this node could not read the identity directory to see ` + - `whether anybody holds it; nothing was written.",` + - `"message":"Can't check who you are right now. Try again in a moment.","seats":["founder"]}` - identityUnavailableNoWait = `{"error":"identity_unavailable",` + - `"hint":"This write removes a human seat, and this node could not read the identity directory to see ` + - `whether anybody holds it; nothing was written.",` + - `"message":"Can't check who you are here, and waiting won't change that.","seats":["founder"]}` + seatHoldersUnavailableSoon = `{"detail":"This node couldn't read who holds the human seats this change ` + + `removes, so nothing was written. Try again in a moment.","error":"seat_holders_unavailable",` + + `"message":"Can't check who holds the seats this change removes right now, so nothing was written. ` + + `Try again in a moment.","seats":["founder"]}` + seatHoldersUnavailableNoWait = `{"detail":"The node answering this has been evicted from the fleet, so it ` + + `can't answer. Try another node, or have an administrator readmit the one that refused.",` + + `"error":"seat_holders_unavailable","message":"Can't check here who holds the seats this change removes, ` + + `so nothing was written. Waiting won't change that.","seats":["founder"]}` ) // TestUnavailableCarriesTheEnginesRetryAfter reads when the engine asked to @@ -713,12 +714,12 @@ func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { body string want time.Duration }{ - {name: "seconds", status: http.StatusServiceUnavailable, header: "2", body: identityUnavailableSoon, + {name: "seconds", status: http.StatusServiceUnavailable, header: "2", body: seatHoldersUnavailableSoon, want: 2 * time.Second}, - {name: "none", status: http.StatusServiceUnavailable, body: identityUnavailableNoWait}, + {name: "none", status: http.StatusServiceUnavailable, body: seatHoldersUnavailableNoWait}, {name: "a date", status: http.StatusServiceUnavailable, header: "Fri, 16 Oct 2026 23:59:00 GMT", - body: identityUnavailableSoon}, - {name: "not a 503", status: http.StatusConflict, header: "2", body: identityUnavailableSoon}, + body: seatHoldersUnavailableSoon}, + {name: "not a 503", status: http.StatusConflict, header: "2", body: seatHoldersUnavailableSoon}, } { t.Run(c.name, func(t *testing.T) { f := newFakeEngine(t) @@ -763,7 +764,7 @@ func TestSeatHeldNamesEachHolderButNotTheirLogin(t *testing.T) { `"release":{"person":"p-2","kind":"machine","login":"svc:release","stage":"active"}},`+ `"hint":"Move each person to another human seat (PATCH /iam/people/{person} with seat) or remove them, `+ `unbind a service account (PATCH /iam/people/{person} with no seat), cancel an open invitation `+ - `(DELETE /iam/invitations/{id}) — each an admin's gesture — then send the write again.",`+ + `(DELETE /iam/invitations/{id}), each an admin's gesture, then send the write again.",`+ `"message":"This change removes a seat someone holds or is invited to. Move or remove them, or cancel `+ `the invitation, first. The detail says who."}`)) _, err := apply(f, write) diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 3ee540b2..2f9e149e 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -235,10 +235,11 @@ func (e *APIError) Is(target error) bool { case ErrUnavailable: // By status alone: a 503 is "not now" whoever wrote it, and the // engine writes it under several codes (unavailable, - // identity_unavailable and the stores' own), each retried the - // same way, whether it said when or that no wait clears it: - // nothing else tells the caller that somebody has cleared it. A - // draining engine's 503 is a *NotReadyError instead. + // identity_unavailable, seat_holders_unavailable and the stores' + // own), each retried the same way, whether it said when or that + // no wait clears it: nothing else tells the caller that somebody + // has cleared it. A draining engine's 503 is a *NotReadyError + // instead. return e.StatusCode == http.StatusServiceUnavailable case ErrRevisionAdvanced: // By code alone: the engine writes it 412 for a write that named From 322058f2af8c889b6a0a49a9a066a062c87abd92 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 14:59:20 +0000 Subject: [PATCH 31/36] fix(controller): promise no retry time for a 503 waiting won't clear crewlet#142 answers a condition no wait clears (a record a node can't read yet, a node the fleet evicted) with 503 under a code of its own and no Retry-After, and words it so: "This can't be done here, and waiting won't change that." The operator still wrote ConfigApplied=False EngineUnavailable as "The operator's request to the company's engine failed, and it tries again in 10s: engine answered 503 unavailable: This can't be done here, and waiting won't change that.", its own half promising the retry the engine's half says changes nothing, and its SecretWithdrawalFailed event did the same. The client now keeps a 503's outcome (APIError.Outcome) and tells this answer apart (Waitless): a 503 under a code of the engine's, with no Retry-After, that is not about a write whose outcome the engine does not know, which may have landed and whose answer says to try it again. The condition and the event then say that the engine answered that waiting won't clear what it met, so somebody has to clear it in the engine, and that the operator asks again every ten seconds until then, since nothing else tells it the condition has cleared. A 503 under a Retry-After keeps its promise of a retry when the engine said. The troubleshooting guide says the condition words it so. TestWaitlessIsTheEnginesNoWait503 holds Waitless to the engine's answers: the no-wait 503 under unavailable and under seat_holders_unavailable, and neither a 503 under a Retry-After, one whose write may have landed, a proxy's nor a 409. It fails with the outcome left unread. TestANoWait503PromisesNoRetry holds the condition and the event to it, and fails with the condition's wording dropped. TestAConditionThatQuotesAnErrorEndsItOnce gains the no-wait 503. Signed-off-by: Ali Yousefian --- docs/guides/troubleshooting.md | 9 +++--- internal/controller/config_run.go | 14 +++++++-- internal/controller/config_withdraw.go | 7 ++++- internal/controller/sentence_test.go | 42 +++++++++++++++++++++++++- internal/engine/client_test.go | 38 +++++++++++++++++++++++ internal/engine/errors.go | 22 ++++++++++++++ internal/engine/seats.go | 3 +- 7 files changed, 125 insertions(+), 10 deletions(-) diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index f5742c0a..748c7b5b 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -378,10 +378,11 @@ requests to `http://-data-0.-data..svc:8080` fail: `503 unavailable` with a `Retry-After` the operator follows, and the condition in the engine's words, which the message quotes. - The engine met something on its side that waiting doesn't clear. It - answers `503` with no `Retry-After`, and the message says waiting won't - change that and, where the engine can tell, who has to act. The engine's - log names what it met. The operator keeps asking every ten seconds, so - `ConfigApplied` turns once that is cleared in the engine. + answers `503` with no `Retry-After`, saying waiting won't change that and, + where it can tell, who has to act, and the condition says so before it + quotes the engine, rather than promise a retry that changes nothing. The + engine's log names what it met. The operator keeps asking every ten + seconds, so `ConfigApplied` turns once that is cleared in the engine. - The engine stored the operator's write and couldn't confirm that it activated it: the message names the revision. The engine may run it or may not. Nothing needs doing: the operator keeps the write in diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index c88165ce..4ebd8f2b 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -1630,11 +1630,19 @@ func (run *configRun) revisionUnreadable(err error) bool { // unavailable records an engine that could not be reached, or that failed a // request for a reason of its own. func (run *configRun) unavailable(err error) { + message := fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s", + retryAfter(err), err) + if engine.Waitless(err) { + // The engine says waiting won't change what it met, so a promise + // of a retry in ten seconds would contradict the words it quotes: + // the operator asks again only because nothing else tells it that + // somebody has cleared it. + message = fmt.Sprintf("The engine answered that waiting won't clear what it met, so somebody has to clear it "+ + "in the engine; the operator asks again every %s until then: %s", engineRetryInterval, err) + } run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: ended(fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s", - retryAfter(err), err)), + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: ended(message), } } diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index 9b6223d5..f02e4980 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -430,9 +430,14 @@ func (run *configRun) settleRefused( func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Result { retry := run.engineRetry(err) logf.FromContext(ctx).Error(err, "Failed to keep the value of a deleted Secret from the engine") + again := fmt.Sprintf("the operator tries again in %s", retry) + if engine.Waitless(err) { + again = fmt.Sprintf("the engine answered that waiting won't clear what it met, so somebody has to clear it in "+ + "the engine, and the operator asks again every %s until then", retry) + } run.event(corev1.EventTypeWarning, "SecretWithdrawalFailed", "ReloadConfig", "%s", ended(fmt.Sprintf( "The engine may still start a tool server or an executor run with the value of a Secret that no longer exists, "+ - "which the revision it runs lets go without; the operator tries again in %s: %s", retry, err))) + "which the revision it runs lets go without; %s: %s", again, err))) return ctrl.Result{RequeueAfter: retry} } diff --git a/internal/controller/sentence_test.go b/internal/controller/sentence_test.go index 190a8546..5c9470ef 100644 --- a/internal/controller/sentence_test.go +++ b/internal/controller/sentence_test.go @@ -41,6 +41,11 @@ var ( `and /ready stops naming reason "starting".`, RetryAfter: 10 * time.Second} errQuotedClause = errors.New("dial tcp 10.0.0.7:8080: connect: connection refused") + // quotedNoWait is the engine's 503 for a condition no wait clears, as + // crewlet#142 words it: quotedSentence's code, with no Retry-After and a + // sentence saying so. + quotedNoWait = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: quotedSentence.Code, + Message: "This node has been evicted from the fleet. Have an administrator readmit it."} ) // endsOnce reports whether message ends with exactly one full stop: one not @@ -77,7 +82,7 @@ func TestEndedEndsASentenceOnce(t *testing.T) { // with an error the operator met to one full stop, whether the error ends in // the engine's sentence or in Go's words. func TestAConditionThatQuotesAnErrorEndsItOnce(t *testing.T) { - for _, err := range []error{quotedSentence, quotedHint, errQuotedClause} { + for _, err := range []error{quotedSentence, quotedHint, quotedNoWait, errQuotedClause} { t.Run(err.Error(), func(t *testing.T) { run := &configRun{CompanyConfigReconciler: &CompanyConfigReconciler{}, company: &crewletaiv1alpha1.Company{}} run.unavailable(err) @@ -100,6 +105,41 @@ func TestAConditionThatQuotesAnErrorEndsItOnce(t *testing.T) { } } +// TestANoWait503PromisesNoRetry holds the condition and the event that quote +// the engine's 503 for a condition no wait clears to saying so: no retry in so +// many seconds, which the engine's own words beside it contradict, but that +// somebody clears it in the engine while the operator goes on asking. A 503 +// under a Retry-After keeps its promise of a retry when the engine said. +func TestANoWait503PromisesNoRetry(t *testing.T) { + ctx := context.Background() + for _, c := range []struct { + name string + err error + noWait bool + }{ + {"no wait clears it", quotedNoWait, true}, + {"a wait clears it", quotedSentence, false}, + } { + t.Run(c.name, func(t *testing.T) { + run := &configRun{CompanyConfigReconciler: &CompanyConfigReconciler{}, company: &crewletaiv1alpha1.Company{}} + run.unavailable(c.err) + w := newWithdrawal(t, runningLetsGo) + w.run.withdrawalFailed(ctx, c.err) + event := strings.Join(drained(w.events), "\n") + for name, message := range map[string]string{ + "ConfigApplied": run.configApplied.Message, "SecretWithdrawalFailed": event, + } { + promised, cleared := strings.Contains(message, "tries again in"), + strings.Contains(message, "somebody has to clear it in the engine") + if promised == c.noWait || cleared != c.noWait { + t.Errorf("%s promises a retry: %t, and says somebody clears it: %t; want %t and %t: %s", + name, promised, cleared, !c.noWait, c.noWait, message) + } + } + }) + } +} + // TestAFailedApplyAccountEndsWhatItQuotesOnce holds ApplyFailed's account to // one full stop after what it quotes: the engine's answer when its record of // the attempt cannot be read, and its account of an apply that failed, which diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 32e88ae6..7973e3b0 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -741,6 +741,44 @@ func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { } } +// TestWaitlessIsTheEnginesNoWait503 pins which answers Waitless reads as the +// engine's 503 for a condition no wait clears: one under a code of its own and +// no Retry-After, as crewlet#142 words it ("waiting won't change that"), and +// no other. A 503 with a Retry-After is one a wait clears; one whose write +// may have landed carries none and says to retry it; and one naming no code +// is a proxy's. +func TestWaitlessIsTheEnginesNoWait503(t *testing.T) { + const noWait = `{"error":"unavailable","message":"This can't be done here, and waiting won't change that.",` + + `"detail":"This node has been evicted from the fleet. Have an administrator readmit it."}` + for _, c := range []struct { + name string + handler http.HandlerFunc + want bool + }{ + {"a condition no wait clears", answer(http.StatusServiceUnavailable, noWait), true}, + {"a seat-holder read no wait clears", answer(http.StatusServiceUnavailable, seatHoldersUnavailableNoWait), true}, + {"a condition a wait clears", answerAfter(http.StatusServiceUnavailable, "2", + `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`), false}, + {"a write that may have landed", answer(http.StatusServiceUnavailable, + `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node.",`+ + `"outcome":"unknown","op_id":"op-1"}`), false}, + {"a proxy's 503", page(http.StatusServiceUnavailable), false}, + {"a refusal that is no 503", answer(http.StatusConflict, `{"error":"revision_advanced"}`), false}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + f.handle("PUT /config", c.handler) + _, err := apply(f, write) + if got := engine.Waitless(err); got != c.want { + t.Errorf("Waitless(%v) = %t, want %t", err, got, c.want) + } + }) + } + if engine.Waitless(nil) { + t.Error("Waitless(nil) = true, want false") + } +} + // seatFounder is the human seat the seat_held tests find a person on, and // stageActive the stage of a person or a service account who has finished // enrolling, both as the engine names them. diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 2f9e149e..1b6c375a 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -199,6 +199,10 @@ type APIError struct { // request is sent again, from a 503's Retry-After; zero when it did // not say. RetryAfter time.Duration + + // Outcome is what the engine knows of a write a 503 answers: "unknown" + // for one it may or may not have landed, empty for every other answer. + Outcome string } func (e *APIError) Error() string { @@ -283,6 +287,22 @@ func UnconfirmedActivation(err error) (string, bool) { return apiErr.StoredRevisionID, true } +// Waitless reports whether err is the engine's 503 for a condition no wait +// clears: one under a code of its own with no Retry-After, which the engine +// sends only when it knows waiting changes nothing (a record a node can't +// read yet, a node the fleet evicted), and words so ("This can't be done +// here, and waiting won't change that."). Somebody clears it in the engine, +// whom its detail names where it can tell. A write whose outcome the engine +// does not know is not one, though it carries no Retry-After either: it may +// have landed, and the engine's answer says to try it again or ask another +// node (UnconfirmedActivation reads the one a config write gets). Nor is a +// 503 that names no code, which something in front of the engine wrote. +func Waitless(err error) bool { + var apiErr *APIError + return errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusServiceUnavailable && apiErr.Code != "" && + apiErr.RetryAfter == 0 && apiErr.Outcome != outcomeUnknown +} + // NotReadyError is an engine that answered but will not take work now. type NotReadyError struct { // Reason is the engine's own word for why: draining, unconfigured, shed @@ -499,6 +519,7 @@ type refusal struct { RevisionID string `json:"revision_id"` ManagedBy []string `json:"managed_by"` Held map[string]heldAnswer `json:"held"` + Outcome string `json:"outcome"` } // heldAnswer is one holder of a seat as the engine's seat_held refusal sends @@ -563,6 +584,7 @@ func errorFromResponse(status int, header http.Header, body []byte) error { Reason: r.Reason, Grants: r.Grants, RetryAfter: retryAfter(status, header), + Outcome: r.Outcome, } } diff --git a/internal/engine/seats.go b/internal/engine/seats.go index 2961a0cb..531b1cac 100644 --- a/internal/engine/seats.go +++ b/internal/engine/seats.go @@ -182,7 +182,8 @@ type pauseAnswer struct { Detail string `json:"detail"` } -// The outcomes a pause tool answers with (Crewlet's statelog outcomes). +// The outcomes a pause tool answers with, the second also a 503's of a write +// the engine may or may not have landed (Crewlet's statelog outcomes). const ( outcomeApplied = "applied" outcomeUnknown = "unknown" From cd340a475185d0844e9712b6eff349f10c39d503 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 04:52:19 +0000 Subject: [PATCH 32/36] test(engine): answer the unreadable revision doubles as #142 words them crewlet#142 words its 409 unreadable_revision and unsealed_revision for a person. The detail says what happened ("This node's keyring doesn't hold the key that revision is sealed under."), where crewlet's main quotes the keyring's own error ("secrets: open document: secret could not be decrypted."). An unsealed revision's hint names the write that replaces it ("... to replace it, an admin sends the whole document from your own copy with PUT /config, or imports it with `crewlet config import`, either of which stores it sealed."), where main's has it imported again; a lost key's hint is main's. The code, the message, the revision and the missing Retry-After are as they were. The client's doubles (TestErrorMapping, TestAnUnreadableRevisionNamesItAndItsRemedy) and the controller's (TestARevisionTheEngineCannotOpenBlocksTheDocument) still gave main's bodies, so they described nothing this engine writes. They now give its details and its unsealed hint. The operator's behaviour holds: the message is read as the detail whatever it says. Signed-off-by: Ali Yousefian --- internal/controller/config_unreadable_test.go | 27 +++++++++---------- internal/engine/client_test.go | 17 +++++------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go index 07574499..3ba50d19 100644 --- a/internal/controller/config_unreadable_test.go +++ b/internal/controller/config_unreadable_test.go @@ -20,15 +20,16 @@ import ( // the resync interval, not ten seconds on with a promise of a retry that // changes nothing. -// keyringLost and sealMissing are the engine's answers to a read of revision, -// sealed under a key its keyring no longer holds or stored without a seal, as -// crewlet's configapi words them (refuseUnreadable) and its httpjson.Envelope -// sends them, the detail and the hint each ended as a sentence. +// keyringLost and sealMissing are the engine's answers to a read of +// revision, sealed under a key its keyring no longer holds or stored without +// a seal, as crewlet's configapi words them (RefuseUnreadable) and its +// httpjson.Envelope sends them, the detail and the hint each ended as a +// sentence. func keyringLost(revision string) string { return fmt.Sprintf(`{"error":"unreadable_revision","revision_id":%q,`+ `"message":"That revision is sealed under a key this deployment no longer holds, so it cannot be read. `+ `Put the key back in the node's keyring first.",`+ - `"detail":"secrets: open document: secret could not be decrypted.",`+ + `"detail":"This node's keyring doesn't hold the key that revision is sealed under.",`+ `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's `+ `secrets.keys first."}`, revision) } @@ -38,12 +39,10 @@ func sealMissing(revision string) string { `"message":"That revision is stored without a seal, which a build older than the mandatory keyring wrote, `+ `and a node reads only sealed revisions because the seal is what authenticates one. The hint says what `+ `brings its document back.",`+ - `"detail":"secrets: this document is stored unsealed, and a node holding a keyring reads only sealed ones `+ - `— a plaintext payload is one anything that can write the store or the coordination bucket could have `+ - `authored.",`+ - `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown nor `+ - `reverted to; to have its document again, import it from your own copy with PUT /config or `+ - "`crewlet config import`"+`, which stores it sealed."}`, revision) + `"detail":"This node's copy of that revision is stored without a seal, so nothing vouches for who wrote it.",`+ + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown, edited nor `+ + `reverted to; to replace it, an admin sends the whole document from your own copy with PUT /config, or `+ + `imports it with `+"`crewlet config import`"+`, either of which stores it sealed."}`, revision) } // TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine whose @@ -61,13 +60,13 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { }{ { name: "a key the keyring lost", answer: keyringLost, - says: []string{"revision r1", "secrets: open document: secret could not be decrypted", + says: []string{"revision r1", "doesn't hold the key that revision is sealed under", "restore that key to the node's secrets.keys first", "Secret acme-keyring"}, }, { name: "a revision stored without a seal", answer: sealMissing, - says: []string{"revision r1", "stored unsealed", "import it from your own copy", - "the engine's hint says what brings its document back"}, + says: []string{"revision r1", "stored without a seal, so nothing vouches", + "an admin sends the whole document from your own copy", "the engine's hint says what brings its document back"}, }, } { t.Run(c.name, func(t *testing.T) { diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 7973e3b0..a679db2d 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -565,25 +565,22 @@ func TestErrorMapping(t *testing.T) { // unreadableRevision and unsealedRevision are the engine's answers to a read of // a revision its keyring does not open, as crewlet's configapi words them -// (refuseUnreadable) and its httpjson.Envelope sends them, the detail and the +// (RefuseUnreadable) and its httpjson.Envelope sends them, the detail and the // hint each ended as a sentence (httpjson.Sentence): the code's own sentence, -// the revision, the keyring's account of the failure as the detail, and the -// remedy as the hint. +// the revision, what happened as the detail, and the remedy as the hint. const ( unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + - `"revision_id":"rev-4","detail":"secrets: open document: secret could not be decrypted.",` + + `"revision_id":"rev-4","detail":"This node's keyring doesn't hold the key that revision is sealed under.",` + `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's ` + `secrets.keys first."}` unsealedRevision = `{"error":"unsealed_revision","message":"That revision is stored without a seal, which ` + `a build older than the mandatory keyring wrote, and a node reads only sealed revisions because the seal ` + `is what authenticates one. The hint says what brings its document back.","revision_id":"rev-4",` + - `"detail":"secrets: this document is stored unsealed, and a node holding a keyring reads only sealed ones ` + - `— a plaintext payload is one anything that can write the store or the coordination bucket could have ` + - `authored.",` + - `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown nor ` + - `reverted to; to have its document again, import it from your own copy with PUT /config or ` + - "`crewlet config import`" + `, which stores it sealed."}` + `"detail":"This node's copy of that revision is stored without a seal, so nothing vouches for who wrote it.",` + + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown, edited nor ` + + `reverted to; to replace it, an admin sends the whole document from your own copy with PUT /config, or ` + + `imports it with ` + "`crewlet config import`" + `, either of which stores it sealed."}` ) // TestAnUnreadableRevisionNamesItAndItsRemedy reads the engine's answer to a From 964a0f546f4875fe8b6555fbd34361d9a391d1d3 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 04:54:42 +0000 Subject: [PATCH 33/36] fix(controller): report the fleet's revision the engine can't open too crewlet#142 reads the active revision off the fleet's activation pointer, and answers one this node's keyring does not open under two codes of its own: 409 unreadable_fleet_revision when the fleet sealed it under a key this node was never given, and 409 unsealed_fleet_revision when it arrived without a seal, which every engine refuses since every engine seals what it activates. Both name the revision, say what happened in their detail and the remedy in their hint, and carry no Retry-After, as unreadable_revision and unsealed_revision do. The client matched only those two, so the fleet's codes went back to being an engine the operator could not reach: ConfigApplied False EngineUnavailable, a promised retry and a look every ten seconds. Now ErrRevisionUnreadable matches all four, and RevisionUnreadable says what restores each. For unreadable_fleet_revision, every pod has its keyring from Secret -keyring, so the data pod lacks a key the Secret gained after it started and takes it when it restarts, or the key has gone from the Secret and is put back from the backup. For unsealed_fleet_revision, the revision came from outside the fleet, and the engine's hint says what to do. The troubleshooting guide names the four codes, and the engine's config_revision_unreadable log line. TestErrorMapping, TestAnUnreadableRevisionNamesItAndItsRemedy and TestARevisionTheEngineCannotOpenBlocksTheDocument gain the fleet's two answers as #142 words and ends them. They fail with the two codes left out of the sentinel, and the controller's cases with the two remedies dropped. Signed-off-by: Ali Yousefian --- api/v1alpha1/common_types.go | 17 ++++---- docs/guides/troubleshooting.md | 15 ++++++- internal/controller/config_run.go | 23 ++++++++--- internal/controller/config_unreadable_test.go | 41 +++++++++++++++++-- internal/engine/client_test.go | 34 ++++++++++++--- internal/engine/errors.go | 25 ++++++----- 6 files changed, 120 insertions(+), 35 deletions(-) diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 8f8c3bc0..2b9dd436 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -658,14 +658,15 @@ const ( ReasonSeatHeld = "SeatHeld" // ReasonRevisionUnreadable is ConfigApplied=False when the engine's - // keyring does not open the fleet's revision of the company document - // (409 unreadable_revision or unsealed_revision): it is sealed under a - // key the keyring no longer holds, or stored without a seal. The engine - // then refuses to show the document and to take another, so the - // operator writes nothing; no wait changes that, and somebody has to - // give the engine what opens the revision, such as the key put back in - // the Secret -keyring. The message names the revision and - // carries the engine's account and remedy. + // keyring does not open the fleet's revision of the company document (409 + // unreadable_revision or unsealed_revision, or crewlet#142's + // unreadable_fleet_revision or unsealed_fleet_revision): it is sealed + // under a key the keyring does not hold, or stored without a seal. The + // engine then refuses to show the document and to take another, so the + // operator writes nothing; no wait changes that, and somebody has to give + // the engine what opens the revision, such as the key put back in the + // Secret -keyring. The message names the revision and carries the + // engine's account and remedy. ReasonRevisionUnreadable = "RevisionUnreadable" // ReasonInvalid is Applied=False (or ConfigApplied=False) when the diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 748c7b5b..d45c4ff5 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -124,7 +124,7 @@ with your last change yet. | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | -| | False | `RevisionUnreadable` | The engine's keyring does not open the fleet's revision of the company document (`409 unreadable_revision` or `unsealed_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | +| | False | `RevisionUnreadable` | The engine's keyring does not open the fleet's revision of the company document (`409 unreadable_revision`, `unsealed_revision`, `unreadable_fleet_revision` or `unsealed_fleet_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | ### EngineClass @@ -491,7 +491,7 @@ no document until it can open that revision. No wait changes that, and nothing the operator sends does, so the operator writes nothing and does not ask again every ten seconds: it looks again when the data pods restart, and every five minutes besides. The message names the revision, quotes the -engine's account and its remedy, and says which of two it is: +engine's account and its remedy, and says which of four it is: - `unreadable_revision`: the revision is sealed under a key the data pod's keyring does not hold. The keyring is the Secret `-keyring`, which @@ -500,16 +500,27 @@ engine's account and its remedy, and says which of two it is: than the volumes. Put the key back in that Secret from the backup kept with the volumes, with its `crewlet.ai/active-key-id` annotation as it was: the data pods restart with it, and their restart runs the reconcile again. +- `unreadable_fleet_revision`: the revision the fleet's activation pointer + carries is sealed under a key the data pod's keyring was never given. + Every pod of the company has its keyring from `-keyring`, so the + data pod lacks a key the Secret gained after it started, and takes it when + it restarts (the pods roll on their own once the operator sees the + change), or a pod that held the key sealed a revision under it and the key + has since gone from the Secret: put it back as above. - `unsealed_revision`: the revision is stored without a seal, which no keyring opens and no engine reads; only an engine older than the keyring every engine now requires stored one. The engine's hint says what brings its document back. +- `unsealed_fleet_revision`: the revision on the fleet's activation pointer + carries no seal. Every engine seals what it activates, so it was put there + from outside the fleet; the engine's hint says what to do. To see which keys the keyring holds without printing them: ```bash kubectl get secret acme-keyring -n acme -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' kubectl get events -n acme --field-selector involvedObject.name=acme,reason=RevisionUnreadable +kubectl logs acme-data-0 -n acme | grep config_revision_unreadable | tail ``` ## The engine did not apply a revision diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 4ebd8f2b..034e6ef8 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -1604,7 +1604,9 @@ func (run *configRun) engineRetry(err error) time.Duration { // restores the revision: a key the revision is sealed under is one of the // keyring Secret's, which the operator renders into every pod and never // rewrites, so a key gone from it is put back there, from the backup kept with -// the data volumes. +// the data volumes, and a key the fleet sealed under that the data pod lacks +// is one it takes when it restarts with the Secret. A revision without a seal +// no key opens, and the engine's hint says what does. func (run *configRun) revisionUnreadable(err error) bool { var apiErr *engine.APIError if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) { @@ -1614,12 +1616,23 @@ func (run *configRun) revisionUnreadable(err error) bool { if apiErr.RevisionID != "" { revision = "revision " + apiErr.RevisionID } - remedy := fmt.Sprintf("The keyring is Secret %s, which the operator never rewrites: put the key %s is sealed under "+ - "back in it from the backup kept with the data volumes, and the data pods restart with it.", - keyringSecretName(run.company), revision) - if apiErr.Code == "unsealed_revision" { + var remedy string + switch apiErr.Code { + case "unsealed_revision": remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + "requires, and no keyring opens it; the engine's hint says what brings its document back." + case "unsealed_fleet_revision": + remedy = "Every engine seals what it activates, so a revision on the fleet's activation pointer without a " + + "seal was put there from outside the fleet, and no keyring opens it; the engine's hint says what to do." + case "unreadable_fleet_revision": + remedy = fmt.Sprintf("Every pod of the company has its keyring from Secret %s: a data pod that has not "+ + "restarted since the Secret gained the key the fleet sealed %s under takes it when it restarts, and a key "+ + "gone from the Secret is put back in it from the backup kept with the data volumes.", + keyringSecretName(run.company), revision) + default: + remedy = fmt.Sprintf("The keyring is Secret %s, which the operator never rewrites: put the key %s is sealed "+ + "under back in it from the backup kept with the data volumes, and the data pods restart with it.", + keyringSecretName(run.company), revision) } run.blockAndTell(crewletaiv1alpha1.ReasonRevisionUnreadable, "ReadConfig", ended(fmt.Sprintf( "The engine's keyring does not open %s of the company document, so the engine neither shows the document "+ diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go index 3ba50d19..c0001aa2 100644 --- a/internal/controller/config_unreadable_test.go +++ b/internal/controller/config_unreadable_test.go @@ -45,6 +45,28 @@ func sealMissing(revision string) string { `imports it with `+"`crewlet config import`"+`, either of which stores it sealed."}`, revision) } +// fleetKeyMissing and fleetSealMissing are the engine's answers to a read of +// revision met through the fleet's activation pointer, sealed under a key this +// node's keyring was never given or carrying no seal, worded and ended as the +// two above. +func fleetKeyMissing(revision string) string { + return fmt.Sprintf(`{"error":"unreadable_fleet_revision","revision_id":%q,`+ + `"message":"This node doesn't hold the key the fleet sealed its revision under, so it can't read it. `+ + `The hint says what fixes it.",`+ + `"detail":"This node doesn't hold the key the fleet sealed this revision under.",`+ + `"hint":"Add the key the fleet seals under to this node's secrets.keys and restart it."}`, revision) +} + +func fleetSealMissing(revision string) string { + return fmt.Sprintf(`{"error":"unsealed_fleet_revision","revision_id":%q,`+ + `"message":"The fleet's revision arrived without a seal, so this node won't read it. The hint says `+ + `what to do.",`+ + `"detail":"The fleet's revision arrived without a seal, so this node won't read it.",`+ + `"hint":"Every node seals what it activates, so this body came from outside the fleet and is neither shown `+ + `nor edited; to replace it, an admin sends the whole document with PUT /config, or imports it with `+ + "`crewlet config import`"+`, either of which stores it sealed and activates it."}`, revision) +} + // TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine whose // keyring does not open the fleet's revision to ConfigApplied=False // RevisionUnreadable: the message names the revision, quotes the engine's @@ -60,13 +82,23 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { }{ { name: "a key the keyring lost", answer: keyringLost, - says: []string{"revision r1", "doesn't hold the key that revision is sealed under", + says: []string{"doesn't hold the key that revision is sealed under", "restore that key to the node's secrets.keys first", "Secret acme-keyring"}, }, { name: "a revision stored without a seal", answer: sealMissing, - says: []string{"revision r1", "stored without a seal, so nothing vouches", - "an admin sends the whole document from your own copy", "the engine's hint says what brings its document back"}, + says: []string{"stored without a seal, so nothing vouches", "an admin sends the whole document from your own copy", + "the engine's hint says what brings its document back"}, + }, + { + name: "the fleet's revision under a key this node lacks", answer: fleetKeyMissing, + says: []string{"doesn't hold the key the fleet sealed this revision under", + "Add the key the fleet seals under", "Secret acme-keyring", "takes it when it restarts"}, + }, + { + name: "the fleet's revision without a seal", answer: fleetSealMissing, + says: []string{"arrived without a seal", "came from outside the fleet", + "put there from outside the fleet", "the engine's hint says what to do"}, }, } { t.Run(c.name, func(t *testing.T) { @@ -83,7 +115,8 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable { t.Fatalf("ConfigApplied = %+v, want False RevisionUnreadable", got) } - for _, s := range c.says { + // Every answer names the revision the engine runs, r1. + for _, s := range append([]string{"revision r1"}, c.says...) { if !strings.Contains(got.Message, s) { t.Errorf("ConfigApplied does not say %q: %s", s, got.Message) } diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index a679db2d..b56ea182 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -497,6 +497,16 @@ func TestErrorMapping(t *testing.T) { handler: answer(http.StatusConflict, unsealedRevision), sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_revision", wantStatus: http.StatusConflict, }, + { + name: "the fleet's revision sealed under a key this node lacks", + handler: answer(http.StatusConflict, unreadableFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unreadable_fleet_revision", wantStatus: http.StatusConflict, + }, + { + name: "the fleet's revision without a seal", + handler: answer(http.StatusConflict, unsealedFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_fleet_revision", wantStatus: http.StatusConflict, + }, { name: "a proxy's 409", handler: page(http.StatusConflict), @@ -563,11 +573,13 @@ func TestErrorMapping(t *testing.T) { } } -// unreadableRevision and unsealedRevision are the engine's answers to a read of -// a revision its keyring does not open, as crewlet's configapi words them -// (RefuseUnreadable) and its httpjson.Envelope sends them, the detail and the -// hint each ended as a sentence (httpjson.Sentence): the code's own sentence, -// the revision, what happened as the detail, and the remedy as the hint. +// unreadableRevision, unsealedRevision, unreadableFleetRevision and +// unsealedFleetRevision are the engine's answers to a read of a revision its +// keyring does not open, this node's copy or one the fleet's activation +// pointer carries, as crewlet's configapi words them (RefuseUnreadable) and +// its httpjson.Envelope sends them, the detail and the hint each ended as a +// sentence (httpjson.Sentence): the code's own sentence, the revision, what +// happened as the detail, and the remedy as the hint. const ( unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + @@ -581,6 +593,16 @@ const ( `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown, edited nor ` + `reverted to; to replace it, an admin sends the whole document from your own copy with PUT /config, or ` + `imports it with ` + "`crewlet config import`" + `, either of which stores it sealed."}` + unreadableFleetRevision = `{"error":"unreadable_fleet_revision","message":"This node doesn't hold the key ` + + `the fleet sealed its revision under, so it can't read it. The hint says what fixes it.",` + + `"revision_id":"rev-4","detail":"This node doesn't hold the key the fleet sealed this revision under.",` + + `"hint":"Add the key the fleet seals under to this node's secrets.keys and restart it."}` + unsealedFleetRevision = `{"error":"unsealed_fleet_revision","message":"The fleet's revision ` + + `arrived without a seal, so this node won't read it. The hint says what to do.","revision_id":"rev-4",` + + `"detail":"The fleet's revision arrived without a seal, so this node won't read it.",` + + `"hint":"Every node seals what it activates, so this body came from outside the fleet and is neither shown ` + + `nor edited; to replace it, an admin sends the whole document with PUT /config, or imports it with ` + + "`crewlet config import`" + `, either of which stores it sealed and activates it."}` ) // TestAnUnreadableRevisionNamesItAndItsRemedy reads the engine's answer to a @@ -589,7 +611,7 @@ const ( // message and its remedy as the hint, which is everything a person has to act // on, since no wait and no request clears it. func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { - for _, body := range []string{unreadableRevision, unsealedRevision} { + for _, body := range []string{unreadableRevision, unsealedRevision, unreadableFleetRevision, unsealedFleetRevision} { f := newFakeEngine(t) f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) f.handle("GET /config", answer(http.StatusConflict, body)) diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 1b6c375a..21497a05 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -84,13 +84,15 @@ var ( ErrUnavailable = errors.New("the engine could not answer now") // ErrRevisionUnreadable is a revision of the company document that the - // engine's keyring does not open (409 unreadable_revision or - // unsealed_revision): sealed under a key the keyring no longer holds, or - // stored without a seal, which no node reads. The engine answers it to a - // read of the document and to a write, which reads the revision it - // replaces, and sends no Retry-After: no wait opens the revision, and + // engine's keyring does not open (409): sealed under a key the keyring + // no longer holds (unreadable_revision), or, met through the fleet's + // activation pointer, under a key this node was never given + // (unreadable_fleet_revision); or stored without a seal, which no node + // reads (unsealed_revision, unsealed_fleet_revision). The engine answers + // it to a read of the document and to a write, which reads the revision + // it replaces, and sends no Retry-After: no wait opens the revision, and // nothing the caller sends does, since what does is given to the engine - // (a key put back in its keyring, which it reads when it starts). The + // (a key put in its keyring, which it reads when it starts). The // *APIError behind it carries the revision in RevisionID, the engine's // account as its Message and the remedy as its Hint. ErrRevisionUnreadable = errors.New("the engine's keyring does not open a revision of its company document") @@ -124,10 +126,13 @@ const ( var codesForbidden = []string{"unauthorized", "token_refused", "seat_unavailable"} // codesRevisionUnreadable are the refusals of a revision the engine's keyring -// does not open, one per cause because the remedies share nothing: a key put -// back in the keyring (unreadable_revision), or the document brought back -// sealed (unsealed_revision). -var codesRevisionUnreadable = []string{"unreadable_revision", "unsealed_revision"} +// does not open, one per cause and origin because the remedies share nothing: +// a key put back in the keyring (unreadable_revision), a key the fleet seals +// under given to this node (unreadable_fleet_revision), or the document +// brought back sealed (unsealed_revision, unsealed_fleet_revision). +var codesRevisionUnreadable = []string{ + "unreadable_revision", "unsealed_revision", "unreadable_fleet_revision", "unsealed_fleet_revision", +} // codesWithProblems are the refusals of a document: the engine located each // failure in it, so they become a *ValidationError. From 4d11d91e8b1a1af22c3738322195f313aca5f2ff Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 04:58:10 +0000 Subject: [PATCH 34/36] test(render): expect the no-account advisory as #142 now quotes it crewlet#142 now opens its advisory on a Human seat with no account on the company's chat vendor with the seat's handle quoted: 'the human seat "jane" has no account on Slack', where it said "jane is a human seat with no account on Slack". A reader that puts the engine's sentence in sentence case wrote the handle that opened it as a seat nobody declared ("Jane"). TestContractContactAdvisoryFollowsTheChatSurface holds the advisory to a substring of what `crewlet validate` warns, so against that engine its "a GitHub login alone" case found no advisory it expected and one it did not. It now expects the engine's words. The guides and the Seat API describe the advisory without quoting it, and hold. Signed-off-by: Ali Yousefian --- internal/render/contract_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/render/contract_test.go b/internal/render/contract_test.go index d4dc3cae..b585c290 100644 --- a/internal/render/contract_test.go +++ b/internal/render/contract_test.go @@ -1262,7 +1262,7 @@ func TestContractContactAdvisoryFollowsTheChatSurface(t *testing.T) { if c.warns { want = append(want, advisory{ path: doc.Sources[i].Path.String() + ".contact", - says: person.Name + " is a human seat with no account on Slack", + says: fmt.Sprintf("the human seat %q has no account on Slack", person.Name), }) } env := bootstrapEnv(tierA) From a9a1f9897b46093751dd877edb6e47fae06c6442 Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Sat, 24 Oct 2026 22:31:49 +0000 Subject: [PATCH 35/36] fix(controller): report a revision sealed by a newer engine build crewlet#142 ("fix(api): let PUT /config replace an unsealed active revision") tells apart three seals it cannot open where it told two. A payload with no seal at all is unsealed_revision or unsealed_fleet_revision as before; one wrapped as a sealed document in a form the engine's build does not read, most likely a newer build's, now answers 409 unknown_seal_revision, one code for either origin, naming the revision, with no Retry-After, and a detail and hint of its own: "This node's copy of that revision is sealed in a form this build of Crewlet doesn't read." and "A newer build of Crewlet most likely sealed it here, and no write here replaces it, since that would discard it unread; run that build on this node again." (for the fleet's revision, "...; upgrade this node to the build the rest of the fleet runs."). The client did not know the code, so that answer read as an engine the operator could not reach: ConfigApplied False EngineUnavailable, a promised retry and a look every ten seconds, over a revision no wait opens. ErrRevisionUnreadable now matches it, and RevisionUnreadable gives it its own account and remedy: the keyring Secret is not at fault and no write replaces the revision; the data pods run the build that sealed it, the engine image of the Company's EngineClass (with spec.runtime.version as its tag where the class allows that), and a data pod still rolling to it takes it when it restarts. Its lead says the engine does not read the revision's seal, where every other cause says its keyring does not open it, which would send the reader to the keyring. TestErrorMapping, TestAnUnreadableRevisionNamesItAndItsRemedy and TestARevisionTheEngineCannotOpenBlocksTheDocument gain both origins of unknown_seal_revision as the engine words them. They fail with the code left out of the sentinel, and the controller's with the remedy dropped (the keyring's words then). The troubleshooting guide names the fifth code and what fixes it. Signed-off-by: Ali Yousefian --- api/v1alpha1/common_types.go | 20 +++--- docs/concepts/applying.md | 9 +-- docs/guides/troubleshooting.md | 18 ++++-- internal/controller/config_run.go | 59 +++++++++++------- internal/controller/config_unreadable_test.go | 61 ++++++++++++++----- internal/engine/client.go | 12 ++-- internal/engine/client_test.go | 40 +++++++++--- internal/engine/errors.go | 44 +++++++------ 8 files changed, 178 insertions(+), 85 deletions(-) diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 2b9dd436..c10b1e79 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -657,16 +657,18 @@ const ( // still exists and is held: one whose kind turned from Human to Agent. ReasonSeatHeld = "SeatHeld" - // ReasonRevisionUnreadable is ConfigApplied=False when the engine's - // keyring does not open the fleet's revision of the company document (409 + // ReasonRevisionUnreadable is ConfigApplied=False when the engine cannot + // open the fleet's revision of the company document (409 // unreadable_revision or unsealed_revision, or crewlet#142's - // unreadable_fleet_revision or unsealed_fleet_revision): it is sealed - // under a key the keyring does not hold, or stored without a seal. The - // engine then refuses to show the document and to take another, so the - // operator writes nothing; no wait changes that, and somebody has to give - // the engine what opens the revision, such as the key put back in the - // Secret -keyring. The message names the revision and carries the - // engine's account and remedy. + // unreadable_fleet_revision, unknown_seal_revision or + // unsealed_fleet_revision): it is sealed under a key the keyring does not + // hold, sealed in a form the engine's build does not read, or stored + // without a seal. The engine then refuses to show the document and to take + // another, so the operator writes nothing; no wait changes that, and + // somebody has to give the engine what opens the revision, such as the + // key put back in the Secret -keyring, or the build that sealed + // it. The message names the revision and carries the engine's account and + // remedy. ReasonRevisionUnreadable = "RevisionUnreadable" // ReasonInvalid is Applied=False (or ConfigApplied=False) when the diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index e25a4479..5b2c644b 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -437,9 +437,10 @@ operator tries again every ten seconds, or sooner where the engine's `Retry-After` says so. Because the address is the cluster's DNS name of the pod, an operator running outside the cluster (`make run`) cannot reach any engine. When the engine answers but -its keyring does not open the fleet's revision, the Company reports -`RevisionUnreadable` instead, which no retry clears, and the operator looks -again when the data pods restart and every five minutes besides +cannot open the fleet's revision, sealed under a key its keyring lacks, +sealed in a form its build does not read, or stored without a seal, the +Company reports `RevisionUnreadable` instead, which no retry clears, and the +operator looks again when the data pods restart and every five minutes besides ([The engine cannot open its company document](../guides/troubleshooting.md#the-engine-cannot-open-its-company-document)). ### What the engine has read @@ -652,7 +653,7 @@ takes no work for a person that a pause could hold. | Object | Condition | When | |---|---|---| | Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know (it names those, and which of the two it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | -| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine's keyring does not open the fleet's revision, so it neither shows the document nor takes another) or `EngineUnavailable`. | +| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine cannot open the fleet's revision, so it neither shows the document nor takes another) or `EngineUnavailable`. | | Company | `EngineHealthy` | The engine's alarm table, read each time the engine is reached: `True`, `NoAlarms`; `False`, `Alarms`, naming the one standing longest; `Unknown`, `AlarmsNotReported`, before the engine has evaluated them. Not part of `Ready`. | | Company | `IdentityClaimed` | Whether anybody has joined the company, read from the engine's `/health` each time the engine is reached, and every 30 seconds while nobody has, since a person joins where no watch sees it: `True`, `PersonEnrolled`, once its identity directory holds a person; `False`, `Unclaimed`, while nobody has joined, naming up to five of the company's Human seats and the access token to bring the first person in with; `False`, `NoHumanSeat`, when nobody can, for want of a Human seat; `Unknown`, `IdentityNotReported`, when the engine could not read its directory, did not say, or could not be asked. Not part of `Ready`: a company nobody has joined runs its agent seats. | | every part | `Accepted` | `True` once its Company exists; `False` with `CompanyNotFound`, or `Conflict` when it collides with another object of its company. | diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index d45c4ff5..620f120a 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -124,7 +124,7 @@ with your last change yet. | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | -| | False | `RevisionUnreadable` | The engine's keyring does not open the fleet's revision of the company document (`409 unreadable_revision`, `unsealed_revision`, `unreadable_fleet_revision` or `unsealed_fleet_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | +| | False | `RevisionUnreadable` | The engine cannot open the fleet's revision of the company document (`409 unreadable_revision`, `unreadable_fleet_revision`, `unknown_seal_revision`, `unsealed_revision` or `unsealed_fleet_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | ### EngineClass @@ -483,15 +483,15 @@ something a person set up. Or keep the seat: restore its Seat as a Human seat. ## The engine cannot open its company document -`ConfigApplied=False RevisionUnreadable` means the engine's keyring does not -open the fleet's revision of the company document. The engine refuses every +`ConfigApplied=False RevisionUnreadable` means the engine cannot open the +fleet's revision of the company document. The engine refuses every read of the document with `409`, and every write too, since a write reads the revision it replaces; it keeps running what it applied, and shows and takes no document until it can open that revision. No wait changes that, and nothing the operator sends does, so the operator writes nothing and does not ask again every ten seconds: it looks again when the data pods restart, and every five minutes besides. The message names the revision, quotes the -engine's account and its remedy, and says which of four it is: +engine's account and its remedy, and says which of five it is: - `unreadable_revision`: the revision is sealed under a key the data pod's keyring does not hold. The keyring is the Secret `-keyring`, which @@ -507,6 +507,16 @@ engine's account and its remedy, and says which of four it is: it restarts (the pods roll on their own once the operator sees the change), or a pod that held the key sealed a revision under it and the key has since gone from the Secret: put it back as above. +- `unknown_seal_revision`: the revision, the data pod's own copy or the one + the fleet's activation pointer carries, is sealed in a form the engine's + build does not read, most likely by a newer build of Crewlet than the data + pod runs. The keyring is not at fault, putting a key back changes nothing, + and no write replaces it, since that would discard a document somebody + sealed: run the data pods on the build that sealed it, the engine image + their EngineClass names (with the Company's `spec.runtime.version` as its + tag where the class allows that). A data pod still rolling onto that image + takes it when it restarts; one taken back to an older image than it ran + needs the newer one again. The engine's hint says which. - `unsealed_revision`: the revision is stored without a seal, which no keyring opens and no engine reads; only an engine older than the keyring every engine now requires stored one. The engine's hint says what brings its diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 034e6ef8..87f05095 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -1581,13 +1581,13 @@ func (run *configRun) engineFailed(ctx context.Context, eng *engine.Client, err } // engineRetry is how soon the reconcile looks at a Company's engine again -// after err. A revision the engine's keyring does not open is looked at again -// at the resync interval, as an engine with nothing to do is: the engine sends -// no Retry-After for it, no wait opens it, and nothing the operator sends -// does. What does is given to the data pods, a key put back in the keyring -// Secret, which they read only when they start, and the data pods restarting -// changes RuntimeReady, which runs the reconcile at once. Anything else is -// asked again as retryAfter says. +// after err. A revision the engine cannot open is looked at again at the +// resync interval, as an engine with nothing to do is: the engine sends no +// Retry-After for it, no wait opens it, and nothing the operator sends does. +// What does is given to the data pods (a key put back in the keyring Secret, +// which they read only when they start, or the build that sealed the +// revision), and the data pods restarting changes RuntimeReady, which runs +// the reconcile at once. Anything else is asked again as retryAfter says. func (run *configRun) engineRetry(err error) time.Duration { if errors.Is(err, engine.ErrRevisionUnreadable) { return run.resyncInterval() @@ -1595,18 +1595,20 @@ func (run *configRun) engineRetry(err error) time.Duration { return retryAfter(err) } -// revisionUnreadable records an engine whose keyring does not open the -// fleet's revision of the company document (engine.ErrRevisionUnreadable), and -// reports false for any other error. The engine refuses then to show the -// document and to take another, since a write reads the revision it replaces, -// so the operator can neither see the document nor change it. It is -// reported as what blocks the document, in the engine's words, with what -// restores the revision: a key the revision is sealed under is one of the -// keyring Secret's, which the operator renders into every pod and never -// rewrites, so a key gone from it is put back there, from the backup kept with -// the data volumes, and a key the fleet sealed under that the data pod lacks -// is one it takes when it restarts with the Secret. A revision without a seal -// no key opens, and the engine's hint says what does. +// revisionUnreadable records an engine that cannot open the fleet's revision +// of the company document (engine.ErrRevisionUnreadable), and reports false +// for any other error. The engine refuses then to show the document and to +// take another, since a write reads the revision it replaces, so the operator +// can neither see the document nor change it. It is reported as what blocks +// the document, in the engine's words, with what restores the revision: a key +// the revision is sealed under is one of the keyring Secret's, which the +// operator renders into every pod and never rewrites, so a key gone from it is +// put back there, from the backup kept with the data volumes, and a key the +// fleet sealed under that the data pod lacks is one it takes when it restarts +// with the Secret; a seal the engine's build does not read is most likely a +// newer build's, which the data pods run again, since no write replaces a +// document somebody sealed. A revision without a seal no key opens, and the +// engine's hint says what does. func (run *configRun) revisionUnreadable(err error) bool { var apiErr *engine.APIError if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) { @@ -1616,8 +1618,22 @@ func (run *configRun) revisionUnreadable(err error) bool { if apiErr.RevisionID != "" { revision = "revision " + apiErr.RevisionID } + said := fmt.Sprintf("The engine's keyring does not open %s of the company document, so the engine neither shows "+ + "the document nor takes another, and no wait changes that", revision) var remedy string switch apiErr.Code { + case "unknown_seal_revision": + said = fmt.Sprintf("The engine does not read the seal of %s of the company document, so the engine neither "+ + "shows the document nor takes another, and no wait changes that", revision) + class := "the Company's EngineClass" + if run.class != nil { + class = "EngineClass " + run.class.Name + } + remedy = fmt.Sprintf("A build of Crewlet newer than the one the data pod runs most likely sealed it, so the "+ + "keyring Secret %s is not at fault, and no write replaces it, since that would discard it unread. Run the "+ + "data pods on the build that sealed it: the engine image of %s, with the Company's spec.runtime.version as "+ + "its tag where the class allows that. A data pod still rolling to it takes it when it restarts.", + keyringSecretName(run.company), class) case "unsealed_revision": remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + "requires, and no keyring opens it; the engine's hint says what brings its document back." @@ -1634,9 +1650,8 @@ func (run *configRun) revisionUnreadable(err error) bool { "under back in it from the backup kept with the data volumes, and the data pods restart with it.", keyringSecretName(run.company), revision) } - run.blockAndTell(crewletaiv1alpha1.ReasonRevisionUnreadable, "ReadConfig", ended(fmt.Sprintf( - "The engine's keyring does not open %s of the company document, so the engine neither shows the document "+ - "nor takes another, and no wait changes that: %s", revision, err))+" "+remedy) + run.blockAndTell(crewletaiv1alpha1.ReasonRevisionUnreadable, "ReadConfig", + ended(fmt.Sprintf("%s: %s", said, err))+" "+remedy) return true } diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go index c0001aa2..91eb3d20 100644 --- a/internal/controller/config_unreadable_test.go +++ b/internal/controller/config_unreadable_test.go @@ -11,18 +11,19 @@ import ( crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" ) -// These tests hold what the operator does with an engine whose keyring does -// not open the fleet's revision of the company document: the engine answers -// every read of the document, and every write, which reads the revision it -// replaces, 409 with no Retry-After, and nothing clears it but somebody giving -// the engine what opens the revision. So it is reported as what blocks the +// These tests hold what the operator does with an engine that cannot open the +// fleet's revision of the company document: the engine answers every read of +// the document, and every write, which reads the revision it replaces, 409 +// with no Retry-After, and nothing clears it but somebody giving the engine +// what opens the revision. So it is reported as what blocks the // document, in the engine's words and with the remedy, and looked at again at // the resync interval, not ten seconds on with a promise of a retry that // changes nothing. -// keyringLost and sealMissing are the engine's answers to a read of -// revision, sealed under a key its keyring no longer holds or stored without -// a seal, as crewlet's configapi words them (RefuseUnreadable) and its +// keyringLost, sealMissing and unknownSeal are the engine's answers to a read +// of revision, sealed under a key its keyring no longer holds, stored without a +// seal, or sealed in a form its build does not read, as crewlet's configapi +// words them (RefuseUnreadable, UnreadableRevisionError) and its // httpjson.Envelope sends them, the detail and the hint each ended as a // sentence. func keyringLost(revision string) string { @@ -45,10 +46,20 @@ func sealMissing(revision string) string { `imports it with `+"`crewlet config import`"+`, either of which stores it sealed."}`, revision) } -// fleetKeyMissing and fleetSealMissing are the engine's answers to a read of -// revision met through the fleet's activation pointer, sealed under a key this -// node's keyring was never given or carrying no seal, worded and ended as the -// two above. +func unknownSeal(revision string) string { + return fmt.Sprintf(`{"error":"unknown_seal_revision","revision_id":%q,`+ + `"message":"That revision is sealed in a form this build of Crewlet doesn't read, most likely by a newer `+ + `build, so this node can't read or replace it. The hint says what fixes it.",`+ + `"detail":"This node's copy of that revision is sealed in a form this build of Crewlet doesn't read.",`+ + `"hint":"A newer build of Crewlet most likely sealed it here, and no write here replaces it, since that `+ + `would discard it unread; run that build on this node again."}`, revision) +} + +// fleetKeyMissing, fleetSealMissing and fleetUnknownSeal are the engine's +// answers to a read of revision met through the fleet's activation pointer, +// sealed under a key this node's keyring was never given, carrying no seal, or +// sealed in a form this node's build does not read, worded and ended as the +// three above. A seal the build does not read is one code for either origin. func fleetKeyMissing(revision string) string { return fmt.Sprintf(`{"error":"unreadable_fleet_revision","revision_id":%q,`+ `"message":"This node doesn't hold the key the fleet sealed its revision under, so it can't read it. `+ @@ -67,8 +78,17 @@ func fleetSealMissing(revision string) string { "`crewlet config import`"+`, either of which stores it sealed and activates it."}`, revision) } -// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine whose -// keyring does not open the fleet's revision to ConfigApplied=False +func fleetUnknownSeal(revision string) string { + return fmt.Sprintf(`{"error":"unknown_seal_revision","revision_id":%q,`+ + `"message":"That revision is sealed in a form this build of Crewlet doesn't read, most likely by a newer `+ + `build, so this node can't read or replace it. The hint says what fixes it.",`+ + `"detail":"The fleet's revision is sealed in a form this build of Crewlet doesn't read.",`+ + `"hint":"A newer build of Crewlet most likely sealed it, and no write here replaces it, since that would `+ + `discard it unread; upgrade this node to the build the rest of the fleet runs."}`, revision) +} + +// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine that +// cannot open the fleet's revision to ConfigApplied=False // RevisionUnreadable: the message names the revision, quotes the engine's // account and its remedy, and says what restores it in the Company's terms; // the operator writes nothing; and the next look is at the resync interval. @@ -85,6 +105,19 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { says: []string{"doesn't hold the key that revision is sealed under", "restore that key to the node's secrets.keys first", "Secret acme-keyring"}, }, + { + name: "a seal this node's build does not read", answer: unknownSeal, + says: []string{"does not read the seal of revision r1", "sealed in a form this build of Crewlet doesn't read", + "run that build on this node again", "newer than the one the data pod runs", + "keyring Secret acme-keyring is not at fault", "no write replaces it", + "the engine image of EngineClass standard", "takes it when it restarts"}, + }, + { + name: "the fleet's revision under a seal this node's build does not read", answer: fleetUnknownSeal, + says: []string{"does not read the seal of revision r1", "The fleet's revision is sealed in a form", + "upgrade this node to the build the rest of the fleet runs", "keyring Secret acme-keyring is not at fault", + "the engine image of EngineClass standard"}, + }, { name: "a revision stored without a seal", answer: sealMissing, says: []string{"stored without a seal, so nothing vouches", "an admin sends the whole document from your own copy", diff --git a/internal/engine/client.go b/internal/engine/client.go index a0ade88b..2559a43b 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -13,8 +13,8 @@ // that takes a held human seat away a *SeatHeldError, a lost compare-and-set // matches ErrRevisionAdvanced, a refused token ErrUnauthorized, a token the // engine will not let do this ErrForbidden, a write a managed document -// refuses this token ErrConfigManaged, a revision the engine's keyring does -// not open ErrRevisionUnreadable, a request the engine could not answer now +// refuses this token ErrConfigManaged, a revision the engine cannot open +// ErrRevisionUnreadable, a request the engine could not answer now // ErrUnavailable, a draining or unconfigured engine ErrNotReady, and anything // else an *APIError with the engine's code and message. // @@ -279,8 +279,8 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { // own store names where that store outlives a restart, as a data pod's does // (a seat pod's does not: it starts on nothing and applies the fleet's // revision). Before the first apply it fails with ErrNoActiveRevision, and -// on an engine whose keyring does not open the fleet's revision with -// ErrRevisionUnreadable. +// on an engine that cannot open the fleet's revision with +// ErrRevisionUnreadable, which names it. // // An engine that cannot read the activation pointer from its coordination // store answers 503 with a Retry-After, which is ErrUnavailable: "the fleet @@ -364,8 +364,8 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // problem; a stale req.IfMatch, or a write that lost a race with another // writer, with ErrRevisionAdvanced (a 412 when req.IfMatch is set, a 409 when // it is not); req.CreateOnly on a configured engine with ErrAlreadyConfigured; -// a write while the engine's keyring does not open the fleet's revision, which -// the write reads first, with ErrRevisionUnreadable; and a write whose +// a write while the engine cannot open the fleet's revision, which the write +// reads first, with ErrRevisionUnreadable; and a write whose // activation the engine sent and could not confirm with ErrUnavailable naming // the revision it stored, which UnconfirmedActivation reads. The fleet may run // that revision or may not: the caller keeps it until a read of the fleet's diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index b56ea182..1488f1de 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -507,6 +507,16 @@ func TestErrorMapping(t *testing.T) { handler: answer(http.StatusConflict, unsealedFleetRevision), sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_fleet_revision", wantStatus: http.StatusConflict, }, + { + name: "a revision sealed in a form this build does not read", + handler: answer(http.StatusConflict, unknownSealRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unknown_seal_revision", wantStatus: http.StatusConflict, + }, + { + name: "the fleet's revision sealed in a form this build does not read", + handler: answer(http.StatusConflict, unknownSealFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unknown_seal_revision", wantStatus: http.StatusConflict, + }, { name: "a proxy's 409", handler: page(http.StatusConflict), @@ -573,13 +583,16 @@ func TestErrorMapping(t *testing.T) { } } -// unreadableRevision, unsealedRevision, unreadableFleetRevision and -// unsealedFleetRevision are the engine's answers to a read of a revision its -// keyring does not open, this node's copy or one the fleet's activation -// pointer carries, as crewlet's configapi words them (RefuseUnreadable) and -// its httpjson.Envelope sends them, the detail and the hint each ended as a +// unreadableRevision, unsealedRevision, unreadableFleetRevision, +// unsealedFleetRevision, unknownSealRevision and unknownSealFleetRevision are +// the engine's answers to a read of a revision it cannot open, this node's +// copy or one the fleet's activation pointer carries, as crewlet's configapi +// words them (RefuseUnreadable, UnreadableRevisionError) and its +// httpjson.Envelope sends them, the detail and the hint each ended as a // sentence (httpjson.Sentence): the code's own sentence, the revision, what -// happened as the detail, and the remedy as the hint. +// happened as the detail, and the remedy as the hint. A seal the engine's +// build does not read is one code for either origin, told apart by its detail +// and its hint. const ( unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + @@ -603,6 +616,18 @@ const ( `"hint":"Every node seals what it activates, so this body came from outside the fleet and is neither shown ` + `nor edited; to replace it, an admin sends the whole document with PUT /config, or imports it with ` + "`crewlet config import`" + `, either of which stores it sealed and activates it."}` + unknownSealRevision = `{"error":"unknown_seal_revision","message":"That revision is sealed in a form this ` + + `build of Crewlet doesn't read, most likely by a newer build, so this node can't read or replace it. The ` + + `hint says what fixes it.","revision_id":"rev-4",` + + `"detail":"This node's copy of that revision is sealed in a form this build of Crewlet doesn't read.",` + + `"hint":"A newer build of Crewlet most likely sealed it here, and no write here replaces it, since that ` + + `would discard it unread; run that build on this node again."}` + unknownSealFleetRevision = `{"error":"unknown_seal_revision","message":"That revision is sealed in a form ` + + `this build of Crewlet doesn't read, most likely by a newer build, so this node can't read or replace it. ` + + `The hint says what fixes it.","revision_id":"rev-4",` + + `"detail":"The fleet's revision is sealed in a form this build of Crewlet doesn't read.",` + + `"hint":"A newer build of Crewlet most likely sealed it, and no write here replaces it, since that would ` + + `discard it unread; upgrade this node to the build the rest of the fleet runs."}` ) // TestAnUnreadableRevisionNamesItAndItsRemedy reads the engine's answer to a @@ -611,7 +636,8 @@ const ( // message and its remedy as the hint, which is everything a person has to act // on, since no wait and no request clears it. func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { - for _, body := range []string{unreadableRevision, unsealedRevision, unreadableFleetRevision, unsealedFleetRevision} { + for _, body := range []string{unreadableRevision, unsealedRevision, unreadableFleetRevision, unsealedFleetRevision, + unknownSealRevision, unknownSealFleetRevision} { f := newFakeEngine(t) f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) f.handle("GET /config", answer(http.StatusConflict, body)) diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 21497a05..aec39f82 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -84,18 +84,21 @@ var ( ErrUnavailable = errors.New("the engine could not answer now") // ErrRevisionUnreadable is a revision of the company document that the - // engine's keyring does not open (409): sealed under a key the keyring - // no longer holds (unreadable_revision), or, met through the fleet's - // activation pointer, under a key this node was never given - // (unreadable_fleet_revision); or stored without a seal, which no node - // reads (unsealed_revision, unsealed_fleet_revision). The engine answers - // it to a read of the document and to a write, which reads the revision - // it replaces, and sends no Retry-After: no wait opens the revision, and - // nothing the caller sends does, since what does is given to the engine - // (a key put in its keyring, which it reads when it starts). The - // *APIError behind it carries the revision in RevisionID, the engine's - // account as its Message and the remedy as its Hint. - ErrRevisionUnreadable = errors.New("the engine's keyring does not open a revision of its company document") + // engine cannot open (409): sealed under a key the keyring no longer + // holds (unreadable_revision), or, met through the fleet's activation + // pointer, under a key this node was never given + // (unreadable_fleet_revision); sealed in a form this build of the engine + // does not read, most likely by a newer build, in either place + // (unknown_seal_revision); or stored without a seal, which no node reads + // (unsealed_revision, unsealed_fleet_revision). The engine answers it to + // a read of the document and to every write built from it, and sends no + // Retry-After: no wait opens the revision. A sealed one is somebody's + // document, which no write replaces, since that would discard it unread: + // what opens it is given to the engine, a key put in its keyring, which + // it reads when it starts, or the build that sealed it. The *APIError + // behind it carries the revision in RevisionID, the engine's account as + // its Message and the remedy as its Hint. + ErrRevisionUnreadable = errors.New("the engine cannot open a revision of its company document") // ErrNotReady is an engine that is up but should not be sent work: it // is draining for a shutdown, has no company yet, or its posture is shed @@ -125,13 +128,16 @@ const ( // has its own sentinel. var codesForbidden = []string{"unauthorized", "token_refused", "seat_unavailable"} -// codesRevisionUnreadable are the refusals of a revision the engine's keyring -// does not open, one per cause and origin because the remedies share nothing: -// a key put back in the keyring (unreadable_revision), a key the fleet seals -// under given to this node (unreadable_fleet_revision), or the document -// brought back sealed (unsealed_revision, unsealed_fleet_revision). +// codesRevisionUnreadable are the refusals of a revision the engine cannot +// open, one per cause and origin because the remedies share nothing: a key +// put back in the keyring (unreadable_revision), a key the fleet seals under +// given to this node (unreadable_fleet_revision), the build that sealed it +// (unknown_seal_revision, one code for either origin, since that remedy is +// the same for both), or the document brought back sealed +// (unsealed_revision, unsealed_fleet_revision). var codesRevisionUnreadable = []string{ - "unreadable_revision", "unsealed_revision", "unreadable_fleet_revision", "unsealed_fleet_revision", + "unreadable_revision", "unreadable_fleet_revision", "unknown_seal_revision", + "unsealed_revision", "unsealed_fleet_revision", } // codesWithProblems are the refusals of a document: the engine located each @@ -183,7 +189,7 @@ type APIError struct { StoredRevisionID string // RevisionID is the revision a refusal is about, when the engine named - // one: the revision its keyring does not open (ErrRevisionUnreadable). + // one: the revision it cannot open (ErrRevisionUnreadable). RevisionID string // ManagedBy are the token ids that may change the company document, From 29c34863e9fe4b6c7b93ef68e52605d8b6d4318a Mon Sep 17 00:00:00 2001 From: Ali Yousefian Date: Fri, 30 Oct 2026 17:14:17 +0100 Subject: [PATCH 36/36] fix(controller): replace a revision the engine stores without a seal crewlet#142 ("fix(api): let PUT /config replace an unsealed active revision") lets a whole document replace an active revision stored without a seal: PUT /config, dry runs included, over one that answers 409 unsealed_revision or unsealed_fleet_revision goes on and reads nothing from it, judged as a document replacing nothing, while If-Match is still judged against that revision and a dry run names it as its base. Its hint for both codes names exactly that write. Every other write is still refused over it, a reload included, and a revision sealed under a key the keyring lacks, or in a form the build does not read, stays refused for every write, PUT included. The operator blocked the Company on both codes as on the others, RevisionUnreadable saying the engine "neither shows the document nor takes another", so under Enforce a revision the engine's own remedy replaces stayed until somebody stopped the data pod. Now, under Enforce, converge takes the refusal of the read (engine.Unsealed, which answers for the two codes and a named revision only) and goes on as over an engine with no document but for the write's base: the rendered document is written conditioned on that revision. The new revision's summary, ConfigApplied and a Warning RevisionUnreadable event say which revision it replaced, as the engine's active revision, whose nodes ran the one before it. Under Seed, which writes only into an engine with no document, and Observe, which writes nothing, it still blocks, and the remedy says what replaces it: an admin's whole document, as the engine's hint says, or the Enforce policy. unreadable_* and unknown_seal_revision block under every policy as before. A revision without a seal that lands after the reconcile's read, the one way it is still reported under Enforce, is looked at again ten seconds on rather than at the resync interval, since the operator's own next look replaces it. Nothing the secret store holds is deleted before that write. Which names such a revision references is what nobody can read, and it is not what the engine's nodes run: none adopts a body it cannot open, so each runs, and boots again when it restarts, the revision it applied before, reading the store as it finds it. A value deleted while the replacement is held (the engine refusing it, a seat somebody holds, or a write not sent yet) is one such a node may start without. secretsync gains Desired.KeepListed: every owned name the store lists is kept, an absent or released one included, whatever the state records of the revision's reading, while every value is still pushed, and Holders names that revision as what keeps each absent name the store retained, although the listing is Sync's and Withdraw's alone, so that SecretKept and the conditions say why; enforce keeps that way before the write over such a revision, and so does the withdrawal of a document held for a Secret, which read the 409 and reported a SecretWithdrawalFailed at every look. The sync after the write decides by the replacement as after any write, and activates it again for the store it lets go. SecretKept, SecretsSynced and a part's ResolvedRefs line say why a value is kept for it: the revision, which they call the engine's active revision and never the one it runs, is stored without a seal and nobody can read it, while the nodes run the one before it. That revision, the one each node reports running on GET /fleet, is read by its id (GET /config/revisions/{id}) and its names kept beside the listing until the nodes run the replacement, as for any revision a node runs other than the fleet's (keepServing). No activation is asked for over such a revision: the engine neither shows nor activates it again, so a reload is refused every time. Asked for, a document the engine refused, or one held for a Secret, came back every ten seconds over a refusal no retry clears, each time with a SecretWithdrawalFailed. A store change owed, a withdrawal or a value written, stays owed, and the replacement's activation settles it. The revision without a seal settles nothing the store owes, although the fleet's pointer names it at an epoch above every mark: its nodes run the revision before it, which may have activated before the change. That takes nothing new: a store change is settled only by the revision GET /config answers with (settleRead), and over a revision without a seal it answers with none. A seat_held over the replacement is worded as the engine asks: the document does not keep those human seats, since nobody can read whether the revision it replaces had them. A write over the revision the engine could not confirm is settled against it, so it is kept while it may still land, records that it replaces a revision without a seal (replaced: Unsealed), says so in its EngineUnavailable message, and is reported once a read shows it runs as a confirmed replacement is, its ConfigApplied and Warning RevisionUnreadable included (reportReplaced). A reload of that write whose answer is lost too replaces nothing of its own (reloadUnconfirmed): its confirmation reports the revision without a seal replaced, once, and never the operator's write as one. The write is now conditioned on the revision the reconcile read, not on the one the dry run names. They differ only when another writer activated a revision in between, and conditioned on the dry run's, the write went over that revision unseen, reported as drift from the one read, with the names the sync kept decided for that one; over an unsealed revision it would have said it replaced that revision. Now the engine refuses the write before storing anything and the reconcile starts over from a fresh read. Seed's override to create-only goes with it: the read found no document wherever Seed writes. With it goes the way TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed reached a write built on a revision no read had found running, the dry run's base, and that test with it. One way stays, and a confirmation still keeps such a write unconfirmed, now saying why: a reconcile that holds its document reads the engine without recording what it finds, so it may activate a running unconfirmed write again for a value it withdrew, and lose that reload's answer too (reloadUnconfirmed). ApplyConfig's doc says what the engine does with If-Match over such a revision: judged against it as against any other, and a write without one replaces it unconditionally. The storeEngine double refuses what reads a revision it cannot open, per revision, as the engine does: a read of the document or of the revision by its id, a reload, and a write (dry runs included) over any but one without a seal; and it logs the requests it answers. Its reload moves to a helper of its own (reload), which keeps serve under the cyclomatic bound. A test that needs the operator to write over r1, which is the rendered document and read every value, rotates a value first. The seat_held doubles, the client's and the storeEngine's, word the detail as #142 does ("that somebody still holds", the seats joined as a list, which the operator does not read), and the storeEngine's carries #142's hint and message beside it. Tests: TestEnforceReplacesARevisionStoredWithoutASeal (both origins: one write, If-Match on the revision, the deleted Secret's value still there when it is written, gone after with the replacement activated again, Applied once the epoch applies, summary and event, nothing written the next time), TestAReplacementOverARevisionWithoutASealKeepsWhatTheNodesRun (the nodes on the revision before it, read by its id, its names kept after the write until they run the replacement), TestAReplacementRefusedOverARevisionWithoutASealKeepsTheStore (a name only the nodes' revision may read kept, the activation owed left owed, no read or reload after the dry run, no SecretWithdrawalFailed, and both settled by the replacement once the document is valid), TestAHeldDocumentOverARevisionWithoutASealKeepsTheStore (and what SecretsSynced and ResolvedRefs say of it), TestAHeldDocumentOverARevisionWithoutASealOwesItsActivation (an activation owed left owed although the pointer names the revision at an epoch above its mark, with no reload asked for and no SecretWithdrawalFailed), TestUnderEnforceAnUnsealedRevisionAfterTheWriteIsLookedAtSoon, TestARevisionTheEngineCannotOpenBlocksTheDocument (the sealed causes under Enforce and Seed, both unsealed origins under Seed and Observe, nothing written and the store untouched), TestAWriteIsConditionedOnTheRevisionTheReconcileRead, TestAReplacementTheEngineCouldNotConfirmIsKept (now its record, its EngineUnavailable message, and its report once it lands), TestASeatHeldOverAReplacementSaysTheDocumentDoesNotKeepIt, TestUnsealedIsTheOneRevisionAWriteReplaces, TestAReloadBuiltOnAnUnconfirmedWriteStaysUnconfirmedOnceTheWriteIs, TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced (two cases more, where the revision the write replaced has no seal), and secretsync's TestKeepListedDeletesNothingTheStoreLists (and Holders naming the revision). Each went red with its guard broken: the replacement removed, allowed under every policy, opened to every unreadable code, deleting before the write, the held document's withdrawal or a recorded reading deleting, a store change settled by it, a reload asked for over the revision by a refused or a held document, the revision called the one the engine runs (or, in the event, ran), the look at the resync interval, the kept value's reason, Holders without the revision, what the nodes run left unread, conditioned on the dry run's base, unconditioned, settled against no revision, the record, message or report of the unconfirmed one, without its message, summary or event, the seat wording, the policy remedies, a write built on the confirmed one dropped or the held reload not recorded, and Unsealed taking unknown_seal_revision or a refusal naming no revision. The troubleshooting guide, the applying concept page and the management policies guide say what replaces such a revision under each policy, what the store keeps until then, and how a replacement the engine could not confirm is reported; the SecretsSynced and ResolvedRefs rows of both condition tables, and the tool servers guide, name the third reason a value is kept beside the other two. The write is conditioned on the engine's own activation of the revision read where the sync made one (the engine activates the revision it runs again at a write or removal of a value it reads), once read back as the same document (syncActivation, split out of reloadedBySync, which rendersAs now completes): conditioned on the read before the sync, every rotation that came with a change of the document was refused as a lost race and written by a second reconcile. TestAWriteAfterTheEnginesActivationAtTheSyncIsConditionedOnIt fails with the condition taken from the read. A rotation whose activation another writer overtook is now refused rather than written over unseen, and written on that writer's revision by the next reconcile (TestARotationAnotherWriteOvertookIsNotWrittenOver, out of the table of TestARotationIsAppliedWhereTheEngineDidNotReactivate). The engine double refuses a write or a removal in the store over a revision it cannot open, as the engine does: it asks which paths of the fleet's revision read each name first (secretsapi, config.Reads), which opens the revision, and fails the request whole. The tests that pin the operator's own activation make the engine's fail (operatorActivates), and TestEnforceReplacesARevisionStoredWithoutASeal expects the replacement activated again by the engine at the removal after it. Signed-off-by: Ali Yousefian --- api/v1alpha1/common_types.go | 15 +- api/v1alpha1/company_types.go | 18 +- config/crd/bases/crewlet.ai_companies.yaml | 26 +- .../templates/crd/companies.crewlet.ai.yaml | 26 +- docs/concepts/applying.md | 134 ++-- docs/guides/management-policies.md | 13 +- docs/guides/tool-servers.md | 8 +- docs/guides/troubleshooting.md | 96 ++- docs/reference/api.md | 5 +- internal/controller/config_move.go | 5 +- internal/controller/config_optional.go | 13 +- internal/controller/config_reload_test.go | 68 +- internal/controller/config_run.go | 310 +++++--- internal/controller/config_unconfirmed.go | 48 +- .../controller/config_unconfirmed_test.go | 134 ++-- internal/controller/config_unreadable_test.go | 664 +++++++++++++++++- internal/controller/config_withdraw.go | 53 +- internal/controller/config_withdraw_test.go | 160 +++-- internal/engine/client.go | 21 +- internal/engine/client_test.go | 58 +- internal/engine/errors.go | 47 +- internal/engine/types.go | 6 +- internal/secretsync/resolve.go | 93 ++- internal/secretsync/sync.go | 6 +- internal/secretsync/sync_test.go | 79 +++ 25 files changed, 1713 insertions(+), 393 deletions(-) diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index c10b1e79..2522e47f 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -657,6 +657,7 @@ const ( // still exists and is held: one whose kind turned from Human to Agent. ReasonSeatHeld = "SeatHeld" + // ReasonRevisionUnreadable is ConfigApplied=False when the engine cannot // ReasonRevisionUnreadable is ConfigApplied=False when the engine cannot // open the fleet's revision of the company document (409 // unreadable_revision or unsealed_revision, or crewlet#142's @@ -664,9 +665,17 @@ const ( // unsealed_fleet_revision): it is sealed under a key the keyring does not // hold, sealed in a form the engine's build does not read, or stored // without a seal. The engine then refuses to show the document and to take - // another, so the operator writes nothing; no wait changes that, and - // somebody has to give the engine what opens the revision, such as the - // key put back in the Secret -keyring, or the build that sealed + // any write built from it, and no wait changes that. A sealed revision is + // replaced by no write, so the operator writes nothing, and somebody has + // to give the engine what opens it: the key put back in the Secret + // -keyring, or the build that sealed it. A revision without a + // seal is replaced by a whole document, which reads nothing from it: under + // Enforce the operator replaces it with the rendered document as it reads + // it, recording a Warning event of this reason, and reports this there + // only for one that lands after that read, until its next look replaces + // it, ten seconds on; while the rendered document is held, the hold's own + // reason is reported and the revision stays. Under Seed and Observe, which + // write no such document, this is reported for as long as the engine runs // it. The message names the revision and carries the engine's account and // remedy. ReasonRevisionUnreadable = "RevisionUnreadable" diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index e6a6e674..267f3c07 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -1638,11 +1638,13 @@ type UnconfirmedRevision struct { // replaced says what the write replaced where that was more than the // operator's own document, which ConfigApplied and a Warning event report // once the engine is seen running the revision, as they report a write the - // engine confirmed: Drifted, a revision the operator did not write whose - // document differed from the rendered one, at driftedAt. Unset for any - // other base, and for a reload: its base is a write of the operator's, - // and where that write is recorded here too, the revision found running - // reports what that write replaced, once, whichever of the two it is. + // engine confirmed: Unsealed, a revision the engine stores without a seal, + // which the write replaces reading nothing from it; Drifted, a revision + // the operator did not write whose document differed from the rendered + // one, at driftedAt. Unset for any other base, and for a reload: its base + // is a write of the operator's, and where that write is recorded here too, + // the revision found running reports what that write replaced, once, + // whichever of the two it is. // +optional Replaced ReplacedRevision `json:"replaced,omitempty"` @@ -1669,10 +1671,14 @@ type UnconfirmedRevision struct { // ReplacedRevision is what a write of the company document replaced where // that was more than the operator's own document. -// +kubebuilder:validation:Enum=Drifted +// +kubebuilder:validation:Enum=Unsealed;Drifted type ReplacedRevision string const ( + // ReplacedUnsealed is a revision the engine stores without a seal, which + // nobody can read and a whole document replaces. + ReplacedUnsealed ReplacedRevision = "Unsealed" + // ReplacedDrifted is a revision the operator did not write whose // document differed from the rendered one. ReplacedDrifted ReplacedRevision = "Drifted" diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index 56e48968..86a906f5 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -1958,12 +1958,15 @@ spec: replaced says what the write replaced where that was more than the operator's own document, which ConfigApplied and a Warning event report once the engine is seen running the revision, as they report a write the - engine confirmed: Drifted, a revision the operator did not write whose - document differed from the rendered one, at driftedAt. Unset for any - other base, and for a reload: its base is a write of the operator's, - and where that write is recorded here too, the revision found running - reports what that write replaced, once, whichever of the two it is. + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. enum: + - Unsealed - Drifted type: string revisionID: @@ -2145,12 +2148,15 @@ spec: replaced says what the write replaced where that was more than the operator's own document, which ConfigApplied and a Warning event report once the engine is seen running the revision, as they report a write the - engine confirmed: Drifted, a revision the operator did not write whose - document differed from the rendered one, at driftedAt. Unset for any - other base, and for a reload: its base is a write of the operator's, - and where that write is recorded here too, the revision found running - reports what that write replaced, once, whichever of the two it is. + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. enum: + - Unsealed - Drifted type: string revisionID: diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index c695dd66..6d984914 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -1961,12 +1961,15 @@ spec: replaced says what the write replaced where that was more than the operator's own document, which ConfigApplied and a Warning event report once the engine is seen running the revision, as they report a write the - engine confirmed: Drifted, a revision the operator did not write whose - document differed from the rendered one, at driftedAt. Unset for any - other base, and for a reload: its base is a write of the operator's, - and where that write is recorded here too, the revision found running - reports what that write replaced, once, whichever of the two it is. + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. enum: + - Unsealed - Drifted type: string revisionID: @@ -2148,12 +2151,15 @@ spec: replaced says what the write replaced where that was more than the operator's own document, which ConfigApplied and a Warning event report once the engine is seen running the revision, as they report a write the - engine confirmed: Drifted, a revision the operator did not write whose - document differed from the rendered one, at driftedAt. Unset for any - other base, and for a reload: its base is a write of the operator's, - and where that write is recorded here too, the revision found running - reports what that write replaced, once, whichever of the two it is. + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. enum: + - Unsealed - Drifted type: string revisionID: diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index 5b2c644b..db8e6f94 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -99,7 +99,10 @@ flowchart TD revision the engine's activation pointer names, which every engine pod answers alike (`GET /health`, then `GET /config`), reads which revisions each engine pod runs and would start on (`GET /fleet`), and lists the - engine's encrypted secret store. Each referenced key is one entry in the store, named as the + engine's encrypted secret store. A revision the engine cannot open ends the + reconcile here (`RevisionUnreadable`), but for one it stores without a + seal, which under `Enforce` the rendered document replaces. Each referenced + key is one entry in the store, named as the document refers to it (`K8S___`). Only values that changed since the last push are written, and any entry the store lost or the engine lists as last written with a credential other than the @@ -194,6 +197,13 @@ flowchart TD document somebody else activated, whose reading the operator does not know, keeps the value it holds for as long as it runs, as the last good revision keeps every value it requires while the next document is held. + A revision the engine stores without a seal keeps every entry the store + holds: nobody can read which values it references, and the engine's nodes + go on running, and boot again when they restart, the revision they + applied before it, which reads the store as it finds it. So nothing is + deleted before the write that replaces it, a value whose Secret no longer + exists included; step 7 deletes what the replacement lets go once it + runs. The Company's `SecretsSynced` and the part's `ResolvedRefs` message say so wherever the reconcile read the Secrets the document references, and a `SecretKept` event, told once, when the reconcile ends with the value @@ -233,7 +243,10 @@ flowchart TD document (step 6), so that once it runs it is the operator's. Nothing makes the engine's reload or its deletion conditional on the revision the operator read, so a revision somebody activates in that moment goes - without a value it may require until the Secret exists again. + without a value it may require until the Secret exists again. Over a + revision the engine stores without a seal, which the engine never + activates again, no activation is asked for: what is owed stays owed + until the write that replaces it activates. 4. **Compare.** The operator compares the engine's active document with the rendered one, and checks that the revision the engine runs activated after every value it wrote into the store. If both hold, there is nothing @@ -269,18 +282,24 @@ flowchart TD the Company does not offer. Each problem the engine reports is traced back, by its place in the document, to the part that rendered that place. 6. **Apply** the document as one revision, on the condition that the engine - still runs the revision the dry run checked against (`If-Match`; into an - engine with no document yet, on the condition that it still has none). - The revision's summary is `crewlet-operator:` followed by the objects - whose part of the document changed. A write that loses a race with - another writer is started over once from a fresh read. When the engine + still runs the revision step 3 read (`If-Match`, a revision stored without + a seal included; into an engine with no document yet, on the condition + that it still has none), the one everything before was decided against: + a revision another writer activated since, which the dry run may have been + checked against, is never written over unseen. The revision's summary is + `crewlet-operator:` followed by the objects whose part of the document + changed, and, over a revision stored without a seal, that it replaced it. + A write that loses a race with another writer is started over once from a + fresh read. When the engine stores the revision and then can't confirm that it activated it (a `503` naming the revision it stored), the fleet may run it or may not, so the operator records the write in `status.config.unconfirmed` and writes nothing more until it has read which revision the engine runs: that one, and it is recorded and reported as the operator's write, as though the - engine had confirmed it, a revision somebody else wrote that it replaced - included (`DriftCorrected`, with the `Drifted` event recorded then), at + engine had confirmed it, what it replaced included: a revision somebody + else wrote (`DriftCorrected`, with the `Drifted` event recorded then) or + one the engine stores without a seal (`Applied` naming it, with the + `RevisionUnreadable` event recorded then); at the epoch the fleet's activation names and as applied when the operator found it, since nothing says when its activation landed; the one the write was built on, and the write is sent again on the same condition, @@ -363,7 +382,7 @@ sequenceDiagram O->>E: DELETE /secrets/NAME that no document references, or of a deleted Secret every revision a pod runs or would start on goes without O->>E: PUT /config?dry_run=true with the rendered document E-->>O: the revision it checked against, or each problem at its place - O->>E: PUT /config, If-Match the revision it checked + O->>E: PUT /config, If-Match the revision it read E-->>O: the new revision and its activation epoch Note over O: records the new revision and the keys it lets go without O->>E: GET /health, then GET /config, the fleet's document now @@ -437,11 +456,14 @@ operator tries again every ten seconds, or sooner where the engine's `Retry-After` says so. Because the address is the cluster's DNS name of the pod, an operator running outside the cluster (`make run`) cannot reach any engine. When the engine answers but -cannot open the fleet's revision, sealed under a key its keyring lacks, -sealed in a form its build does not read, or stored without a seal, the -Company reports `RevisionUnreadable` instead, which no retry clears, and the -operator looks again when the data pods restart and every five minutes besides +cannot open the fleet's revision, sealed under a key its keyring lacks or in a +form its build does not read, the Company reports `RevisionUnreadable` +instead, which no retry clears, and the operator looks again when the data +pods restart and every five minutes besides ([The engine cannot open its company document](../guides/troubleshooting.md#the-engine-cannot-open-its-company-document)). +A revision the engine stores without a seal is the exception: a whole +document replaces it, and under `Enforce` the operator writes the rendered one +over it (below). ### What the engine has read @@ -533,6 +555,22 @@ old. A fleet whose activation names another revision by then had another write activate one between the two reads, and the operator records nothing and starts over once from a fresh read, as it does a write that lost its race. +A revision the engine stores without a seal, which it shows to nobody and +neither applies nor activates again, is not compared, since nobody can read +it: the engine's remedy for one is a whole document, which reads nothing from +it, and the operator writes the rendered document over it, conditioned on that +revision. It reports `ConfigApplied=True` with the reason `Applied` naming the +revision it replaced, records a Warning `RevisionUnreadable` event saying so, +and summarises the new revision as one that replaced it, the same once a read +shows the engine runs a replacement it could not confirm activating. Nothing +of the old revision is read, so nothing in the secret store is deleted before +the write: the engine's nodes run, and boot again, the revision they applied +before it, which may read any value the store holds, and the sync after the +write removes what the replacement lets go. A revision sealed under a key the +keyring lacks, or in a form the engine's build does not read, is never written +over: it is somebody's document, and the Company reports `RevisionUnreadable` +until the engine can open it. + The comparison is over what the engine serves back, which is its own encoding of the document rather than the bytes it was given. It disregards the engine's shorter spelling of a seat's model chain, and the defaults the @@ -553,6 +591,7 @@ secret store (`SecretsSynced` is `Unknown` with the reason `NotManaged`). | the rendered document | `True`, `InSync` | `True`, `InRevision` | | another one | `False`, `Drifted`, naming the places that differ | `False`, `Drifted`, on each part whose place differs; `True` on the others | | no document | `False`, `Drifted` | `False`, `Drifted` | +| a revision it cannot open | `False`, `RevisionUnreadable`, naming it | `False`, `Blocked`, but where the part's current generation already runs | A `Drifted` event is recorded when the difference first appears or changes. Observe is how to hand the objects a company the engine already runs (one @@ -570,7 +609,13 @@ values it references, and from then on leaves the engine alone: the Company reports `ConfigApplied=True` with the reason `Seeded`, and every part `Applied=False` with the reason `NotManaged`, since a change to it is no longer applied. An engine that already had a document when the Company was created -is never written to; the Company reports `Seeded` all the same, saying so. +is never written to; the Company reports `Seeded` all the same, saying so. One +whose fleet's revision it cannot open is not written to either, a revision it +stores without a seal included: the Company reports `RevisionUnreadable` +with the remedy for its cause (the key put back into the keyring, or the +engine build that sealed the revision run), and for a revision stored +without a seal, that an admin of the company replaces it with a whole +document, or that `Enforce` replaces it with the rendered one. --- @@ -652,12 +697,12 @@ takes no work for a person that a pause could hold. | Object | Condition | When | |---|---|---| -| Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know (it names those, and which of the two it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | -| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine cannot open the fleet's revision, so it neither shows the document nor takes another) or `EngineUnavailable`. | +| Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know, or for the engine's active revision where that is one stored without a seal, which nobody can read and its nodes do not run (it names those, and which of the three it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | +| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine cannot open the fleet's revision, so it neither shows the document nor takes a write built from it; a revision it stores without a seal blocks under `Seed` and `Observe`; under `Enforce` the operator replaces it, and reports it only for one that lands after the operator's read, until the next look ten seconds on replaces it) or `EngineUnavailable`. | | Company | `EngineHealthy` | The engine's alarm table, read each time the engine is reached: `True`, `NoAlarms`; `False`, `Alarms`, naming the one standing longest; `Unknown`, `AlarmsNotReported`, before the engine has evaluated them. Not part of `Ready`. | | Company | `IdentityClaimed` | Whether anybody has joined the company, read from the engine's `/health` each time the engine is reached, and every 30 seconds while nobody has, since a person joins where no watch sees it: `True`, `PersonEnrolled`, once its identity directory holds a person; `False`, `Unclaimed`, while nobody has joined, naming up to five of the company's Human seats and the access token to bring the first person in with; `False`, `NoHumanSeat`, when nobody can, for want of a Human seat; `Unknown`, `IdentityNotReported`, when the engine could not read its directory, did not say, or could not be asked. Not part of `Ready`: a company nobody has joined runs its agent seats. | | every part | `Accepted` | `True` once its Company exists; `False` with `CompanyNotFound`, or `Conflict` when it collides with another object of its company. | -| every part | `ResolvedRefs` | `True`, `Resolved`, its message naming any value it gives a tool server or a seat's executor run that the engine starts with and goes without, under a key an `optionalKeys` list lets go (the message names it: the MCPServer's for a tool server's value, a Seat's or Unit's included), whose Secret or key does not exist or holds an empty value, also while a Secret another part requires holds the document, and saying where the engine keeps the value it held for the revision it runs, which requires it, or whose reading the operator does not know. `False` with `RefNotFound`, `Cycle`, `HumanSeat`, `ReferenceNotAllowed`, `SecretNotFound`, `SecretTooLarge`, `OptionalKeyUnused`, `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty` or `NotAnAccountID`, saying what it references that is missing or cannot be there. Most hold the document, and the part reports the same reason on `Applied`; `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty`, `NotAnAccountID`, and `HumanSeat` on a GitLab `accessLevels` entry hold nothing, because only the part at fault suffers them: it is applied (`Applied=True`) and reported, with a Warning event when the condition turns so. | +| every part | `ResolvedRefs` | `True`, `Resolved`, its message naming any value it gives a tool server or a seat's executor run that the engine starts with and goes without, under a key an `optionalKeys` list lets go (the message names it: the MCPServer's for a tool server's value, a Seat's or Unit's included), whose Secret or key does not exist or holds an empty value, also while a Secret another part requires holds the document, and saying where the engine keeps the value it held for the revision it runs, which requires it, or whose reading the operator does not know, or for its active revision, where that is one stored without a seal, which nobody can read and its nodes do not run. `False` with `RefNotFound`, `Cycle`, `HumanSeat`, `ReferenceNotAllowed`, `SecretNotFound`, `SecretTooLarge`, `OptionalKeyUnused`, `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty` or `NotAnAccountID`, saying what it references that is missing or cannot be there. Most hold the document, and the part reports the same reason on `Applied`; `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty`, `NotAnAccountID`, and `HumanSeat` on a GitLab `accessLevels` entry hold nothing, because only the part at fault suffers them: it is applied (`Applied=True`) and reported, with a Warning event when the condition turns so. | | every part | `Applied` | `True`, `InRevision`, naming the revision, with the generation that revision was rendered from as its `observedGeneration`, once the engine has applied it. `False` with the part's own problem (`Invalid`, `RefNotFound`, a Seat's `PerSeatNotAllowed`, `PoolCapacityExceeded`, `CLIAuthNotPortable`, `NoExecutorCLI`, or `SeatHeld` when it turns a held human seat into an agent's, an LLMProvider's `SandboxPlacementNotOffered`, ...), a Seat's `PlacementPending` while it stays where it runs until a pod of the pool it moves to is ready (or for good, while the Company does not declare that pool), `Blocked` when another object keeps the document from being applied, `Applying` while the engine applies the revision, or `ApplyFailed` when it did not. A part whose current generation already runs keeps `True` meanwhile: only what changed is held back. | | Seat (agent) | `Ready` | Whether the seat runs: see [A Seat's `Ready` condition](#a-seats-ready-condition). A Human seat has none. | | Seat (Agent) | `Paused` | Whether the engine holds the seat paused, once it runs the seat: `True`, `SeatPaused`, naming who paused it, when and why; `False`, `SeatRunning`; with `PauseNotManaged` instead when the pause differs from `spec.paused` under `Observe` or `Seed`. `Unknown`, `PauseUnknown`, while a pause or resume the engine may or may not have recorded is retried; `Unknown`, `PauseRefused`, with the engine's words when it refused one, which is asked for again ten seconds later. None while the engine runs no seat of the Seat's (not in its revision, or a handle another Seat derives too). | @@ -672,27 +717,28 @@ Besides the conditions: `status.config.unconfirmed` lists the operator's writes the engine stored and couldn't confirm activating, at most 16, the oldest dropped first, until a read of the revision the engine runs settles them (step 6 above), each - with what it replaced where that was somebody else's revision - (`replaced: Drifted`, and `driftedAt`, where it differed), for the report - made once the engine is seen running it. `status.config.overtaken` lists the - ones written into an engine that had no document that another revision - overtook, which can no longer land, and of which `Seed` takes one for the - seed, at most 16, until a reconcile records a revision of the rendered - document in sync, a write of it, or the seed. `status.config.placement` is - where the document the engine runs pins its agent seats: every value of - `crewlet.ai/pool` (`data` and the Shared pools that run a seat) and of - `crewlet.ai/seat` (each seat of a PerSeat pool). It is read from the - engine's own document under every policy, so under Observe and Seed too. A - new revision's pins join the record before the operator's write that - activates it, and while the engine applies it the record is the previous - revision's pins and the new one's together, because the engine keeps running - the previous revision until it has applied the new one, and for good if that - apply fails. Once the engine reports the new revision's epoch applied, the - record is its pins alone; a document the engine owns (Observe, Seed) is - narrowed to only once the engine's own record of its last apply names that - revision and says it ended well. The runtime reconciler reads it to keep a - seat pool's pods running for as long as the running revision pins a seat to - them ([A pool removed while seats still run in + with what it replaced where that was more than the operator's own document + (`replaced: Unsealed` for a revision the engine stores without a seal; + `replaced: Drifted`, and `driftedAt`, where it differed, for somebody + else's), for the report made once the engine is seen running it. + `status.config.overtaken` lists the ones written into an engine that had no + document that another revision overtook, which can no longer land, and of + which `Seed` takes one for the seed, at most 16, until a reconcile records a + revision of the rendered document in sync, a write of it, or the seed. + `status.config.placement` is where the document the engine runs pins its + agent seats: every value of `crewlet.ai/pool` (`data` and the Shared pools + that run a seat) and of `crewlet.ai/seat` (each seat of a PerSeat pool). It + is read from the engine's own document under every policy, so under Observe + and Seed too. A new revision's pins join the record before the operator's + write that activates it, and while the engine applies it the record is the + previous revision's pins and the new one's together, because the engine + keeps running the previous revision until it has applied the new one, and + for good if that apply fails. Once the engine reports the new revision's + epoch applied, the record is its pins alone; a document the engine owns + (Observe, Seed) is narrowed to only once the engine's own record of its last + apply names that revision and says it ended well. The runtime reconciler + reads it to keep a seat pool's pods running for as long as the running + revision pins a seat to them ([A pool removed while seats still run in it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). - **Seat**: its handle, its identity in the engine (`seatID`, derived from the company name and the handle), the pool that runs it and the @@ -716,11 +762,17 @@ pause, which is recorded on the Seat (`SeatPaused`, `SeatResumed`, and with the revision, its epoch and its summary (for a write the engine couldn't confirm, with the revision and its epoch once a read shows the engine runs it), `Applied` once the engine has -applied it, `ApplyFailed` when it did not, `SecretsSynced` with the store +applied it, `ApplyFailed` when it did not, a Warning `RevisionUnreadable` +when a document is held because the engine cannot open the fleet's revision, +or when the rendered document replaced a revision the engine stores without a +seal (for a write the engine couldn't confirm, once a read shows it runs), +`SecretsSynced` with the store entries it wrote and deleted, a Warning `SecretOverwritten` naming each entry it wrote back and who had replaced it, `SecretKept` naming each value of a deleted Secret it keeps in the store for the revision the engine runs, which -requires it or whose reading the operator does not know, `Reactivated` when +requires it or whose reading the operator does not know, or for the engine's +active revision where that is one stored without a seal, which nobody can +read, `Reactivated` when it activated the running revision again so the engine reads its store without the value of a deleted Secret, or with a value it wrote while no write of the document was coming, a Warning `Reactivated` when the revision diff --git a/docs/guides/management-policies.md b/docs/guides/management-policies.md index f77c8301..43fff532 100644 --- a/docs/guides/management-policies.md +++ b/docs/guides/management-policies.md @@ -53,6 +53,14 @@ it stays in the engine's revision history. kubectl get events -n acme --field-selector involvedObject.name=acme,reason=Drifted ``` +A revision the engine stores without a seal, which no engine reads, is +replaced the same way, without a comparison: nobody can read it, and a whole +document is what replaces it. The operator reports `ConfigApplied=True Applied` +naming the revision it replaced, with a Warning `RevisionUnreadable` event. A +revision the engine cannot open because it is sealed, under a key its keyring +lacks or in a form its build does not read, is never written over; see +[The engine cannot open its company document](troubleshooting.md#the-engine-cannot-open-its-company-document). + ## Observe The operator renders the document, has the engine dry-run it, and compares @@ -101,7 +109,10 @@ the Secret values it references, and from then on leaves the engine alone. The Company reports `ConfigApplied=True Seeded`; every object reports `Applied=False NotManaged`, because a change to it is no longer applied, and `SecretsSynced` is `Unknown`. An engine that already had a document is never -written to. +written to, even one it cannot open: a revision it stores without a seal is +reported as `ConfigApplied=False RevisionUnreadable`, for an admin of the +company to replace with a whole document, or for `Enforce` to replace with the +rendered one. > **Under Seed, rotate in the engine.** Rotating a Secret is not applied under > Seed either: rotate the credential through the engine's own secret store diff --git a/docs/guides/tool-servers.md b/docs/guides/tool-servers.md index ac116681..7ffc1476 100644 --- a/docs/guides/tool-servers.md +++ b/docs/guides/tool-servers.md @@ -134,9 +134,11 @@ operator's under the `Enforce` policy, under which it keeps the store in step with the Secrets ([Management policies](../concepts/applying.md#management-policies)), and it holds where the revision the engine runs goes without the value too. A revision that reads the key as required, as the one from before you listed -the key does, or another document somebody else activated, whose reading the -operator does not know, keeps the value it holds until the next document -runs, as the last good revision keeps every value it requires; the Company's +the key does, another document somebody else activated, whose reading the +operator does not know, or a revision the engine stores without a seal, which +nobody can read while its nodes run the one before it, keeps the value it +holds until the next document runs, as the last good revision keeps every +value it requires; the Company's `SecretsSynced`, the object's `ResolvedRefs` message and a `SecretKept` event say so (a document the render holds is told by the event alone). Once no engine pod runs, or would start on, a revision that requires it, the operator diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 620f120a..429c9734 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -87,7 +87,7 @@ with your last change yet. | | False | `GatewayAPINotInstalled` | The class routes through Gateways and the cluster had no Gateway API when the operator started. Install it and restart the operator. | | | False | `PublicBaseURLMissing` | The class's webhooks mode is `External` and the Company has no `spec.exposure.publicBaseURL`. | | | False | `RouteNotHTTPS` | No Gateway listener the artifacts route attaches to is HTTPS, so the data pod opens no artifact origin; add one with a certificate for the route's host. | -| `SecretsSynced` | True | `Synced` | The engine's secret store holds every referenced value. The message counts the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which the store does not hold, so the engine leaves each out ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)); names, in a sentence of its own, those the store keeps as they were for the revision the engine runs, because that revision reads them as required or is one whose reading the operator does not know (another document somebody else activated), each to go once a revision that lets it go runs; names, in another, those it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions the operator does not know, with the pods, each to go once no pod runs or would start on a revision that references it ([The engine did not apply a revision](#the-engine-did-not-apply-a-revision)); and, the Company having no `ResolvedRefs`, names the values of its own setup steps a run goes without, under a key a step's `optionalKeys` lists, and a value of theirs that holds nothing under a key none lists, which every executor run starts without. | +| `SecretsSynced` | True | `Synced` | The engine's secret store holds every referenced value. The message counts the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which the store does not hold, so the engine leaves each out ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)); names, in a sentence of its own, those the store keeps as they were for the revision the engine runs, because that revision reads them as required or is one whose reading the operator does not know (another document somebody else activated), or for the engine's active revision, where that is one stored without a seal, which nobody can read while its nodes run the revision before it, each to go once a revision that lets it go runs; names, in another, those it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions the operator does not know, with the pods, each to go once no pod runs or would start on a revision that references it ([The engine did not apply a revision](#the-engine-did-not-apply-a-revision)); and, the Company having no `ResolvedRefs`, names the values of its own setup steps a run goes without, under a key a step's `optionalKeys` lists, and a value of theirs that holds nothing under a key none lists, which every executor run starts without. | | | False | `SecretNotFound` | A referenced Secret or key is missing; the message names each. | | | False | `SecretTooLarge` | A value is over the 64 KiB the store accepts. | | | False | `EngineUnavailable` | Writing to the store failed; the message says why. Retried. | @@ -124,7 +124,7 @@ with your last change yet. | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | -| | False | `RevisionUnreadable` | The engine cannot open the fleet's revision of the company document (`409 unreadable_revision`, `unreadable_fleet_revision`, `unknown_seal_revision`, `unsealed_revision` or `unsealed_fleet_revision`), so the engine neither shows the document nor takes another. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | +| | False | `RevisionUnreadable` | The engine cannot open the fleet's revision of the company document (`409 unreadable_revision`, `unreadable_fleet_revision`, `unknown_seal_revision`, `unsealed_revision` or `unsealed_fleet_revision`), so the engine shows the document to nobody and takes no write built from it. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. A revision stored without a seal is reported under `Seed` and `Observe`; under `Enforce` the operator replaces it with the rendered document, and reports it only for one that lands after the operator's read, until the next look ten seconds on replaces it. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | ### EngineClass @@ -148,7 +148,7 @@ conditions. `kubectl get engineclasses` shows it. | | False | `CompanyNotFound` | `spec.companyRef` names no Company in the namespace. | | | False | `CompanyNotAccepted` | Its Company is not `Accepted` (its class is gone, or it is over its class's limits); the message carries why. Nothing of the company is applied until it is. | | | False | `Conflict` | Crewlet would read it and another object as one: two Seats with one handle, email address or contact account; two Integrations for one vendor; an LLMProvider named like a platform provider of the class; a Seat's header for an HTTP tool server spelled unlike its Unit spells it (`authorization` beside `Authorization`), which Crewlet would send either of. The message names the other object. | -| `ResolvedRefs` | True | `Resolved` | Every object and Secret key it references exists; or the message names the values it gives a tool server or a seat's executor run that it goes without, whose Secret or key does not exist or holds an empty value, under a key an `optionalKeys` list lets the engine leave out, and that list: the MCPServer's for a value of a tool server's, a Seat's or Unit's included ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). A line that adds that the engine keeps the value it held for the revision it runs names a deleted Secret's value that revision reads as required, or whose reading the operator does not know; it goes once a revision that lets it go runs. It says so while a Secret another object requires holds the document too; while another object keeps the document from rendering, the Secrets are not read, and the message says only that every object it references exists. | +| `ResolvedRefs` | True | `Resolved` | Every object and Secret key it references exists; or the message names the values it gives a tool server or a seat's executor run that it goes without, whose Secret or key does not exist or holds an empty value, under a key an `optionalKeys` list lets the engine leave out, and that list: the MCPServer's for a value of a tool server's, a Seat's or Unit's included ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). A line that adds that the engine keeps the value it held for the revision it runs names a deleted Secret's value that revision reads as required, or whose reading the operator does not know; one that says it keeps it for its active revision names one kept for a revision stored without a seal, which nobody can read while the nodes run the revision before it. Either goes once a revision that lets it go runs. It says so while a Secret another object requires holds the document too; while another object keeps the document from rendering, the Secrets are not read, and the message says only that every object it references exists. | | | False | `RefNotFound` | It references an object that does not exist (a Unit, a Seat, an LLMProvider, an MCPServer); the message names it. | | | False | `Cycle` | A Unit's `parentRef` chain leads back to itself. | | | False | `PoolNotFound` | A Seat's `spec.placement.pool` is neither `data` nor one of its Company's `spec.topology.seatPools`. The seat is applied all the same, pinned to the label `crewlet.ai/undeclared-pool`, which no pod carries: it runs nowhere (`Ready=False`, `PoolNotFound`), or keeps running where it ran (`Applied=False`, `PlacementPending`), until the Company declares the pool or the Seat names another. Nothing else is held. | @@ -387,23 +387,26 @@ requests to `http://-data-0.-data..svc:8080` fail: activated it: the message names the revision. The engine may run it or may not. Nothing needs doing: the operator keeps the write in `status.config.unconfirmed`, reads which revision the engine runs before it - writes again, and records the write as its own if the engine runs it, with - a `Normal` event `Activated` and, where it replaced a revision somebody else - wrote, `ConfigApplied=True DriftCorrected` and a Warning `Drifted`, as for - a write the engine confirmed, dated when it found it running. It never - writes the document a second time on top of one that landed, nor on top of - one that landed and was activated again before it looked (a reload): that - revision activated after every value the write was made with, so where it - is the rendered document the engine runs it with them, and it is in sync. - Under `Seed` a write into an engine that had no document is kept in - `status.config.overtaken` once another revision runs, and reported as the - seed, whatever revision runs now. A request to activate the operator's own - revision again, for the value of a deleted Secret or one written since, - that the engine stored and couldn't confirm is kept the same way, whatever - the operator could read back after it, beside the `SecretWithdrawalFailed` - or `SecretRefreshFailed` Warning, and recorded as the operator's once it - runs that revision's document; one found running another document reloaded - a revision somebody activated before the request landed, and is not the + writes again, and records the write as its own if the engine runs it, with a + `Normal` event `Activated` and, where it replaced a revision somebody else + wrote, `ConfigApplied=True DriftCorrected` and a Warning `Drifted`, or one + the engine stores without a seal, `ConfigApplied=True Applied` naming it and + a Warning `RevisionUnreadable`, as for a write the engine confirmed, dated + when it found it running. The message of the write it could not confirm + names such a revision already. It never writes the document a second time + on top of one that landed, nor on top of one that landed and was activated + again before it looked (a reload): that revision activated after every + value the write was made with, so where it is the rendered document the + engine runs it with them, and it is in sync. Under `Seed` a write into an + engine that had no document is kept in `status.config.overtaken` once + another revision runs, and reported as the seed, whatever revision runs + now. A request to activate the operator's own revision again, for the + value of a deleted Secret or one written since, that the engine stored and + couldn't confirm is kept the same way, whatever the operator could read + back after it, beside the `SecretWithdrawalFailed` or + `SecretRefreshFailed` Warning, and recorded as the operator's once it runs + that revision's document; one found running another document reloaded a + revision somebody activated before the request landed, and is not the operator's. ## Nobody can sign in @@ -484,14 +487,16 @@ something a person set up. Or keep the seat: restore its Seat as a Human seat. ## The engine cannot open its company document `ConfigApplied=False RevisionUnreadable` means the engine cannot open the -fleet's revision of the company document. The engine refuses every -read of the document with `409`, and every write too, since a write reads the -revision it replaces; it keeps running what it applied, and shows and takes -no document until it can open that revision. No wait changes that, and -nothing the operator sends does, so the operator writes nothing and does not -ask again every ten seconds: it looks again when the data pods restart, and -every five minutes besides. The message names the revision, quotes the -engine's account and its remedy, and says which of five it is: +fleet's revision of the company document. The engine refuses every read of the +document with `409`, and every write built from it too; it keeps running what +it applied, and shows no document until it can open that revision. No wait +changes that. A revision that is sealed, under a key the keyring lacks or in a +form the engine's build does not read, is somebody's document, which no write +replaces, since that would discard it unread: nothing the operator sends opens +it, so the operator writes nothing and does not ask again every ten seconds. +It looks again when the data pods restart, and every five minutes besides. The +message names the revision, quotes the engine's account and its remedy, and +says which of five it is: - `unreadable_revision`: the revision is sealed under a key the data pod's keyring does not hold. The keyring is the Secret `-keyring`, which @@ -510,20 +515,43 @@ engine's account and its remedy, and says which of five it is: - `unknown_seal_revision`: the revision, the data pod's own copy or the one the fleet's activation pointer carries, is sealed in a form the engine's build does not read, most likely by a newer build of Crewlet than the data - pod runs. The keyring is not at fault, putting a key back changes nothing, - and no write replaces it, since that would discard a document somebody - sealed: run the data pods on the build that sealed it, the engine image + pod runs. The keyring is not at fault, and putting a key back changes + nothing: run the data pods on the build that sealed it, the engine image their EngineClass names (with the Company's `spec.runtime.version` as its tag where the class allows that). A data pod still rolling onto that image takes it when it restarts; one taken back to an older image than it ran needs the newer one again. The engine's hint says which. - `unsealed_revision`: the revision is stored without a seal, which no keyring opens and no engine reads; only an engine older than the keyring every - engine now requires stored one. The engine's hint says what brings its - document back. + engine now requires stored one. - `unsealed_fleet_revision`: the revision on the fleet's activation pointer carries no seal. Every engine seals what it activates, so it was put there - from outside the fleet; the engine's hint says what to do. + from outside the fleet. + +A revision without a seal is the one the engine lets a write replace: a whole +document (`PUT /config`), which reads nothing from it, as the engine's hint +says. Under `Enforce` the operator does exactly that, as soon as nothing holds +the rendered document: it writes the rendered document over the revision, +conditioned on it, and reports `ConfigApplied=True Applied` naming the +revision it replaced, with a Warning `RevisionUnreadable` event saying so. A +document held for a missing Secret, refused by the engine or by the render +keeps the revision in place, and `ConfigApplied` carries the hold's own +reason. Nothing is read from the revision, so nothing in the engine's secret +store is deleted before the write: the engine's nodes go on running, and boot +again when they restart, the revision they applied before it, which may read +any value the store holds; the sync after the write removes what the +replacement lets go. So under `Enforce`, `RevisionUnreadable` reports a +revision without a seal only where one lands after the operator's read, until +its next look ten seconds on replaces it. Under `Seed` and `Observe`, which +write no document over the fleet's revision, it stays reported: an admin of +the company replaces it with a whole document, or the Company's +`managementPolicy` is set to `Enforce`, under which the operator replaces it +with the rendered one. Should the rendered document drop a human seat somebody +holds in the engine, the replacement is refused as any write is +([`SeatHeld`](#the-engine-refuses-a-change-to-the-document)), naming every +seat the document does not keep as a human seat that a person, a service +account or an open invitation holds, since nobody can read which seats the +revision it replaces had. To see which keys the keyring holds without printing them: diff --git a/docs/reference/api.md b/docs/reference/api.md index 0b7d6993..77458032 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -2987,13 +2987,14 @@ ReplacedRevision is what a write of the company document replaced where that was more than the operator's own document. _Validation:_ -- Enum: [Drifted] +- Enum: [Unsealed Drifted] _Appears in:_ - [UnconfirmedRevision](#unconfirmedrevision) | Field | Description | | --- | --- | +| `Unsealed` | ReplacedUnsealed is a revision the engine stores without a seal, which
nobody can read and a whole document replaces.
| | `Drifted` | ReplacedDrifted is a revision the operator did not write whose
document differed from the rendered one.
| @@ -4186,7 +4187,7 @@ _Appears in:_ | `appliedHash` _string_ | appliedHash is the appliedHash of the write: the rendered document's
and its values' hash as they were when it was written, which
status.config.appliedHash records once the engine is seen running the
revision. | | MaxLength: 128
MinLength: 1
Required: \{\}
| | `writtenAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | writtenAt is when the operator wrote it. A write the engine is seen
running is recorded as applied when the operator found it, since
nothing says when its activation landed; a seed taken from overtaken,
which nothing waits on, as applied when it was written. | | Required: \{\}
| | `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from, which
status.config.objects records once the engine is seen running it. | | MinProperties: 1
Optional: \{\}
| -| `replaced` _[ReplacedRevision](#replacedrevision)_ | replaced says what the write replaced where that was more than the
operator's own document, which ConfigApplied and a Warning event report
once the engine is seen running the revision, as they report a write the
engine confirmed: Drifted, a revision the operator did not write whose
document differed from the rendered one, at driftedAt. Unset for any
other base, and for a reload: its base is a write of the operator's,
and where that write is recorded here too, the revision found running
reports what that write replaced, once, whichever of the two it is. | | Enum: [Drifted]
Optional: \{\}
| +| `replaced` _[ReplacedRevision](#replacedrevision)_ | replaced says what the write replaced where that was more than the
operator's own document, which ConfigApplied and a Warning event report
once the engine is seen running the revision, as they report a write the
engine confirmed: Unsealed, a revision the engine stores without a seal,
which the write replaces reading nothing from it; Drifted, a revision
the operator did not write whose document differed from the rendered
one, at driftedAt. Unset for any other base, and for a reload: its base
is a write of the operator's, and where that write is recorded here too,
the revision found running reports what that write replaced, once,
whichever of the two it is. | | Enum: [Unsealed Drifted]
Optional: \{\}
| | `driftedAt` _string_ | driftedAt names where the Drifted revision the write replaced differed
from the rendered document, as ConfigApplied names it: up to five
places, and how many more there were. | | MaxLength: 32768
MinLength: 1
Optional: \{\}
| | `documentSHA256` _string_ | documentSHA256 is set for a reload: the SHA-256, in hex, of the
document of the revision it activates again, as the engine answered
it. The engine activates again whichever revision is active when the
request lands, so the revision it stored is the operator's only where
its document is that one; one found running another document was a
reload of a revision somebody activated in between, and goes. | | MaxLength: 64
MinLength: 64
Pattern: `^[0-9a-f]\{64\}$`
Optional: \{\}
| diff --git a/internal/controller/config_move.go b/internal/controller/config_move.go index 59a8546f..a0e39a2b 100644 --- a/internal/controller/config_move.go +++ b/internal/controller/config_move.go @@ -42,7 +42,10 @@ const placementHoldInterval = 5 * time.Second // holdMoves keeps each agent seat that moves to a pool with no ready pod // pinned where the engine runs it, re-rendering the document with those pins // when there is any. active is the engine's document, nil when it has none: -// a seat the engine does not run yet has nowhere to stay. +// a seat the engine does not run yet has nowhere to stay. It is nil too where +// the engine runs a revision stored without a seal, which the write replaces +// (configRun.unsealed): where that revision pins a seat is what nobody can +// read, and the engine shows no other revision in its place. func (run *configRun) holdMoves(ctx context.Context, active *engine.ActiveConfig) error { if active == nil { return nil diff --git a/internal/controller/config_optional.go b/internal/controller/config_optional.go index 33701eed..fa72e64f 100644 --- a/internal/controller/config_optional.go +++ b/internal/controller/config_optional.go @@ -192,7 +192,8 @@ type leftOutValue struct { // leftOutLines are the lines of values, each saying too where the engine's // store keeps the value it held for a revision an engine pod runs or starts -// on (run.retained). +// on, or for the engine's active revision where it stores that one without a +// seal (run.retained). func (run *configRun) leftOutLines(values []leftOutValue) []string { out := make([]string, len(values)) for i, v := range values { @@ -200,8 +201,14 @@ func (run *configRun) leftOutLines(values []leftOutValue) []string { switch { case !slices.Contains(run.retained, v.key): case run.keptPlainly(): - out[i] += "; the engine keeps the value it held for the revision it runs, which " + - strings.TrimPrefix(run.keptBecause(false, "the key"), "the revision the engine runs ") + + held, subject := "the revision it runs", "the revision the engine runs " + if run.keptFor[0].unsealed { + // The engine's active revision, one stored without a seal, + // which its nodes do not run. + held, subject = "its active revision", "the engine's active revision " + } + out[i] += "; the engine keeps the value it held for " + held + ", which " + + strings.TrimPrefix(run.keptBecause(false, "the key"), subject) + ", until a revision that lets it go runs" default: out[i] += "; the engine keeps the value it held, as " + run.keptBecause(false, "the key") + diff --git a/internal/controller/config_reload_test.go b/internal/controller/config_reload_test.go index 96f168d6..3a3e4235 100644 --- a/internal/controller/config_reload_test.go +++ b/internal/controller/config_reload_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "slices" "strings" "testing" @@ -9,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crewlet/crewlet-operator/internal/engine" "github.com/crewlet/crewlet-operator/internal/secretref" "github.com/crewlet/crewlet-operator/internal/secretsync" ) @@ -92,33 +94,25 @@ func TestARotationIsTheEnginesOneActivation(t *testing.T) { } // TestARotationIsAppliedWhereTheEngineDidNotReactivate: an engine whose -// write re-activated nothing, or failed to, or whose re-activation somebody -// else's write overtook before the operator read it back, leaves the apply -// to the operator, which writes the rendered document itself. +// write re-activated nothing, or failed to, leaves the apply to the +// operator, which writes the rendered document itself. func TestARotationIsAppliedWhereTheEngineDidNotReactivate(t *testing.T) { for _, tc := range []struct { name string // reload is what the engine answers of its re-activation instead // of making one; nil for one it makes. reload *map[string]any - // overtaken is whether somebody else activates the rendered - // document again right after the engine's re-activation. - overtaken bool - want []string + want []string }{ {name: "nothing read the value", reload: new(map[string]any(nil)), want: []string{"r1", "r2"}}, {name: "the re-activation lost a race", reload: failedReload("revision_advanced", "another configuration write activated first"), want: []string{"r1", "r2"}}, {name: "the re-activation could not be confirmed", reload: failedReload("unavailable", "the coordination store did not answer"), want: []string{"r1", "r2"}}, - {name: "somebody else's write overtook it", overtaken: true, want: []string{"r1", "r2", "r3", "r4"}}, } { t.Run(tc.name, func(t *testing.T) { w := newRotation(t) w.engine.reloadAnswer = tc.reload - if tc.overtaken { - w.engine.afterSecretsReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } - } if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -145,3 +139,55 @@ func failedReload(code, detail string) *map[string]any { failed := refusal(code, detail) return &failed } + +// TestARotationAnotherWriteOvertookIsNotWrittenOver: somebody else activates +// the rendered document again right after the engine's re-activation at the +// rotation, before the operator reads it back. The operator's write is +// conditioned on the revision its reconcile read, so the engine refuses it +// rather than store it over theirs unseen, and the next reconcile, from a +// fresh read, writes the document on their revision: it cannot tell their +// activation came after the rotated value, since it read the fleet after +// both. +func TestARotationAnotherWriteOvertookIsNotWrittenOver(t *testing.T) { + w := newRotation(t) + w.engine.afterSecretsReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + if err := w.enforce(t); !errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("enforce() = %v, want the write refused as a lost race", err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3"}) { + t.Fatalf("the revisions are %v, want r1, the engine's r2 and the other writer's r3: nothing written over r3", got) + } + w.again(t, w.statusWritten()) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r4"}) || + !slices.Equal(conditions, []string{"r3"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r4 on r3", written, conditions) + } + if owed := w.state(t).Written(); len(owed) != 0 { + t.Errorf("the state owes %v, want nothing after r4", owed) + } +} + +// TestAWriteAfterTheEnginesActivationAtTheSyncIsConditionedOnIt: the engine +// runs somebody else's document, and a rotated value has the engine activate +// it again as the sync writes the value. The operator's write of the rendered +// document is conditioned on that activation, read back as the same document, +// not on the revision the reconcile read before the sync: conditioned on that +// one, every rotation that came with a change of the document was refused as +// a lost race and written again by a second reconcile. +func TestAWriteAfterTheEnginesActivationAtTheSyncIsConditionedOnIt(t *testing.T) { + w := newWithdrawal(t, runningOther) + w.rotate(t) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + written, refused := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || refused != 0 || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v (and %d were refused), want r3 alone, conditioned on "+ + "r2, the engine's activation at the sync", written, conditions, refused) + } +} diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 87f05095..74a245c9 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -123,8 +123,16 @@ type configRun struct { rendered, running *crewletaiv1alpha1.RevisionPlacement runningRevision string // active is the document the engine ran when converge last read it, - // nil before it did or while the engine had none. + // nil before it did, while the engine had none, and while it could not + // open the one it ran. active []byte + // unsealed is the engine's active revision when converge last read it, + // where the engine stores it without a seal and the reconcile replaces + // it (engine.Unsealed, under Enforce): read by nothing, the operator and + // the engine's own nodes included, which run the revision before it, so + // the reconcile goes on with no active document and the write is + // conditioned on it (enforce). Empty for any other revision. + unsealed string // placement is what status.config.placement becomes once this reconcile // writes the status; nil leaves it as it is. placement *placementUpdate @@ -406,16 +414,34 @@ func (run *configRun) convergeOnce(ctx context.Context, eng *engine.Client) (ctr // the reconcile starts over. run.confirmed = "" active, err := activeConfig(ctx, eng) + run.active, run.unsealed = nil, "" + if revision, ok := engine.Unsealed(err); ok && run.policy() == crewletaiv1alpha1.ManagementPolicyEnforce { + // A REVISION STORED WITHOUT A SEAL IS REPLACED, under Enforce: the + // engine's own remedy for one is a whole document, which goes on + // over it reading nothing from it, and the rendered document is + // one. Nothing reads what it holds, the operator included, so the + // reconcile goes on as over an engine with no document, but for the + // write's base, which is this revision. Under Seed the operator + // writes only into an engine with no document, and under Observe it + // writes nothing, so there it stays what blocks the document + // (revisionUnreadable), as a revision sealed in a way the engine + // cannot open does under every policy: a write over one of those + // would discard a document somebody sealed, and the engine refuses + // it. + run.unsealed, err = revision, nil + } if err != nil { return ctrl.Result{}, err } + runs := run.unsealed if active != nil { run.active = active.Document + runs = active.RevisionID } // Before anything is decided from it: the revision the engine runs may be // a write of the operator's that the engine could not confirm // (config_unconfirmed.go). - if err := run.settleUnconfirmed(ctx, eng, active); err != nil { + if err := run.settleUnconfirmed(ctx, eng, runs, active); err != nil { return ctrl.Result{}, err } switch policy := run.policy(); { @@ -481,7 +507,9 @@ func activeConfig(ctx context.Context, eng *engine.Client) (*engine.ActiveConfig // enforce makes the engine run the rendered document: under Enforce always, // and under Seed for the one write into an engine with no document (active -// is nil then). +// is nil then). Under Enforce active is nil too where the engine's active +// revision is one it stores without a seal, which the write replaces +// (run.unsealed), and which its nodes do not run. func (run *configRun) enforce( ctx context.Context, eng *engine.Client, desired secretsync.Desired, active *engine.ActiveConfig, ) (ctrl.Result, error) { @@ -528,8 +556,22 @@ func (run *configRun) enforce( // the store at every apply attempt, and a data pod starts on the // revision its volume names, which does not start without them // (keepServing). + // + // A revision stored without a seal keeps everything the store holds: + // which names it references is what nobody can read, the operator + // included, and it is not what the engine's nodes run. The engine + // activates it again neither by a reload nor by a revert to it, so each + // node goes on running the revision it applied before it, and boots that + // one again when it restarts (it never adopts a body it cannot open), + // reading the store as it finds it then. A name deleted now is one such a + // node may start without, read as required by a revision nobody sees, for + // as long as the replacement is held: by the engine refusing it, by a + // seat somebody holds, or by a write that has not been sent yet. So + // nothing is deleted before the write; the sync after it decides by the + // revision then running, the replacement, as after any write. pushing := desired - if active != nil { + switch { + case active != nil: state = run.adopt(state, active, diffs) pushing.Keep(active.RevisionID, storeNamesIn(active.Document)...) released, err := secretsync.Absent(ctx, run.APIReader, state.LetGoKeys(active.RevisionID)) @@ -537,6 +579,8 @@ func (run *configRun) enforce( return ctrl.Result{}, err } pushing.Release(released...) + case run.unsealed != "": + pushing.KeepListed(run.unsealed) } if err := run.keepServing(ctx, eng, &pushing); err != nil { return ctrl.Result{}, err @@ -571,16 +615,16 @@ func (run *configRun) enforce( // the engine at the removal or else by the operator. One that did not is // activated again, which is how a rotated credential reaches a running // engine: by the engine itself, which re-activates the revision that - // reads a value it is written (reloadedBySync), or else by the + // reads a value it is written (syncActivation), or else by the // operator's apply below. if active != nil && len(diffs) == 0 && len(state.Written()) == 0 { return run.keepInSync(ctx, eng, active, hash, pushed.Reload, pushed.ReloadAction, state, stateRef) } - reloaded, err := run.reloadedBySync(ctx, eng, active, diffs, pushed.Reload) + synced, err := run.syncActivation(ctx, eng, active, pushed.Reload) if err != nil { return ctrl.Result{}, err } - if reloaded != nil { + if reloaded := synced; reloaded != nil && run.rendersAs(active, diffs) { // The engine runs the rendered document with every value the sync // left in its store, activated by the sync's own last request after // every write and removal: applying it again would be a second @@ -600,8 +644,7 @@ func (run *configRun) enforce( // had before the operator took it over. drifted := active != nil && len(diffs) > 0 && active.RevisionID != recorded.RevisionID - checked, err := eng.ApplyConfig(ctx, engine.ApplyRequest{Document: run.doc.YAML, DryRun: true}) - if err != nil { + if _, err := eng.ApplyConfig(ctx, engine.ApplyRequest{Document: run.doc.YAML, DryRun: true}); err != nil { if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef, err); refused { return result, saveErr } @@ -627,19 +670,33 @@ func (run *configRun) enforce( return ctrl.Result{}, err } req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, state.Written())} - if checked.BaseRevisionID == "" { + // THE WRITE IS CONDITIONED ON THE REVISION THIS RECONCILE READ, which its + // diff, its drift, its summary and the names its sync kept were decided + // against: the engine's document, a revision stored without a seal, which + // the write replaces, or none, which makes it create-only (the one write + // Seed makes, since Seed writes only where the read found no document). + // Not on the revision the dry run names: a write that landed since the + // read is the one the dry run was checked against, and a write + // conditioned on that went over it unseen, reported as replacing the + // revision the reconcile read. Conditioned on the read, the engine + // refuses it before storing anything, and the reconcile starts over from + // a fresh read (converge). + // The engine's own activation of that revision at the sync is the + // same document, read back (syncActivation): the write is conditioned + // on it, or it would be refused over the sync's own activation. + switch { + case synced != nil: + req.IfMatch = synced.RevisionID + case active != nil: + req.IfMatch = active.RevisionID + case run.unsealed != "": + req.IfMatch = run.unsealed + default: req.CreateOnly = true - } else { - // What the dry run checked against: a write in between is lost - // rather than overwritten unseen. - req.IfMatch = checked.BaseRevisionID - } - if run.policy() == crewletaiv1alpha1.ManagementPolicySeed { - req.IfMatch, req.CreateOnly = "", true } applied, err := eng.ApplyConfig(ctx, req) if err != nil { - return run.writeFailed(ctx, eng, err, writeAttemptOf(req, hash, drifted, diffs), active, pushed.Reload, + return run.writeFailed(ctx, eng, err, run.writeAttemptOf(req, hash, drifted, diffs), active, pushed.Reload, pushed.ReloadAction, state, stateRef) } run.recordApply(applied, hash, drifted, active, diffs, req.Summary) @@ -751,11 +808,15 @@ type writeAttempt struct { } // writeAttemptOf is req, a write of the rendered document whose appliedHash -// is hash, as a writeAttempt: drifted where the revision it was built on is -// somebody else's, which differed from the rendered document at diffs. -func writeAttemptOf(req engine.ApplyRequest, hash string, drifted bool, diffs []render.Path) writeAttempt { +// is hash, as a writeAttempt: over a revision stored without a seal +// (run.unsealed), or drifted where the revision it was built on is somebody +// else's, which differed from the rendered document at diffs. +func (run *configRun) writeAttemptOf(req engine.ApplyRequest, hash string, drifted bool, diffs []render.Path) writeAttempt { attempt := writeAttempt{base: req.IfMatch, hash: hash} - if drifted { + switch { + case run.unsealed != "": + attempt.replaced = crewletaiv1alpha1.ReplacedUnsealed + case drifted: attempt.replaced, attempt.driftedAt = crewletaiv1alpha1.ReplacedDrifted, placesOf(diffs) } return attempt @@ -807,40 +868,36 @@ func (run *configRun) adopt(state secretsync.State, active *engine.ActiveConfig, return state.Adopted(active.RevisionID, run.doc.Optional) } -// reloadedBySync is the revision the engine activated when the sync changed -// its store, where that is the rendered document running with every value -// the sync left there: nil where the operator has to apply the document -// itself. +// syncActivation is the engine's own activation of active at the sync, read +// back: the revision it activated when the sync changed its store, where the +// engine runs it with active's document, the one the operator read before +// the sync; nil where the operator has to activate a document itself. // // The engine re-activates its active revision whenever a request writes or // removes a value that revision reads, once for all the values of one write, // and answers what that did; reload, secretsync.Result.Reload, is the answer -// to the last such request of the sync, after which the store held, of every value -// the revision reads, what it holds now. Where that re-activation landed and -// the document the engine ran before it is the rendered one (diffs is empty, -// and both reference every store name as often, so that the rendered -// document's reading of the store is that revision's, as adopt says), the -// engine runs what an apply of the rendered document would activate. It -// names no revision, so as in reactivate it is read back: the engine must run -// the revision it answered, with the document the operator read before the -// sync, or somebody else's write landed in between and the document is -// applied. +// to the last such request of the sync, after which the store held, of every +// value the revision reads, what it holds now. It names no revision it was +// built on, so as in reactivate it is read back: the engine must run the +// revision it answered, with the document the operator read before the +// sync, or somebody else's write landed in between, and what the reconcile +// decided against its read no longer holds. // // A re-activation that failed, one the engine did not make because nothing // the revision reads changed, and a write whose answer is not read, leave the -// apply to the operator, as before the engine re-activated at all. -func (run *configRun) reloadedBySync( - ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, diffs []render.Path, reload *engine.Reload, +// activation to the operator, as before the engine re-activated at all. +func (run *configRun) syncActivation( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, ) (*engine.ApplyResult, error) { - if active == nil || reload == nil || len(diffs) > 0 || - !maps.Equal(storeReferences(run.doc.YAML), storeReferences(active.Document)) { + if active == nil || reload == nil { return nil, nil } log := logf.FromContext(ctx) if !reload.Landed() { // The engine sealed the values and says why it did not - // re-activate: the apply below is the activation instead. - log.Info("The engine did not activate its document again after the secret sync; applying it", + // re-activate: the operator's write or reload is the activation + // instead. + log.Info("The engine did not activate its document again after the secret sync", "error", reload.Error, "detail", reload.Detail) return nil, nil } @@ -849,13 +906,24 @@ func (run *configRun) reloadedBySync( return nil, err } if now == nil || now.RevisionID != reload.RevisionID || !bytes.Equal(now.Document, active.Document) { - log.Info("The engine runs another revision than the one it activated at the secret sync; applying the document", + log.Info("The engine runs another revision than the one it activated at the secret sync", "reloaded", reload.RevisionID) return nil, nil } return &engine.ApplyResult{RevisionID: reload.RevisionID, Epoch: reload.Epoch, Warnings: []string{}}, nil } +// rendersAs reports whether active, the engine's document, is the rendered +// one: diffs is empty, and both reference every store name as often, so that +// the rendered document's reading of the store is that revision's, as adopt +// says. An activation of it again with every value the sync left in the +// store (syncActivation) is then what an apply of the rendered document +// would activate. +func (run *configRun) rendersAs(active *engine.ActiveConfig, diffs []render.Path) bool { + return active != nil && len(diffs) == 0 && + maps.Equal(storeReferences(run.doc.YAML), storeReferences(active.Document)) +} + // settleRead is state with every change of the store settled that the // fleet's activation came after (State.Ran), or that awaits the revision // this attempt read (State.Found: one the engine stored for an activation @@ -1155,6 +1223,8 @@ func (run *configRun) recordApply( } run.applied = appliedVerdict{kind: appliedInRevision, revision: applied.RevisionID} switch { + case run.unsealed != "": + run.reportReplaced(&condition, applied.RevisionID, run.unsealed, crewletaiv1alpha1.ReplacedUnsealed, "") case run.policy() == crewletaiv1alpha1.ManagementPolicySeed: condition.Reason = crewletaiv1alpha1.ReasonSeeded condition.Message = fmt.Sprintf("Seeded the engine with the rendered document as revision %s; "+ @@ -1179,23 +1249,35 @@ func (run *configRun) recordApply( // reportReplaced says on condition, ConfigApplied for revision written, and in // a Warning event what written replaced where that was more than the -// operator's own document: base, which was replaced (a Drifted revision the -// operator did not write, which differed from the rendered document at -// driftedAt). recordApply reports a write the engine confirmed with it, and -// confirmUnconfirmed one it could not confirm, once a read shows it runs, so -// both say the same. +// operator's own document: base, which was replaced (an Unsealed revision the +// engine stores without a seal, or a Drifted revision the operator did not +// write, which differed from the rendered document at driftedAt). recordApply +// reports a write the engine confirmed with it, and confirmUnconfirmed one it +// could not confirm, once a read shows it runs, so both say the same. func (run *configRun) reportReplaced( condition *metav1.Condition, written, base string, replaced crewletaiv1alpha1.ReplacedRevision, driftedAt string, ) { - if replaced != crewletaiv1alpha1.ReplacedDrifted { - return + switch replaced { + case crewletaiv1alpha1.ReplacedUnsealed: + // Not drift, which says what differed: what the revision held is + // what nobody could read. Told as a Warning all the same, under the + // reason of the condition it blocks a document with under the other + // policies, since a revision went that nobody saw. + condition.Message = fmt.Sprintf("Applied the rendered document as revision %s, replacing revision %s, which the "+ + "engine stores without a seal and does not open: a whole document replaces such a revision, reading nothing "+ + "from it.", written, base) + run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonRevisionUnreadable, "ApplyConfig", + "The engine's active revision %s was one it stores without a seal and does not open, so nobody could read "+ + "what it held and its nodes ran the revision before it; under the Enforce policy revision %s, the "+ + "rendered document, replaced it, reading nothing from it.", base, written) + case crewletaiv1alpha1.ReplacedDrifted: + condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected + condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ + "revision %s replaced it.", base, driftedAt, written) + run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", + "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ + "under the Enforce policy revision %s replaced it.", base, driftedAt, written) } - condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected - condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ - "revision %s replaced it.", base, driftedAt, written) - run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", - "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ - "under the Enforce policy revision %s replaced it.", base, driftedAt, written) } // observe compares the rendered document with the engine's and reports the @@ -1583,12 +1665,21 @@ func (run *configRun) engineFailed(ctx context.Context, eng *engine.Client, err // engineRetry is how soon the reconcile looks at a Company's engine again // after err. A revision the engine cannot open is looked at again at the // resync interval, as an engine with nothing to do is: the engine sends no -// Retry-After for it, no wait opens it, and nothing the operator sends does. -// What does is given to the data pods (a key put back in the keyring Secret, -// which they read only when they start, or the build that sealed the -// revision), and the data pods restarting changes RuntimeReady, which runs -// the reconcile at once. Anything else is asked again as retryAfter says. +// Retry-After for it, no wait opens it, and nothing the operator sends does +// while it blocks the document. What does is given to the data pods (a key +// put back in the keyring Secret, which they read only when they start, or +// the build that sealed the revision), and the data pods restarting changes +// RuntimeReady, which runs the reconcile at once; or, for a revision stored +// without a seal, the Company's policy set to Enforce, which runs it at once +// too, or a whole document an admin sends. Under Enforce, though, the +// operator's own next look replaces a revision stored without a seal (one +// that landed after the reconcile's read, which is the only way one reaches +// here), so that look comes engineRetryInterval on rather than a resync +// later. Anything else is asked again as retryAfter says. func (run *configRun) engineRetry(err error) time.Duration { + if _, ok := engine.Unsealed(err); ok && run.policy() == crewletaiv1alpha1.ManagementPolicyEnforce { + return engineRetryInterval + } if errors.Is(err, engine.ErrRevisionUnreadable) { return run.resyncInterval() } @@ -1597,18 +1688,21 @@ func (run *configRun) engineRetry(err error) time.Duration { // revisionUnreadable records an engine that cannot open the fleet's revision // of the company document (engine.ErrRevisionUnreadable), and reports false -// for any other error. The engine refuses then to show the document and to -// take another, since a write reads the revision it replaces, so the operator -// can neither see the document nor change it. It is reported as what blocks -// the document, in the engine's words, with what restores the revision: a key -// the revision is sealed under is one of the keyring Secret's, which the -// operator renders into every pod and never rewrites, so a key gone from it is -// put back there, from the backup kept with the data volumes, and a key the -// fleet sealed under that the data pod lacks is one it takes when it restarts -// with the Secret; a seal the engine's build does not read is most likely a -// newer build's, which the data pods run again, since no write replaces a -// document somebody sealed. A revision without a seal no key opens, and the -// engine's hint says what does. +// for any other error. The engine shows that document to nobody and takes no +// write built from it, so the operator can neither see the document nor +// change it, but for the one write the engine takes over a revision stored +// without a seal: a whole document, which reads nothing from it and replaces +// it, and which converge sends under the Enforce policy. It is reported as +// what blocks the document, in the engine's words, with what restores the +// revision or replaces it: a key the revision is sealed under is one of the +// keyring Secret's, which the operator renders into every pod and never +// rewrites, so a key gone from it is put back there, from the backup kept with +// the data volumes, and a key the fleet sealed under that the data pod lacks +// is one it takes when it restarts with the Secret; a seal the engine's build +// does not read is most likely a newer build's, which the data pods run +// again, since no write replaces a document somebody sealed; and a revision +// without a seal no key opens, and a whole document replaces it: the rendered +// one under Enforce (unsealedRemedy). func (run *configRun) revisionUnreadable(err error) bool { var apiErr *engine.APIError if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) { @@ -1622,6 +1716,17 @@ func (run *configRun) revisionUnreadable(err error) bool { "the document nor takes another, and no wait changes that", revision) var remedy string switch apiErr.Code { + case "unsealed_revision", "unsealed_fleet_revision": + said = fmt.Sprintf("The engine does not open %s of the company document, which it stores without a seal, so "+ + "the engine shows the document to nobody and takes no write but a whole document that replaces it, reading "+ + "nothing from it, and no wait changes that", revision) + remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + + "requires, and no keyring opens it. " + if apiErr.Code == "unsealed_fleet_revision" { + remedy = "Every engine seals what it activates, so a revision on the fleet's activation pointer without a " + + "seal was put there from outside the fleet, and no keyring opens it. " + } + remedy += run.unsealedRemedy() case "unknown_seal_revision": said = fmt.Sprintf("The engine does not read the seal of %s of the company document, so the engine neither "+ "shows the document nor takes another, and no wait changes that", revision) @@ -1634,12 +1739,6 @@ func (run *configRun) revisionUnreadable(err error) bool { "data pods on the build that sealed it: the engine image of %s, with the Company's spec.runtime.version as "+ "its tag where the class allows that. A data pod still rolling to it takes it when it restarts.", keyringSecretName(run.company), class) - case "unsealed_revision": - remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + - "requires, and no keyring opens it; the engine's hint says what brings its document back." - case "unsealed_fleet_revision": - remedy = "Every engine seals what it activates, so a revision on the fleet's activation pointer without a " + - "seal was put there from outside the fleet, and no keyring opens it; the engine's hint says what to do." case "unreadable_fleet_revision": remedy = fmt.Sprintf("Every pod of the company has its keyring from Secret %s: a data pod that has not "+ "restarted since the Secret gained the key the fleet sealed %s under takes it when it restarts, and a key "+ @@ -1655,6 +1754,28 @@ func (run *configRun) revisionUnreadable(err error) bool { return true } +// unsealedRemedy is what replaces a revision the engine stores without a seal, +// under the Company's policy: a whole document, which the operator sends +// itself only under Enforce. Seed writes only into an engine with no document +// and Observe writes nothing, so under those an admin of the company sends +// one, as the engine's hint says, or the policy is set to Enforce. Under +// Enforce converge replaces it as it reads it, so it blocks a document only +// where it lands after that read, until the next look. +func (run *configRun) unsealedRemedy() string { + switch policy := run.policy(); policy { + case crewletaiv1alpha1.ManagementPolicyEnforce: + return "Under the Company's Enforce policy the operator replaces it with the rendered document at its next look." + case crewletaiv1alpha1.ManagementPolicySeed: + return "Under the Company's Seed policy the operator writes only into an engine with no document, so it does " + + "not replace it: an admin of the company replaces it with a whole document, as the engine's hint says, or " + + "the Company's policy is set to Enforce, under which the operator replaces it with the rendered document." + default: + return fmt.Sprintf("Under the Company's %s policy the operator writes nothing, so it does not replace it: an "+ + "admin of the company replaces it with a whole document, as the engine's hint says, or the Company's "+ + "policy is set to Enforce, under which the operator replaces it with the rendered document.", policy) + } +} + // unavailable records an engine that could not be reached, or that failed a // request for a reason of its own. func (run *configRun) unavailable(err error) { @@ -1729,7 +1850,10 @@ func (run *configRun) notGranted(err error) bool { // seatHeld records an engine that refuses the document because it takes a // human seat out of the company, or makes it an agent's, while something in // the engine's identity directory holds the seat (engine.SeatHeldError), and -// reports false for any other error. The document is valid and stays +// reports false for any other error. A document replacing a revision stored +// without a seal (run.unsealed) is refused for every seat it does not keep as +// a human seat that something holds, since nobody can read which seats that +// revision had, and is told so. The document is valid and stays // unapplied until an admin of the company frees each seat, which the // operator neither can nor should do: a seat's holder is a person, or what a // person set up. The message names each seat and what kind of thing holds @@ -1777,10 +1901,18 @@ func (run *configRun) seatHeld(err error) bool { if len(remedies) > 0 { remedy = ": " + strings.Join(remedies, "; ") } - message := fmt.Sprintf("The engine refuses the rendered document: it takes human seats out of the company, or makes "+ - "them agents', while its identity directory holds them: %s. An admin of the company frees each seat first, in "+ - "the engine's dashboard or with `crewlet iam` (`crewlet iam seats` lists what holds each seat)%s. Or keep "+ - "each seat as a Human Seat.", strings.Join(seats, "; "), remedy) + what := "it takes human seats out of the company, or makes them agents', while its identity directory holds them" + if run.unsealed != "" { + // Replacing a revision nothing was read from, the engine asked about + // every seat the document does not keep as a human seat, so it + // cannot say the revision it replaces had them, and neither can the + // operator. + what = fmt.Sprintf("it replaces revision %s, which the engine does not open, and does not keep as human seats "+ + "these seats its identity directory holds", run.unsealed) + } + message := fmt.Sprintf("The engine refuses the rendered document: %s: %s. An admin of the company frees each seat "+ + "first, in the engine's dashboard or with `crewlet iam` (`crewlet iam seats` lists what holds each seat)%s. Or "+ + "keep each seat as a Human Seat.", what, strings.Join(seats, "; "), remedy) run.blockAndTell(crewletaiv1alpha1.ReasonSeatHeld, "DryRunConfig", message) return true } @@ -1867,7 +1999,8 @@ func placesOf(diffs []render.Path) string { // summary is the X-Summary of a write: "crewlet-operator: " and the objects // whose part of the document changed, or the Secrets whose values, written // since a revision last activated (written, secretsync.State.Written), the -// write makes the engine read. +// write makes the engine read; for a write that replaces a revision stored +// without a seal (run.unsealed), that revision, and every object. func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, written []string) string { var objects []string add := func(s string) { @@ -1905,6 +2038,11 @@ func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, if len(objects) > shown { text += fmt.Sprintf(" and %d more", len(objects)-shown) } + if run.unsealed != "" { + // What it replaced is the one record of it the engine's history + // keeps: the revision itself shows nothing. + return fmt.Sprintf("crewlet-operator: replaced revision %s, stored without a seal, with %s", run.unsealed, text) + } return "crewlet-operator: " + text } diff --git a/internal/controller/config_unconfirmed.go b/internal/controller/config_unconfirmed.go index 4552b486..bbb3d452 100644 --- a/internal/controller/config_unconfirmed.go +++ b/internal/controller/config_unconfirmed.go @@ -142,24 +142,30 @@ func documentDigest(document []byte) string { } // unconfirmed records write, whose activation the engine could not confirm, -// as revision stored, reports it on ConfigApplied, and returns when the next +// as revision storedID, reports it on ConfigApplied, and returns when the next // reconcile settles it (settleUnconfirmed): as soon as the engine's // Retry-After says, the scale of one batch on its coordination store, and ten // seconds on where it sends none (engineRetry). -func (run *configRun) unconfirmed(stored string, write writeAttempt, err error) time.Duration { +func (run *configRun) unconfirmed(storedID string, write writeAttempt, err error) time.Duration { retry := run.engineRetry(err) recorded := run.recordedConfig() config := recorded.DeepCopy() now := metav1.NewTime(run.clock()) config.Unconfirmed = boundedWrites(append(config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ - RevisionID: stored, BaseRevisionID: write.base, AppliedHash: write.hash, WrittenAt: &now, + RevisionID: storedID, BaseRevisionID: write.base, AppliedHash: write.hash, WrittenAt: &now, Objects: run.objectCounts(), Replaced: write.replaced, DriftedAt: write.driftedAt, })) run.config = config + stored := fmt.Sprintf("revision %s", storedID) + if write.replaced == crewletaiv1alpha1.ReplacedUnsealed { + // Said now, since the next report may be a while: the engine + // runs a revision nobody can read, which the write would replace. + stored += fmt.Sprintf(", which would replace revision %s, stored without a seal,", write.base) + } run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: ended(fmt.Sprintf("The engine stored the rendered document as revision %s and could not confirm that it "+ + Message: ended(fmt.Sprintf("The engine stored the rendered document as %s and could not confirm that it "+ "activated it, so it may run it or may not. The operator reads which revision the engine runs before it writes "+ "again, in %s: %s", stored, retry, err)), } @@ -217,24 +223,24 @@ func overtakenBy( } // settleUnconfirmed settles the writes status.config.unconfirmed records -// against active, the revision the engine runs as this reconcile read it. One -// of them is active, with the document it was recorded with -// (runsItsDocument): it is recorded as the operator's write -// (confirmUnconfirmed). Otherwise the ones built on active stay, and every -// other one goes, to status.config.overtaken where it was written into an -// engine that ran no document; a reload found running another document was -// built on a revision somebody else activated, and goes too. Under Observe -// nothing is settled: the operator writes nothing there, and Observe records -// the revision it finds in sync. -func (run *configRun) settleUnconfirmed(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) error { +// against runs, the revision the engine runs as this reconcile read it, empty +// for none: active, whose document it read, or one it could not open whose +// write the reconcile replaces (configRun.unsealed), on which a write the +// engine could not confirm may have been built as well. One of them runs, with +// the document it was recorded with (runsItsDocument): it is recorded as the +// operator's write (confirmUnconfirmed). Otherwise the ones built on runs +// stay, and every other one goes, to status.config.overtaken where it was +// written into an engine that ran no document; a reload found running +// another document was built on a revision somebody else activated, and goes +// too. Under Observe nothing is settled: the operator writes nothing there, +// and Observe records the revision it finds in sync. +func (run *configRun) settleUnconfirmed( + ctx context.Context, eng *engine.Client, runs string, active *engine.ActiveConfig, +) error { recorded := run.recordedConfig() if len(recorded.Unconfirmed) == 0 || run.policy() == crewletaiv1alpha1.ManagementPolicyObserve { return nil } - runs := "" - if active != nil { - runs = active.RevisionID - } if i := unconfirmedIndex(recorded.Unconfirmed, runs); i >= 0 && runsItsDocument(&recorded.Unconfirmed[i], active) { return run.confirmUnconfirmed(ctx, eng, &recorded, i) } @@ -258,7 +264,11 @@ func (run *configRun) settleUnconfirmed(ctx context.Context, eng *engine.Client, // record kept, or where it is a reload of a write recorded here too, what // that write replaced, which only it records (reloadUnconfirmed), and which // then goes with it: the revision it activated again landed, since the reload -// runs. A write built on it stays unconfirmed, and every other one is +// runs. A write built on it stays unconfirmed, since it may land yet: +// a reconcile that holds its document reads the engine without recording what +// it finds (status.config is left to the one that applies a document), so it +// may find this one running and activate it again for a value it withdrew, and +// that reload's answer can be lost too (reloadUnconfirmed). Every other one is // overtaken by it, and so is every write status.config.overtaken keeps: none // can land now, nor be the seed, since a write of the operator's runs. // diff --git a/internal/controller/config_unconfirmed_test.go b/internal/controller/config_unconfirmed_test.go index 9574f449..4f524ee4 100644 --- a/internal/controller/config_unconfirmed_test.go +++ b/internal/controller/config_unconfirmed_test.go @@ -602,45 +602,6 @@ func TestASeedActivatedAgainIsReportedAsTheSeed(t *testing.T) { } } -// TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed: a write the operator -// built on r2 in the moment r2 landed, before any read showed it running, may -// land yet once r2 is confirmed: it stays unconfirmed, and is confirmed in -// turn once it runs. Taken for a write r2 overtook, it was adopted as a -// revision the operator had not written once it landed. -func TestAWriteBuiltOnTheOneConfirmedStaysUnconfirmed(t *testing.T) { - w, company := unconfirmedWrite(t, landed) - w.engine.mu.Lock() - w.engine.inert = append(w.engine.inert, storeRevision{id: w.engine.mint(), document: w.engine.active().document}) - w.engine.mu.Unlock() - company.Status.Config.Unconfirmed = append(company.Status.Config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ - RevisionID: "r3", BaseRevisionID: "r2", AppliedHash: company.Status.Config.Unconfirmed[0].AppliedHash, - WrittenAt: &metav1.Time{}, - }) - w.again(t, company) - reconcileOnce(t, w) - company = w.statusWritten() - if config := company.Status.Config; config.RevisionID != "r2" || len(config.Unconfirmed) != 1 || - config.Unconfirmed[0].RevisionID != "r3" || len(config.Overtaken) != 0 { - t.Fatalf("status.config is %+v, want r2 recorded and r3, built on it, still unconfirmed", config) - } - w.engine.mu.Lock() - w.engine.land("r3") - w.engine.mu.Unlock() - w.again(t, company) - reconcileOnce(t, w) - if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { - t.Errorf("status.config is %+v, want r3 recorded as the operator's, and nothing unconfirmed", config) - } - if recorded := drained(w.events); !slices.ContainsFunc(recorded, func(e string) bool { - return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") - }) { - t.Errorf("events %q, want an Activated event saying the engine runs r3, the operator's write, as epoch 3", recorded) - } - if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { - t.Errorf("the operator wrote %v, want r2 alone", written) - } -} - // TestAWriteAReactivationOvertookIsNotSentAgain: a held reconcile activates // r1 again (r3), for the withdrawal of the registry, while a write built on it // (r2) is unconfirmed. r2 can no longer land, and r3 activated after the @@ -990,6 +951,68 @@ func TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns(t *testing.T) { } } +// TestAReloadBuiltOnAnUnconfirmedWriteStaysUnconfirmedOnceTheWriteIs: a held +// reconcile finds r2, a write of the operator's whose activation went +// unconfirmed, running, and activates it again for the registry it withdrew; +// the engine stores that reload as r3, conditioned on r2, and loses the +// answer, and r2 still runs by the read back, so r3 is recorded as a write +// built on r2 (reloadUnconfirmed). The next reconcile confirms r2 and keeps +// r3, which may land yet, and its own reload of r2 (r4) is lost the same way; +// once r3 lands it is confirmed in turn, as the operator's. Taken for a write +// r2 overtook, r3 was dropped with r2's confirmation, and its landing read as +// a revision somebody else activated. +func TestAReloadBuiltOnAnUnconfirmedWriteStaysUnconfirmedOnceTheWriteIs(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 2 || config.Unconfirmed[1].RevisionID != "r3" || + config.Unconfirmed[1].BaseRevisionID != "r2" { + t.Fatalf("status.config is %+v, want r2 unconfirmed and the reload r3 recorded on it", config) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != "r2" || !slices.ContainsFunc(config.Unconfirmed, + func(u crewletaiv1alpha1.UnconfirmedRevision) bool { + return u.RevisionID == "r3" && u.BaseRevisionID == "r2" + }) { + t.Fatalf("status.config is %+v, want r2 recorded as the operator's and r3, built on it, still unconfirmed", config) + } + w.engine.reloadActivation = confirmed + w.engine.mu.Lock() + w.engine.land("r3") + w.engine.mu.Unlock() + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 recorded as the operator's, and nothing unconfirmed", config) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonDrifted) || + !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") + }) { + t.Errorf("events %q, want r3 confirmed as the operator's, at epoch 3, and no drift reported", recorded) + } +} + // TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound: the write's // answer was lost three minutes before the reconcile that finds it running, // and the engine is one epoch behind. Nothing says when the activation landed, @@ -1306,20 +1329,23 @@ func TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce(t *testing.T) { } // TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced: the -// operator's write r2 replaced r1, somebody else's document, and its answer -// was lost; a held reconcile then found r2 running and its reload of r2 was -// stored as r3, its answer lost too. Once r3 runs, its confirmation reports -// that the rendered document replaced r1, once, and never names r2, the -// operator's own write, as somebody else's: whether r2 was confirmed first or -// not at all. +// operator's write r2 replaced r1, somebody else's document or one the engine +// stores without a seal, and its answer was lost; a held reconcile then found +// r2 running and its reload of r2 was stored as r3, its answer lost too. Once +// r3 runs, its confirmation reports that the rendered document replaced r1, +// once, and never names r2, the operator's own write, as somebody else's or +// as stored without a seal: whether r2 was confirmed first or not at all. func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing.T) { for _, c := range []struct { - name string + name string + unsealed bool // confirmFirst confirms r2 in a reconcile of its own before r3 lands. confirmFirst bool }{ - {name: "r3 lands first"}, - {name: "r2 confirmed first", confirmFirst: true}, + {name: "r1 somebody else's, r3 lands first"}, + {name: "r1 somebody else's, r2 confirmed first", confirmFirst: true}, + {name: "r1 stored without a seal, r3 lands first", unsealed: true}, + {name: "r1 stored without a seal, r2 confirmed first", unsealed: true, confirmFirst: true}, } { t.Run(c.name, func(t *testing.T) { ctx := context.Background() @@ -1328,6 +1354,9 @@ func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing company := w.run.company.DeepCopy() company.Status.Config.RevisionID = "r0" w.again(t, company) + if c.unsealed { + w.engine.cannotOpen(sealMissing) + } if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { t.Fatal(err) } @@ -1368,6 +1397,9 @@ func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing reconcileOnce(t, w) reports = append(reports, drained(w.events)...) reason := crewletaiv1alpha1.ReasonDrifted + if c.unsealed { + reason = crewletaiv1alpha1.ReasonRevisionUnreadable + } var warnings []string for _, e := range reports { if strings.HasPrefix(e, "Warning "+reason+" ") { @@ -1375,9 +1407,11 @@ func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing } } // What the event says of the revision replaced: "revision r1, - // which the operator did not write". + // which the operator did not write", or "active revision r1 was + // one it stores without a seal". names := func(revision string) bool { - return strings.Contains(warnings[0], "revision "+revision+", which") + return strings.Contains(warnings[0], "revision "+revision+", which") || + strings.Contains(warnings[0], "revision "+revision+" was one") } if len(warnings) != 1 || !names("r1") || names("r2") { t.Errorf("Warning %s events %q, want one, naming r1 and never r2", reason, warnings) diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go index 91eb3d20..ddf803a2 100644 --- a/internal/controller/config_unreadable_test.go +++ b/internal/controller/config_unreadable_test.go @@ -2,23 +2,35 @@ package controller import ( "context" + "encoding/json" "fmt" + "net/http" + "slices" "strings" "testing" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretref" + "github.com/crewlet/crewlet-operator/internal/secretsync" ) // These tests hold what the operator does with an engine that cannot open the // fleet's revision of the company document: the engine answers every read of -// the document, and every write, which reads the revision it replaces, 409 -// with no Retry-After, and nothing clears it but somebody giving the engine -// what opens the revision. So it is reported as what blocks the +// the document 409 with no Retry-After, and every write built from it too, and +// no wait clears it. A revision sealed in a way the engine cannot open (under a +// key its keyring lacks, or in a form its build does not read) is somebody's +// document, which no write replaces: it is reported as what blocks the // document, in the engine's words and with the remedy, and looked at again at // the resync interval, not ten seconds on with a promise of a retry that -// changes nothing. +// changes nothing. A revision stored without a seal is the one a whole document +// replaces, reading nothing from it: under Enforce the operator replaces it +// with the rendered document, and under Seed and Observe, which write no such +// thing, it blocks the document as the others do. // keyringLost, sealMissing and unknownSeal are the engine's answers to a read // of revision, sealed under a key its keyring no longer holds, stored without a @@ -87,26 +99,93 @@ func fleetUnknownSeal(revision string) string { `discard it unread; upgrade this node to the build the rest of the fleet runs."}`, revision) } -// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine that -// cannot open the fleet's revision to ConfigApplied=False -// RevisionUnreadable: the message names the revision, quotes the engine's -// account and its remedy, and says what restores it in the Company's terms; -// the operator writes nothing; and the next look is at the resync interval. +// cannotOpen makes the fleet's revision one the engine cannot open, answering +// every request that reads it with answer of its id. +func (e *storeEngine) cannotOpen(answer func(revision string) string) { + e.mu.Lock() + defer e.mu.Unlock() + e.revisions[len(e.revisions)-1].unopened = answer +} + +// readsActive reports whether r is a request the engine answers by reading +// active, which it cannot open, and so refuses as it refuses a read of it: a +// read of the document; a reload, which activates it again; and a write +// (a dry run included) over any but one stored without a seal, which a whole +// document replaces, reading nothing from it. +func readsActive(r *http.Request, active storeRevision) bool { + switch { + case r.URL.Path == configRoute && r.Method == http.MethodGet: + return true + case r.URL.Path == configRoute && r.Method == http.MethodPut: + return !replaceable(active) + } + return r.URL.Path == reloadRoute && r.Method == http.MethodPost +} + +// replaceable reports whether a write replaces revision, which the engine +// cannot open, as crewlet's configapi decides it (Service.prepare): a whole +// document goes on over a revision stored without a seal, and over no other. +func replaceable(revision storeRevision) bool { + var answer struct { + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(revision.unopened(revision.id)), &answer); err != nil { + panic(err) + } + return answer.Error == "unsealed_revision" || answer.Error == "unsealed_fleet_revision" +} + +// What the RevisionUnreadable message says, as more than one case expects +// it: what the keyring does not do and which Secret it is, what the engine +// does not do with a revision without a seal, and what replaces one under a +// policy that does not. +const ( + saysKeyring = "keyring does not open" + saysKeyringSecret = "Secret acme-keyring" + saysUnsealed = "which it stores without a seal" + saysEnforce = "set to Enforce" +) + +// withPolicy makes w's reconcile one of its Company under policy. +func withPolicy(t *testing.T, w *withdrawal, policy crewletaiv1alpha1.ManagementPolicy) { + t.Helper() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = policy + w.again(t, company) +} + +// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine that cannot +// open the fleet's revision, where no write of the operator's may replace it, +// to ConfigApplied=False RevisionUnreadable: the message names the revision, +// quotes the engine's account and its remedy, and says what restores it, or +// replaces it, in the Company's terms; the operator writes nothing, neither +// the document nor the store; and the next look is at the resync interval. // Reported as an engine that is unavailable, it promised a retry in ten -// seconds that no retry could keep. +// seconds that no retry could keep. A revision sealed in a way the engine +// cannot open blocks under every policy, Enforce included, and one stored +// without a seal under Seed and Observe, which write no document over the +// fleet's revision. func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { for _, c := range []struct { name string answer func(revision string) string + policy crewletaiv1alpha1.ManagementPolicy says []string }{ { - name: "a key the keyring lost", answer: keyringLost, - says: []string{"doesn't hold the key that revision is sealed under", - "restore that key to the node's secrets.keys first", "Secret acme-keyring"}, + name: "a key the keyring lost", answer: keyringLost, policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{saysKeyring, "doesn't hold the key that revision is sealed under", + "restore that key to the node's secrets.keys first", saysKeyringSecret}, + }, + { + name: "the fleet's revision under a key this node lacks", answer: fleetKeyMissing, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{saysKeyring, "doesn't hold the key the fleet sealed this revision under", + "Add the key the fleet seals under", saysKeyringSecret, "takes it when it restarts"}, }, { name: "a seal this node's build does not read", answer: unknownSeal, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, says: []string{"does not read the seal of revision r1", "sealed in a form this build of Crewlet doesn't read", "run that build on this node again", "newer than the one the data pod runs", "keyring Secret acme-keyring is not at fault", "no write replaces it", @@ -114,33 +193,50 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { }, { name: "the fleet's revision under a seal this node's build does not read", answer: fleetUnknownSeal, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, says: []string{"does not read the seal of revision r1", "The fleet's revision is sealed in a form", "upgrade this node to the build the rest of the fleet runs", "keyring Secret acme-keyring is not at fault", "the engine image of EngineClass standard"}, }, { - name: "a revision stored without a seal", answer: sealMissing, - says: []string{"stored without a seal, so nothing vouches", "an admin sends the whole document from your own copy", - "the engine's hint says what brings its document back"}, + name: "a key the keyring lost, under Seed", answer: keyringLost, policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysKeyring, saysKeyringSecret}, }, { - name: "the fleet's revision under a key this node lacks", answer: fleetKeyMissing, - says: []string{"doesn't hold the key the fleet sealed this revision under", - "Add the key the fleet seals under", "Secret acme-keyring", "takes it when it restarts"}, + name: "a revision stored without a seal, under Seed", answer: sealMissing, + policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysUnsealed, "stored without a seal, so nothing vouches", + "an admin sends the whole document from your own copy with PUT /config", + "stored by an engine older than the keyring", "Under the Company's Seed policy", + "writes only into an engine with no document", saysEnforce}, + }, + { + name: "a revision stored without a seal, under Observe", answer: sealMissing, + policy: crewletaiv1alpha1.ManagementPolicyObserve, + says: []string{saysUnsealed, "an admin sends the whole document from your own copy", + "Under the Company's Observe policy the operator writes nothing", saysEnforce}, + }, + { + name: "the fleet's revision without a seal, under Seed", answer: fleetSealMissing, + policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysUnsealed, "arrived without a seal", "came from outside the fleet", + "put there from outside the fleet", "Under the Company's Seed policy", saysEnforce}, }, { - name: "the fleet's revision without a seal", answer: fleetSealMissing, - says: []string{"arrived without a seal", "came from outside the fleet", - "put there from outside the fleet", "the engine's hint says what to do"}, + name: "the fleet's revision without a seal, under Observe", answer: fleetSealMissing, + policy: crewletaiv1alpha1.ManagementPolicyObserve, + says: []string{"arrived without a seal", "put there from outside the fleet", + "Under the Company's Observe policy the operator writes nothing", saysEnforce}, }, } { t.Run(c.name, func(t *testing.T) { ctx := context.Background() w := newWithdrawal(t, runningLetsGo) - w.engine.unreadable = c.answer + withPolicy(t, w, c.policy) + w.engine.cannotOpen(c.answer) _, err := w.run.converge(ctx, w.client) if err == nil { - t.Fatal("converge() = nil over a revision the engine's keyring does not open") + t.Fatal("converge() = nil over a revision the engine cannot open") } result := w.run.engineFailed(ctx, w.client, err) @@ -148,7 +244,7 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable { t.Fatalf("ConfigApplied = %+v, want False RevisionUnreadable", got) } - // Every answer names the revision the engine runs, r1. + // Every answer names the fleet's revision, r1. for _, s := range append([]string{"revision r1"}, c.says...) { if !strings.Contains(got.Message, s) { t.Errorf("ConfigApplied does not say %q: %s", s, got.Message) @@ -166,6 +262,10 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { t.Errorf("the operator wrote %v (and %d refused) over a revision the engine cannot open, want nothing", written, refused) } + if !w.engine.holds(w.registry) || !slices.Equal(w.engine.ids(), []string{"r1"}) { + t.Errorf("the store holds the registry: %t, and the revisions are %v; want both as they were", + w.engine.holds(w.registry), w.engine.ids()) + } if recorded := drained(w.events); !warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { t.Errorf("events %q, want a Warning RevisionUnreadable", recorded) } @@ -184,6 +284,518 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { } } +// TestEnforceReplacesARevisionStoredWithoutASeal holds the operator, under +// Enforce, to the engine's remedy for a revision it stores without a seal, +// this node's copy or the fleet's: the rendered document replaces it, as one +// write conditioned on that revision (If-Match), which is still the base +// whatever nobody can read of it, and the Company ends Applied, the engine +// having applied the epoch, with what was replaced said in the condition, the +// new revision's summary and a Warning event. Nothing is read from the +// revision, so nothing the store holds is deleted before the write, the value +// of a deleted Secret included: the engine's nodes run, and boot again, the +// revision before it, whose reading nobody knows. The sync after the write +// removes it, the replacement letting it go, and the replacement is activated +// again at once for the engine to read the store without it. The next +// reconcile finds the engine running the rendered document and writes +// nothing. +func TestEnforceReplacesARevisionStoredWithoutASeal(t *testing.T) { + for _, c := range []struct { + name string + answer func(revision string) string + }{ + {"this node's copy without a seal", sealMissing}, + {"the fleet's revision without a seal", fleetSealMissing}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + // The state records no reading of r1, so nothing says which of + // the store's names r1 reads. + w := newWithdrawal(t, runningNoRecord) + w.engine.cannotOpen(c.answer) + var heldAtWrite bool + w.engine.beforeWrite = func(e *storeEngine) { _, heldAtWrite = e.store[w.registry] } + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the rendered document written over a revision without a seal", err) + } + written, refused := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r2"}) || refused != 0 || + !slices.Equal(conditions, []string{"r1"}) { + t.Fatalf("the operator wrote %v on the conditions %v (and %d were refused), want r2 alone, "+ + "conditioned on r1", written, conditions, refused) + } + if !heldAtWrite { + t.Error("the store lost the value of a deleted Secret before the write, while the nodes run a revision " + + "nobody can read") + } + if summary := w.engine.summaryOfRevision("r2"); !strings.HasPrefix(summary, + "crewlet-operator: replaced revision r1, stored without a seal, with Company/acme") { + t.Errorf("r2 is summarised %q, want it saying it replaced r1, stored without a seal", summary) + } + // The engine activates r2 again itself as the value r2 reads is + // removed after it. + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) || + w.engine.summaryOf() != "reload: secret "+w.registry+" removed" { + t.Errorf("the engine's revisions are %v, the last summarised %q; want r1, the replacement r2, and r2 "+ + "activated again as r3 at the removal of the deleted Secret's value", ids, w.engine.summaryOf()) + } + if w.engine.holds(w.registry) { + t.Error("the store keeps the value of a deleted Secret the replacement lets go") + } + if withdrawn := w.state(t).Withdrawn(); len(withdrawn) != 0 { + t.Errorf("the state owes an activation without %v, want none: r3 read the store without it", withdrawn) + } + + health, healthErr := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, health, healthErr, result) + got := w.run.configApplied + if got == nil || got.Status != metav1.ConditionTrue || got.Reason != crewletaiv1alpha1.ReasonApplied || + !strings.Contains(got.Message, "as revision r2, replacing revision r1, which the engine stores without a seal") { + t.Errorf("ConfigApplied = %+v, want True Applied, saying r2 replaced r1, stored without a seal", got) + } + if w.run.applied.kind != appliedInRevision || w.run.applied.revision != "r3" { + t.Errorf("the parts are %+v, want in revision r3, r2 activated again", w.run.applied) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonRevisionUnreadable+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "active revision r1 was one it stores without a seal") || + !strings.Contains(recorded[i], "revision r2, the rendered document, replaced it") { + t.Errorf("events %q, want a Warning RevisionUnreadable saying r2 replaced r1", recorded) + } + if !told(recorded, "Reactivated") || warned(recorded, "SecretWithdrawalFailed") || told(recorded, "SecretKept") { + t.Errorf("events %q, want r2 activated again, and nothing kept or failed", recorded) + } + + w.again(t, w.statusWritten()) + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: the next reconcile finds it running", written) + } + }) + } +} + +// TestAReplacementOverARevisionWithoutASealKeepsWhatTheNodesRun holds the +// replacement of a revision the engine stores without a seal, under Enforce, +// to what the fleet says its nodes run: having refused that revision, each +// runs the one before it, which the operator reads (GET +// /config/revisions/{id}) and keeps the names of, beside every name the store +// lists, while the rendered document is written over the revision without a +// seal, conditioned on it, all the same. The value of a deleted Secret the +// revision before reads as required stays in the store after the write, for +// as long as a node runs that revision, and goes once the nodes run the +// replacement. Deciding by the fleet's revision alone, the operator would +// delete a value a running node reads at its next apply attempt. +func TestAReplacementOverARevisionWithoutASealKeepsWhatTheNodesRun(t *testing.T) { + ctx := context.Background() + // r1 requires the registry, as the state records, and the rendered + // document lets it go. r2, on the fleet's pointer, carries no seal, so + // the node refused it and runs r1. + w := newWithdrawal(t, runningRequires) + w.engine.mu.Lock() + w.engine.revisions = append(w.engine.revisions, storeRevision{id: w.engine.mint(), unopened: fleetSealMissing}) + w.engine.serving = map[string]string{storeNode: "r1"} + w.engine.mu.Unlock() + w.engine.takeRequests() + + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the rendered document written over a revision without a seal", err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r3 alone, conditioned on r2", written, conditions) + } + if requests := w.engine.takeRequests(); !slices.Contains(requests, "GET /config/revisions/r1") { + t.Errorf("the engine answered %q, want r1, which the node runs, read", requests) + } + if !w.engine.holds(w.registry) { + t.Error("the store lost the value of a deleted Secret r1 requires while a node runs r1") + } + + // The node applies the replacement. + w.engine.mu.Lock() + w.engine.serving = nil + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + if w.engine.holds(w.registry) { + t.Error("the store keeps the value of a deleted Secret once the nodes run the replacement, which lets it go") + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r3"}) { + t.Errorf("the operator wrote %v, want r3 alone: the next reconcile finds the nodes running it", written) + } +} + +// TestAReplacementRefusedOverARevisionWithoutASealKeepsTheStore holds a +// document the engine refuses over a revision it stores without a seal to the +// store as it was and to an activation owed and not asked for. Nothing the +// store holds is deleted: the nodes run, and boot again, the revision before +// the unsealed one, which may read a name the rendered document does not. And +// the engine neither shows nor activates again a revision without a seal, so +// asking for one is refused every time: asked, the reconcile came back every +// ten seconds over a refusal no retry clears, each time with a +// SecretWithdrawalFailed. The write that replaces it, once the document is +// valid, settles what is owed and removes what it lets go. +func TestAReplacementRefusedOverARevisionWithoutASealKeepsTheStore(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 let the registry go, and an earlier sync withdrew it, the + // engine's re-activation at the removal failing: an activation without + // it is owed. + operatorActivates(w) + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + if !slices.Equal(owed.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry", owed.Withdrawn()) + } + // A value the store holds that the rendered document does not + // reference: one the revision the nodes run may. + other := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: "earlier", Key: optionalKey}) + w.engine.mu.Lock() + w.engine.store[other] = "earlier" + w.engine.mu.Unlock() + w.engine.cannotOpen(sealMissing) + w.engine.refusal = invalidDocument + w.engine.takeRequests() + + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: a refused document waits for its objects", result.RequeueAfter) + } + requests := w.engine.takeRequests() + if i := slices.Index(requests, "PUT "+configRoute); i < 0 || + slices.ContainsFunc(requests[i+1:], func(r string) bool { return strings.HasSuffix(r, configRoute) || strings.HasSuffix(r, reloadRoute) }) { + t.Errorf("the engine answered %q, want the dry run, and no read or reload of the document after it", requests) + } + if !w.engine.holds(other) || len(w.engine.ids()) != 1 { + t.Errorf("the store holds %s: %t, and the revisions are %v; want it kept and r1 alone", other, + w.engine.holds(other), w.engine.ids()) + } + if withdrawn := w.state(t).Withdrawn(); !slices.Equal(withdrawn, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry still owed", withdrawn) + } + if recorded := drained(w.events); warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } + + w.again(t, w.statusWritten()) + w.engine.refusal = nil + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r2"}) || !slices.Equal(conditions, []string{"r1"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r2, conditioned on r1", written, conditions) + } + if withdrawn := w.state(t).Withdrawn(); len(withdrawn) != 0 || w.engine.holds(other) { + t.Errorf("the state owes %v and the store holds %s: %t; want r2's activation to settle both", withdrawn, + other, w.engine.holds(other)) + } +} + +// TestAHeldDocumentOverARevisionWithoutASealKeepsTheStore holds a document +// held for a missing Secret, over a revision the engine stores without a seal, +// to what enforce does before the write that replaces it: the store keeps the +// value of a deleted Secret, told as kept for a revision nobody can read. +func TestAHeldDocumentOverARevisionWithoutASealKeepsTheStore(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.cannotOpen(fleetSealMissing) + w.engine.takeRequests() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: the hold's own reason brings the reconcile back", result.RequeueAfter) + } + if requests := w.engine.takeRequests(); slices.Contains(requests, "POST "+reloadRoute) || + slices.ContainsFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) { + t.Errorf("the engine answered %q, want no reload and no deletion", requests) + } + if !w.engine.holds(w.registry) { + t.Error("the store lost the value of a deleted Secret while the nodes run a revision nobody can read") + } + recorded := drained(w.events) + if warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal SecretKept ") }) + if i < 0 || !strings.Contains(recorded[i], "revision r1, the engine's active revision, is stored without a seal") || + strings.Contains(recorded[i], "which the engine runs") { + t.Errorf("events %q, want SecretKept calling r1, stored without a seal, the engine's active revision, which its "+ + "nodes do not run", recorded) + } + // And what SecretsSynced and a part's ResolvedRefs line say of it. + if got := w.run.retainedSentence(); !strings.Contains(got, ": the engine's active revision is stored without a seal") { + t.Errorf("SecretsSynced's sentence is %q, want it calling r1 the engine's active revision", got) + } + registry := secretref.Key{Namespace: "ns", Name: optionalSecret, Key: optionalKey} + if got := w.run.leftOutLines([]leftOutValue{{key: registry, line: registry.String()}}); len(got) != 1 || + !strings.Contains(got[0], "the value it held for its active revision, which is stored without a seal") { + t.Errorf("ResolvedRefs's line is %q, want it calling r1 the engine's active revision", got) + } +} + +// TestAHeldDocumentOverARevisionWithoutASealOwesItsActivation holds a +// document held for a missing Secret, over a revision the engine stores +// without a seal, with an activation owed for a value an earlier sync +// withdrew, to the withdrawal owed and not asked for: the engine neither shows +// nor activates again a revision without a seal, so asking for one is refused +// every time, and the reconcile came back every ten seconds over a refusal no +// retry clears, each time with a SecretWithdrawalFailed. Nor does the +// revision without a seal settle it, although the fleet's pointer names it +// at an epoch above the one the withdrawal was marked by: the nodes run the +// revision before it, which activated before the withdrawal. The write that +// replaces the revision settles it. +func TestAHeldDocumentOverARevisionWithoutASealOwesItsActivation(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 let the registry go, and an earlier sync withdrew it, the + // engine's re-activation at the removal failing: an activation without + // it is owed. + operatorActivates(w) + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + // Marked by r1's epoch, the pointer's when the withdrawal was made. + owed = owed.Observed(1) + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + if !slices.Equal(owed.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry", owed.Withdrawn()) + } + // Then somebody's revision r2, stored without a seal, takes the + // pointer, at epoch 2. + w.engine.mu.Lock() + w.engine.revisions = append(w.engine.revisions, storeRevision{id: w.engine.mint(), document: w.engine.active().document}) + w.engine.mu.Unlock() + w.engine.cannotOpen(fleetSealMissing) + w.engine.takeRequests() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: the hold's own reason brings the reconcile back", result.RequeueAfter) + } + requests := w.engine.takeRequests() + if i := slices.Index(requests, "GET "+configRoute); i < 0 || slices.ContainsFunc(requests[i+1:], func(r string) bool { + return strings.HasSuffix(r, configRoute) || strings.HasSuffix(r, reloadRoute) + }) { + t.Errorf("the engine answered %q, want the one read of the document and no reload after it", requests) + } + if withdrawn := w.state(t).Withdrawn(); !slices.Equal(withdrawn, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry still owed", withdrawn) + } + if recorded := drained(w.events); warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } +} + +// TestUnderEnforceAnUnsealedRevisionAfterTheWriteIsLookedAtSoon holds the one +// report of a revision without a seal under Enforce, one that lands on the +// fleet's pointer after the operator's read (from outside the fleet), to the +// operator's own remedy and its timing: ConfigApplied says the next look +// replaces it, and that look comes ten seconds on, not at the resync interval, +// since nothing but that look is waited for. +func TestUnderEnforceAnUnsealedRevisionAfterTheWriteIsLookedAtSoon(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 is the rendered document, so a rotated value, whose re-activation + // by the engine fails, is what has the operator write it. + w.rotate(t) + operatorActivates(w) + w.engine.afterApply = func(e *storeEngine) { + e.revisions = append(e.revisions, storeRevision{id: e.mint(), unopened: fleetSealMissing}) + } + _, err := w.run.converge(ctx, w.client) + if _, ok := engine.Unsealed(err); !ok { + t.Fatalf("converge() = %v, want the read after the write refused over a revision without a seal", err) + } + result := w.run.engineFailed(ctx, w.client, err) + if result.RequeueAfter != engineRetryInterval { + t.Errorf("the next look is %s on, want %s", result.RequeueAfter, engineRetryInterval) + } + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable || + !strings.Contains(got.Message, "the operator replaces it with the rendered document at its next look") { + t.Errorf("ConfigApplied = %+v, want RevisionUnreadable with the Enforce remedy", got) + } +} + +// TestAReplacementTheEngineCouldNotConfirmIsKept holds a write over a +// revision stored without a seal whose activation the engine could not +// confirm to the record every such write keeps (config_unconfirmed.go): while +// the engine still runs that revision, the write may land, so it is kept, and +// the write sent again on the same condition is recorded beside it; once one +// lands, the reconcile records it as the operator's, reports what it replaced +// as a confirmed replacement is reported, and writes nothing more. Settled +// against no revision, the first would be dropped as one that can never land, +// and its landing read as somebody else's document; confirmed from a record +// that did not keep what it replaced, it was reported as a plain Applied, and +// nothing said a revision nobody could read had gone. +func TestAReplacementTheEngineCouldNotConfirmIsKept(t *testing.T) { + w := newWithdrawal(t, runningNoRecord) + w.engine.cannotOpen(sealMissing) + for range 2 { + w.engine.activation = pending + reconcileOnce(t, w) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonEngineUnavailable || + !strings.Contains(c.Message, "which would replace revision r1, stored without a seal") { + t.Errorf("ConfigApplied is %+v, want EngineUnavailable saying the write would replace r1", c) + } + w.again(t, w.statusWritten()) + } + got := w.run.company.Status.Config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) || len(got) != 2 || + got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || got[1].RevisionID != "r3" || + got[1].BaseRevisionID != "r1" || got[0].Replaced != crewletaiv1alpha1.ReplacedUnsealed { + t.Fatalf("the operator wrote %v and keeps %+v unconfirmed; want r2 and r3, each built on r1, kept, "+ + "recorded as replacing a revision without a seal", written, got) + } + drained(w.events) + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.engine.activation = confirmed + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) { + t.Errorf("the operator wrote %v, want nothing beyond r2 and r3: r2 landed", written) + } + if config := w.statusWritten().Status.Config; config.AppliedHash != got[0].AppliedHash || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r2's write recorded, with its hash, and nothing unconfirmed", config) + } + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue || c.Reason != crewletaiv1alpha1.ReasonApplied || + !strings.Contains(c.Message, "as revision r2, replacing revision r1, which the engine stores without a seal") { + t.Errorf("ConfigApplied is %+v, want True Applied, saying r2 replaced r1, stored without a seal", c) + } + recorded := drained(w.events) + if i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated ") }); i < 0 || + !strings.Contains(recorded[i], "runs revision r2 as epoch") { + t.Errorf("events %q, want an Activated event saying the engine runs r2", recorded) + } + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonRevisionUnreadable+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "active revision r1 was one it stores without a seal") || + !strings.Contains(recorded[i], "revision r2, the rendered document, replaced it") { + t.Errorf("events %q, want a Warning RevisionUnreadable saying r2 replaced r1", recorded) + } +} + +// TestAWriteIsConditionedOnTheRevisionTheReconcileRead holds the operator's +// write to the revision its reconcile read, which its diff, its drift and the +// names it kept in the store were decided against. Another writer's revision +// that lands between that read and the dry run is not written over unseen, nor +// reported as the revision the reconcile read: the reconcile starts over from +// a fresh read, and writes over what it read then. Over a revision the engine +// stores without a seal, that is what keeps the operator from saying it +// replaced one when it replaced another writer's document. +func TestAWriteIsConditionedOnTheRevisionTheReconcileRead(t *testing.T) { + for _, c := range []struct { + name string + // unsealed makes r1 a revision the engine stores without a seal. + unsealed bool + }{ + {name: "the reconcile read the engine's document"}, + {name: "the reconcile read a revision stored without a seal", unsealed: true}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if c.unsealed { + // The operator writes over a revision it cannot read. + w.engine.cannotOpen(sealMissing) + } else { + // r1 is the rendered document, so a rotated value, whose + // re-activation by the engine fails, is what has the + // operator write it. + w.rotate(t) + operatorActivates(w) + } + // Somebody activates another document, which the engine opens, + // after the operator read r1 and before its dry run. + w.engine.beforeCheck = func(e *storeEngine) { e.activateOther(w.requiring) } + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the reconcile started over and the rendered document written", err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r3 alone, conditioned on r2, the revision "+ + "its second read found", written, conditions) + } + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonDriftCorrected || + !strings.HasPrefix(got.Message, "Revision r2, which the operator did not write") { + t.Errorf("ConfigApplied = %+v, want DriftCorrected over r2, the revision the write replaced", got) + } + if summary := w.engine.summaryOfRevision("r3"); strings.Contains(summary, "without a seal") { + t.Errorf("r3 is summarised %q, which says it replaced a revision without a seal; it replaced r2", summary) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want none saying a revision without a seal was replaced", recorded) + } + }) + } +} + +// TestASeatHeldOverAReplacementSaysTheDocumentDoesNotKeepIt holds the SeatHeld +// report on a document replacing a revision stored without a seal to what the +// engine can say of it: the document does not keep those seats as human seats, +// since nobody can read whether the revision it replaces had them, and not that +// it takes them out of the company. +func TestASeatHeldOverAReplacementSaysTheDocumentDoesNotKeepIt(t *testing.T) { + held := asReturned(&engine.SeatHeldError{Held: map[string]engine.SeatHolder{ + seatFounder: {Kind: engine.HolderPerson, Stage: "active"}, + }}) + for _, c := range []struct { + name, unsealed, says, not string + }{ + {name: "a document replacing the engine's", says: "it takes human seats out of the company", + not: "does not keep"}, + {name: "a document replacing a revision without a seal", unsealed: "r1", + says: "it replaces revision r1, which the engine does not open, and does not keep as human seats these " + + "seats its identity directory holds: founder (held by a person, active)", + not: "takes human seats out"}, + } { + t.Run(c.name, func(t *testing.T) { + run, _, _, _ := refusalRun() + run.unsealed = c.unsealed + if !run.seatHeld(held) { + t.Fatal("seatHeld() = false for a *SeatHeldError") + } + if got := run.configApplied.Message; !strings.Contains(got, c.says) || strings.Contains(got, c.not) { + t.Errorf("ConfigApplied says %q; want it saying %q, and not %q", got, c.says, c.not) + } + }) + } +} + // TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt holds the // withdrawal of a deleted Secret's value to the same next look: the engine // cannot take its revision again while its keyring does not open it either, @@ -191,7 +803,7 @@ func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { func TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt(t *testing.T) { ctx := context.Background() w := newWithdrawal(t, runningLetsGo) - w.engine.unreadable = keyringLost + w.engine.cannotOpen(keyringLost) result, err := w.run.withdrawHeld(ctx, w.resolve(t)) if err != nil { t.Fatal(err) diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index f02e4980..d665e3c8 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -58,7 +58,7 @@ import ( // pushes in one request (engine.Client.PutSecrets), one activation for all of // them, and a rotation of a value the running document reads needs no apply // of the same document from the operator, who records the engine's -// re-activation as its own (reloadedBySync). +// re-activation as its own (syncActivation). // // The store serves every revision by that revision's own reading of a key, // though, and the revision the engine runs is the last good one, which keeps @@ -115,7 +115,7 @@ import ( // deletion of a key that one lets go may require the key. The operator // records an activation again, its own or the engine's, only once it has read // back that the revision activated is the document it read (reactivate, -// reloadedBySync), and otherwise reads the revision activated as one whose +// syncActivation), and otherwise reads the revision activated as one whose // reading it does not know, and says so. // withdrawHeld does, for a document this reconcile could not apply, what the @@ -127,7 +127,10 @@ import ( // again if it references one, and says what the store keeps for it. It does // nothing but under the Enforce policy, on an engine that runs, and nothing // where no such reference is absent, no activation is owed, and the store -// keeps nothing it has said it keeps. +// keeps nothing it has said it keeps. Over a revision the engine stores +// without a seal it removes nothing and activates nothing: nobody can read +// what that revision reads, and the write that replaces it once the document +// is no longer held settles what is owed (KeepListed, reactivate). // // An error is the API server's, for the reconcile to retry. One of the // engine's, or one reaching it, is reported (withdrawalFailed) and tried @@ -170,6 +173,16 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) return run.withdrawalFailed(ctx, err), nil } active, err := activeConfig(ctx, eng) + if revision, ok := engine.Unsealed(err); ok { + // A revision stored without a seal, which the rendered document + // replaces once nothing holds it (converge): until then nobody can + // read which values it reads and the nodes run, and boot again, the + // revision before it, so the store keeps everything it lists, as + // enforce keeps it before that write, and no activation can be made + // over it (reactivate). + run.unsealed, err = revision, nil + read.KeepListed(revision) + } if err != nil { return run.withdrawalFailed(ctx, err), nil } @@ -272,7 +285,8 @@ func (run *configRun) keep(read secretsync.Desired, next secretsync.State, retai // The fleet's revision first, which is the one the conditions // have always named, then those pods run beside it. for _, revision := range slices.Sorted(maps.Keys(holders)) { - k := keptRevision{revision: revision, fleets: revision == read.Running(), unknown: !next.Records(revision)} + k := keptRevision{revision: revision, fleets: revision == read.Running(), unknown: !next.Records(revision), + unsealed: run.unsealed != "" && revision == run.unsealed} if run.serving != nil { k.pods = run.serving.revisions[revision] } @@ -297,12 +311,14 @@ func (run *configRun) keep(read secretsync.Desired, next secretsync.State, retai // keptRevision is a revision that keeps a value whose Secret is gone: the // fleet's, or one engine pods run or start on beside it (pods), which reads // it as required, or is one the operator does not know the reading of -// (unknown). +// (unknown), or the fleet's that the engine stores without a seal, which +// nobody can read and its nodes do not run (unsealed). type keptRevision struct { revision string fleets bool pods revisionPods unknown bool + unsealed bool } // tellKept records a Normal SecretKept event naming the names keep found the @@ -330,7 +346,9 @@ func (run *configRun) keptPlainly() bool { // revision the engine runs" otherwise, reads them as required, or is one the // operator does not know the reading of and takes to; and a pod whose // revisions the operator does not know may run or start on one that does. -// them names what they read. +// The fleet's revision, where the engine stores it without a seal, is "the +// engine's active revision" and never the one it runs: nobody can read it, +// and its nodes run the revision before it. them names what they read. func (run *configRun) keptBecause(byID bool, them string) string { var reasons []string for _, k := range run.keptFor { @@ -338,11 +356,20 @@ func (run *configRun) keptBecause(byID bool, them string) string { switch { case !k.fleets: subject = fmt.Sprintf("revision %s, which %s,", k.revision, k.pods.phrase()) + case k.unsealed && byID: + subject = fmt.Sprintf("revision %s, the engine's active revision,", k.revision) + case k.unsealed: + subject = "the engine's active revision" case byID: subject = fmt.Sprintf("revision %s, which the engine runs,", k.revision) } reading := "reads " + them + " as required" - if k.unknown { + switch { + case k.unsealed: + reading = "is stored without a seal and nobody can read it, while the engine's nodes run the revision they " + + "applied before it, which reads the store again whenever a node restarts, so the operator takes it to " + + "read " + them + " as required" + case k.unknown: reading = "is one the operator has no record of activating and does not render, so it takes it to read " + them + " as required" } @@ -541,7 +568,9 @@ func (c storeChanges) made() string { // state does not record, which keeps what it read until the next one runs. A // value written under a name the running revision does not reference stays // owed, for the next write of the document to read. active is the engine's -// document as this reconcile read it, nil to read it here. +// document as this reconcile read it, nil to read it here, and nil with +// run.unsealed set where the engine runs a revision stored without a seal, +// over which nothing is activated and state is returned as it is. // // It is the activation for every store change no write of the document is // coming to settle: a document the engine refused, or one held for a Secret. @@ -575,6 +604,14 @@ func (run *configRun) reactivate( if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { return state, nil } + if active == nil && run.unsealed != "" { + // The engine runs a revision it stores without a seal, which it + // neither shows nor activates again: no activation can be made over + // it, and asking for one is refused every time. The withdrawals stay + // owed, and the write that replaces it settles them, its activation + // reading the store without them (State.Activated). + return state, nil + } if active == nil { var err error if active, err = activeConfig(ctx, eng); err != nil { diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index 7b896560..7ec4d913 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -48,16 +48,18 @@ import ( // operator's client reads them. A write of the document is refused on the // condition it names, as the engine refuses it, and so is every document while // it is to refuse them, as refusal says; written are the revisions the writes -// it took stored, and refused counts the ones it refused on their condition. -// beforeReload, afterReload, beforeWrite and afterApply, when set, run once, -// under its lock, just before a reload or a write is answered and just after -// a reload or an apply is: somebody else's write landing between two of the -// operator's. activation is how the next write's activation goes, and inert -// the revisions writes stored whose activation has not landed +// it took stored, conditions the If-Match each of those was sent with, and +// refused counts the ones it refused on their condition. beforeReload, +// afterReload, beforeCheck, beforeWrite and afterApply, when set, run once, +// under its lock, just before a reload, a dry run or a write is answered and +// just after a reload or an apply is: somebody else's write landing between +// two of the operator's. activation is how the next write's activation goes, +// and inert the revisions writes stored whose activation has not landed // (config_unconfirmed_test.go). minted counts the revisions it has named, -// active or not, so that no two share an id. Its fleet answers the activation -// pointer beside its nodes: the fleet's revision at its place in the history, -// which is its epoch. +// active or not, so that no two share an id, and requests are the requests it +// answered since the last takeRequests, each its method and path. Its fleet +// answers the activation pointer beside its nodes: the fleet's revision at its +// place in the history, which is its epoch. // // A write into its store (PUT /secrets, PUT /secrets/{name}), and a removal // of a value (DELETE /secrets/{name}), re-activates the fleet's revision when its @@ -77,10 +79,9 @@ import ( // (fleetUnreadable), and missing names the revisions GET /config/revisions // does not find. configDown, secretsDown and reloadDown make it answer a read // of the fleet's document, a write to the store, and a reload of the fleet's -// revision, as an engine that cannot decide who is asking does; unreadable, -// when set, gives the body it answers a read of the fleet's document with, as -// an engine whose keyring does not open that revision does -// (config_unreadable_test.go). +// revision, as an engine that cannot decide who is asking does; a revision it +// cannot open answers what reads it as the engine does (unopened, +// config_unreadable_test.go). // // Each node's row says what it starts on as the engine's does: a data node // (dataNode) the revision it runs, and a seat node, whose store is scratch, @@ -88,12 +89,12 @@ import ( // in place of that, as a data node whose record of it the engine has not // moved yet to the revision it runs, "" for none. type storeEngine struct { - mu sync.Mutex - revisions []storeRevision - minted int - written []string - refused int - store map[string]string + mu sync.Mutex + revisions []storeRevision + minted int + written, conditions []string + refused int + store map[string]string // refusal, when set, is how the engine refuses every PUT of the document, // the dry run included: invalidDocument, a document it finds invalid, or // for what only its own users can clear, heldSeat, a human seat somebody @@ -104,6 +105,7 @@ type storeEngine struct { configDown, secretsDown bool reloadDown bool beforeReload, afterReload func(e *storeEngine) + beforeCheck func(e *storeEngine) beforeWrite, afterApply func(e *storeEngine) reloadAnswer *map[string]any afterSecretsReload func(e *storeEngine) @@ -112,12 +114,12 @@ type storeEngine struct { runsNone map[string]bool fleetDown bool missing map[string]bool - unreadable func(revision string) string activation outcome // reloadActivation is how the activation of the next reload goes, as // activation is the next write's. reloadActivation outcome inert []storeRevision + requests []string // beforeConfig and beforeFleet run once, before the engine answers the // next read of the fleet's document or of its fleet: an activation // somebody else makes in that moment. @@ -199,6 +201,10 @@ func fleetUnreadable(w http.ResponseWriter) { type storeRevision struct { id, summary string document []byte + // unopened, when set, is the engine's answer to a request that reads the + // revision, which it cannot open (config_unreadable_test.go): nil for + // one it opens. + unopened func(revision string) string } // newStoreEngine runs the engine with document as its active revision r1. @@ -227,10 +233,15 @@ var ( invalidDocument = &engineRefusal{status: http.StatusBadRequest, body: `{"error":"validation_error",` + `"detail":"refused","problems":[{"path":"roles","kind":"invalid","message":"refused"}]}`} // heldSeat is the engine's refusal of a document that takes away a - // human seat a person holds. + // human seat a person holds, as crewlet#142 words it. heldSeat = &engineRefusal{status: http.StatusConflict, body: `{"error":"seat_held",` + - `"detail":"This change removes human seats that people still hold: founder (held by a person).",` + - `"held":{"founder":{"person":"p-1","kind":"person","stage":"active"}}}`} + `"detail":"This change removes human seats that somebody still holds: founder (held by p-1).",` + + `"held":{"founder":{"person":"p-1","kind":"person","stage":"active"}},` + + `"hint":"Move each person to another human seat (PATCH /iam/people/{person} with seat) or remove them, ` + + `unbind a service account (PATCH /iam/people/{person} with no seat), cancel an open invitation ` + + `(DELETE /iam/invitations/{id}), each an admin's gesture, then send the write again.",` + + `"message":"This change removes a seat someone holds or is invited to. Move or remove them, or cancel ` + + `the invitation, first. The detail says who."}`} // managedDocument is the engine's refusal of a write by a token that // api.auth.company_writers does not name. managedDocument = &engineRefusal{status: http.StatusForbidden, body: `{"error":"config_managed",` + @@ -240,6 +251,7 @@ var ( func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { e.mu.Lock() defer e.mu.Unlock() + e.requests = append(e.requests, r.Method+" "+r.URL.Path) body, _ := io.ReadAll(r.Body) answer := func(status int, v any) { answerJSON(w, status, v) } e.before(r) @@ -261,6 +273,8 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { return } switch { + case active.unopened != nil && readsActive(r, active): + answerRaw(w, http.StatusConflict, active.unopened(active.id)) case r.Method == http.MethodGet && r.URL.Path == "/health": posture := engine.PostureServe if e.appliedLag > 0 { @@ -274,22 +288,8 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { e.readActive(w, active) case r.Method == http.MethodPut && r.URL.Path == configRoute: e.put(w, r, body, answer) - case r.Method == http.MethodPost && r.URL.Path == reloadRoute && e.reloadActivation == pending: - stored := storeRevision{id: e.mint(), document: active.document, summary: r.Header.Get("X-Summary")} - e.inert = append(e.inert, stored) - answerLost(w, stored.id, active.id) case r.Method == http.MethodPost && r.URL.Path == reloadRoute: - reloaded := activate(active.document) - if e.afterReload != nil { - hook := e.afterReload - e.afterReload = nil - hook(e) - } - if e.reloadActivation == landed { - answerLost(w, reloaded.id, active.id) - return - } - activated(reloaded) + e.reload(w, active, r.Header.Get("X-Summary"), activate, activated) case r.Method == http.MethodGet && r.URL.Path == secretsRoute: listing := make([]map[string]string, 0, len(e.store)) for _, name := range slices.Sorted(maps.Keys(e.store)) { @@ -310,6 +310,14 @@ func (e *storeEngine) serveSecretWrite( ) bool { reads := func(name string) bool { return strings.Contains(string(active.document), "${"+name+"}") } name, isSecret := strings.CutPrefix(r.URL.Path, "/secrets/") + if active.unopened != nil && (r.URL.Path == secretsRoute || isSecret) { + // The engine asks which paths of the fleet's revision read each + // name before it writes or removes one (crewlet's secretsapi, + // config.Reads), which opens the revision: over one it cannot + // open, the request fails whole, as an internal error. + answerJSON(w, http.StatusInternalServerError, refusal("internal_error", "Something went wrong on our side.")) + return true + } switch { case r.Method == http.MethodPut && r.URL.Path == secretsRoute: var batch struct { @@ -457,6 +465,12 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st answerJSON(w, http.StatusNotFound, refusal("not_found", "no revision "+revisionID)) return true } + if unopened := e.revisions[i].unopened; unopened != nil { + // The engine opens the revision to answer it, and refuses one + // it cannot open as it refuses a read of the document. + answerRaw(w, http.StatusConflict, unopened(e.revisions[i].id)) + return true + } payload, err := yaml.YAMLToJSON(e.revisions[i].document) if err != nil { answerJSON(w, http.StatusInternalServerError, refusal("internal", err.Error())) @@ -470,6 +484,32 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st return true } +// reload answers a request, summarised as summary, to activate active again as +// reloadActivation says it goes: stored and never activated (pending), +// activated with the answer lost (landed), or activated. +func (e *storeEngine) reload( + w http.ResponseWriter, active storeRevision, summary string, activate func([]byte) storeRevision, + activated func(storeRevision), +) { + if e.reloadActivation == pending { + stored := storeRevision{id: e.mint(), document: active.document, summary: summary} + e.inert = append(e.inert, stored) + answerLost(w, stored.id, active.id) + return + } + reloaded := activate(active.document) + if e.afterReload != nil { + hook := e.afterReload + e.afterReload = nil + hook(e) + } + if e.reloadActivation == landed { + answerLost(w, reloaded.id, active.id) + return + } + activated(reloaded) +} + // before runs the hook, if one is set, that r's route takes before the engine // answers it, once: beforeReload, beforeConfig or beforeFleet. func (e *storeEngine) before(r *http.Request) { @@ -526,13 +566,8 @@ func refusal(code, detail string) map[string]any { } // readActive answers a read of the fleet's document, active: the document and -// its ETag, or, where unreadable is set, the engine's refusal of a revision -// its keyring does not open. +// its ETag. func (e *storeEngine) readActive(w http.ResponseWriter, active storeRevision) { - if e.unreadable != nil { - answerRaw(w, http.StatusConflict, e.unreadable(active.id)) - return - } w.Header().Set("ETag", `"`+active.id+`"`) _, _ = w.Write(active.document) } @@ -564,6 +599,10 @@ func (e *storeEngine) put(w http.ResponseWriter, r *http.Request, body []byte, a return } if dryRun(r) { + if hook := e.beforeCheck; hook != nil { + e.beforeCheck = nil + hook(e) + } answer(http.StatusOK, map[string]any{"base_revision_id": e.active().id}) return } @@ -592,6 +631,7 @@ func (e *storeEngine) put(w http.ResponseWriter, r *http.Request, body []byte, a } revision := storeRevision{id: e.mint(), document: body, summary: r.Header.Get("X-Summary")} e.written = append(e.written, revision.id) + e.conditions = append(e.conditions, strings.Trim(r.Header.Get("If-Match"), `"`)) switch e.activation { case confirmed: e.revisions = append(e.revisions, revision) @@ -624,6 +664,24 @@ func (e *storeEngine) writes() ([]string, int) { return slices.Clone(e.written), e.refused } +// takeRequests returns the requests the engine answered since the last +// takeRequests, each its method and path. +func (e *storeEngine) takeRequests() []string { + e.mu.Lock() + defer e.mu.Unlock() + requests := e.requests + e.requests = nil + return requests +} + +// writtenOn is the If-Match each write the engine stored was sent with, as +// writes names them, "" for none. +func (e *storeEngine) writtenOn() []string { + e.mu.Lock() + defer e.mu.Unlock() + return slices.Clone(e.conditions) +} + // ids are the engine's revisions, oldest first. func (e *storeEngine) ids() []string { e.mu.Lock() @@ -642,6 +700,18 @@ func (e *storeEngine) summaryOf() string { return e.revisions[len(e.revisions)-1].summary } +// summaryOfRevision is the summary of revision id, "" for none. +func (e *storeEngine) summaryOfRevision(id string) string { + e.mu.Lock() + defer e.mu.Unlock() + for _, r := range e.revisions { + if r.id == id { + return r.summary + } + } + return "" +} + func (e *storeEngine) holds(name string) bool { e.mu.Lock() defer e.mu.Unlock() @@ -2212,7 +2282,7 @@ func TestAnActivationNoNodeCanOpenSettlesNothing(t *testing.T) { w.engine.mu.Lock() w.engine.beforeFleet = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) - e.unreadable = keyringLost + e.revisions[len(e.revisions)-1].unopened = keyringLost } w.engine.mu.Unlock() if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { diff --git a/internal/engine/client.go b/internal/engine/client.go index 2559a43b..c8987e35 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -280,7 +280,8 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { // (a seat pod's does not: it starts on nothing and applies the fleet's // revision). Before the first apply it fails with ErrNoActiveRevision, and // on an engine that cannot open the fleet's revision with -// ErrRevisionUnreadable, which names it. +// ErrRevisionUnreadable, which names it (Unsealed reads the one a whole +// document replaces). // // An engine that cannot read the activation pointer from its coordination // store answers 503 with a Retry-After, which is ErrUnavailable: "the fleet @@ -365,13 +366,17 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // writer, with ErrRevisionAdvanced (a 412 when req.IfMatch is set, a 409 when // it is not); req.CreateOnly on a configured engine with ErrAlreadyConfigured; // a write while the engine cannot open the fleet's revision, which the write -// reads first, with ErrRevisionUnreadable; and a write whose -// activation the engine sent and could not confirm with ErrUnavailable naming -// the revision it stored, which UnconfirmedActivation reads. The fleet may run -// that revision or may not: the caller keeps it until a read of the fleet's -// activation (Fleet.Activation) says which, and sends the write again, on the -// condition it was first sent on, only once that read shows the fleet still -// runs the revision the write was built on. +// reads first, with ErrRevisionUnreadable, except over one stored without a +// seal (Unsealed), which the write (a dry run included) replaces, reading +// nothing from it: that revision stays the write's base, so a req.IfMatch is +// judged against it as against any other, and a write without one replaces +// it unconditionally; and a write whose activation the engine sent and could +// not confirm with ErrUnavailable naming the revision it stored, which +// UnconfirmedActivation reads. The fleet may run that revision or may not: the +// caller keeps it until a read of the fleet's activation (Fleet.Activation) +// says which, and sends the write again, on the condition it was first sent +// on, only once that read shows the fleet still runs the revision the write +// was built on. func (c *Client) ApplyConfig(ctx context.Context, req ApplyRequest) (ApplyResult, error) { if req.IfMatch != "" && req.CreateOnly { return ApplyResult{}, errors.New("apply the company document: IfMatch and CreateOnly are mutually exclusive: " + diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 1488f1de..69a54b8b 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -592,8 +592,10 @@ func TestErrorMapping(t *testing.T) { // sentence (httpjson.Sentence): the code's own sentence, the revision, what // happened as the detail, and the remedy as the hint. A seal the engine's // build does not read is one code for either origin, told apart by its detail -// and its hint. +// and its hint. Each names refusedRevision. const ( + refusedRevision = "rev-4" + unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + `"revision_id":"rev-4","detail":"This node's keyring doesn't hold the key that revision is sealed under.",` + @@ -651,7 +653,7 @@ func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { if err := json.Unmarshal([]byte(body), &want); err != nil { t.Fatal(err) } - if apiErr.RevisionID != "rev-4" || apiErr.Message != want.Detail || apiErr.Hint != want.Hint || + if apiErr.RevisionID != refusedRevision || apiErr.Message != want.Detail || apiErr.Hint != want.Hint || apiErr.RetryAfter != 0 { t.Errorf("APIError = {revision %q, message %q, hint %q, retry after %s}, want {rev-4, the detail, "+ "the hint, 0}", apiErr.RevisionID, apiErr.Message, apiErr.Hint, apiErr.RetryAfter) @@ -659,6 +661,54 @@ func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { } } +// TestUnsealedIsTheOneRevisionAWriteReplaces holds Unsealed to the one cause +// the engine lets a whole document replace: a revision stored without a seal, +// this node's copy or the fleet's, named by the refusal. A revision sealed +// under a key the keyring lacks, or in a form the engine's build does not +// read, is somebody's document, which the engine refuses every write over; +// read as unsealed, the operator would send a write the engine refuses, or, +// were the engine to take it, discard that document unread. A refusal that +// names no revision is no base to condition a write on. +func TestUnsealedIsTheOneRevisionAWriteReplaces(t *testing.T) { + for _, c := range []struct { + name, body string + unsealed bool + }{ + {"this node's copy without a seal", unsealedRevision, true}, + {"the fleet's revision without a seal", unsealedFleetRevision, true}, + {"a key the keyring lost", unreadableRevision, false}, + {"a key the fleet seals under that this node lacks", unreadableFleetRevision, false}, + {"this node's copy sealed in a form its build does not read", unknownSealRevision, false}, + {"the fleet's revision sealed in a form this build does not read", unknownSealFleetRevision, false}, + {"a revision without a seal the engine does not name", + strings.Replace(unsealedRevision, `"revision_id":"rev-4",`, "", 1), false}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) + f.handle("GET /config", answer(http.StatusConflict, c.body)) + _, err := f.client().ActiveConfig(context.Background()) + if !errors.Is(err, engine.ErrRevisionUnreadable) { + t.Fatalf("ActiveConfig() = %v, want ErrRevisionUnreadable", err) + } + revision, ok := engine.Unsealed(err) + want := "" + if c.unsealed { + want = refusedRevision + } + if ok != c.unsealed || revision != want { + t.Errorf("Unsealed() = %q, %t; want %q, %t", revision, ok, want, c.unsealed) + } + }) + } + for _, err := range []error{nil, errors.New("dial tcp: connection refused"), + &engine.APIError{StatusCode: http.StatusConflict, RevisionID: refusedRevision}} { + if revision, ok := engine.Unsealed(err); ok { + t.Errorf("Unsealed(%v) = %q, true; want false: it is no refusal of a revision without a seal", err, revision) + } + } +} + // TestRevisionAdvancedNamesTheCurrentRevision reads both revisions a lost // compare-and-set names, from the answer the operator meets: its writes are // conditioned on the revision they were built on, so the engine refuses one @@ -839,8 +889,8 @@ const ( func TestSeatHeldNamesEachHolderButNotTheirLogin(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", answer(http.StatusConflict, `{"detail":"This change removes human seats that `+ - `people still hold: ana (held by an open invitation (inv-1)); founder (held by jane.doe@nimbus.example); `+ - `release (held by service account svc:release).",`+ + `somebody still holds: ana (held by an open invitation (inv-1)), founder (held by jane.doe@nimbus.example) `+ + `and release (held by service account svc:release).",`+ `"error":"seat_held",`+ `"held":{"ana":{"invitation":"inv-1","expires_at":"2026-10-23T09:00:00Z"},`+ `"founder":{"person":"p-1","kind":"person","login":"jane.doe@nimbus.example","stage":"active"},`+ diff --git a/internal/engine/errors.go b/internal/engine/errors.go index aec39f82..2f8ce45d 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -95,9 +95,11 @@ var ( // Retry-After: no wait opens the revision. A sealed one is somebody's // document, which no write replaces, since that would discard it unread: // what opens it is given to the engine, a key put in its keyring, which - // it reads when it starts, or the build that sealed it. The *APIError - // behind it carries the revision in RevisionID, the engine's account as - // its Message and the remedy as its Hint. + // it reads when it starts, or the build that sealed it. One without a + // seal no key opens, and the one write that goes on over it is a whole + // document (ApplyConfig), which reads nothing from it: Unsealed reports + // it. The *APIError behind it carries the revision in RevisionID, the + // engine's account as its Message and the remedy as its Hint. ErrRevisionUnreadable = errors.New("the engine cannot open a revision of its company document") // ErrNotReady is an engine that is up but should not be sent work: it @@ -133,12 +135,15 @@ var codesForbidden = []string{"unauthorized", "token_refused", "seat_unavailable // put back in the keyring (unreadable_revision), a key the fleet seals under // given to this node (unreadable_fleet_revision), the build that sealed it // (unknown_seal_revision, one code for either origin, since that remedy is -// the same for both), or the document brought back sealed -// (unsealed_revision, unsealed_fleet_revision). -var codesRevisionUnreadable = []string{ +// the same for both), or a whole document replacing it (codesUnsealed). +var codesRevisionUnreadable = append([]string{ "unreadable_revision", "unreadable_fleet_revision", "unknown_seal_revision", - "unsealed_revision", "unsealed_fleet_revision", -} +}, codesUnsealed...) + +// codesUnsealed are the refusals of a revision stored without a seal, this +// node's copy (unsealed_revision) or the fleet's (unsealed_fleet_revision): +// the one cause a write replaces (Unsealed). +var codesUnsealed = []string{"unsealed_revision", "unsealed_fleet_revision"} // codesWithProblems are the refusals of a document: the engine located each // failure in it, so they become a *ValidationError. @@ -280,6 +285,32 @@ func (e *APIError) Is(target error) bool { return false } +// Unsealed is the engine's active revision, one it stores without a seal, and +// true, when err is the engine's refusal to open it (ErrRevisionUnreadable +// under unsealed_revision or unsealed_fleet_revision, naming the revision). +// The fleet's pointer names it, and no node runs it: each runs the revision it +// applied before. +// +// It is the one revision the engine cannot open that a write replaces: a +// whole document sent with ApplyConfig goes on over it, reading nothing from +// it, so no masked value is restored from it and no field carried, and the +// engine judges the write as one replacing nothing. The revision is still the +// write's base: a dry run names it as BaseRevisionID, and IfMatch is judged +// against it. Every other write is built from the active document and is +// refused over it as a read is. A revision sealed in any way the engine cannot +// open (unreadable_revision, unreadable_fleet_revision, +// unknown_seal_revision) is refused for every write, ApplyConfig included, +// since replacing it would discard a document somebody sealed, and is not +// reported here. Nor is a refusal that names no revision, on which no write +// can be conditioned; the engine always names one. +func Unsealed(err error) (string, bool) { + var apiErr *APIError + if !errors.As(err, &apiErr) || !slices.Contains(codesUnsealed, apiErr.Code) || apiErr.RevisionID == "" { + return "", false + } + return apiErr.RevisionID, true +} + // UnconfirmedActivation is the revision a config write stored whose activation // the engine sent to its fleet and could not confirm, and true, when err is // that answer: a 503 naming the revision it stored (ErrUnavailable with a diff --git a/internal/engine/types.go b/internal/engine/types.go index 638020a8..91d6dd3d 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -264,8 +264,10 @@ type ApplyResult struct { Epoch int64 // BaseRevisionID is, for a dry run, the revision the check was built on, - // empty when nothing was active. A caller that dry-runs and then applies - // with IfMatch set to it applies exactly what it checked. + // empty when nothing was active; one stored without a seal, which the + // write replaces having read nothing from it (Unsealed), included. A + // caller that dry-runs and then applies with IfMatch set to it applies + // exactly what it checked. BaseRevisionID string // Warnings are what the engine will run but a person should know about, diff --git a/internal/secretsync/resolve.go b/internal/secretsync/resolve.go index b44c268f..2fa4fa93 100644 --- a/internal/secretsync/resolve.go +++ b/internal/secretsync/resolve.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crewlet/crewlet-operator/internal/engine" "github.com/crewlet/crewlet-operator/internal/secretref" ) @@ -99,6 +100,13 @@ type Desired struct { // and no value, so Sync refuses it rather than delete every entry it // does not push. partial bool + // keepsListed is whether running, which KeepListed named then, is a + // revision nobody can read, for which every owned name the store lists + // is kept, each read as required (withListed). A bool rather than the + // revision's id beside running, and beside partial, which it shares a + // word with, keeps a Desired, a value every reconcile copies, inside the + // size a value parameter is held to. + keepsListed bool } // Names returns the names Desired pushes, sorted. @@ -192,7 +200,28 @@ func (d *Desired) keep(revision string, names []string) { d.kept = kept } -// Running is the revision Keep named, empty for none. +// KeepListed is Keep for a revision whose document nobody can read, one the +// engine stores without a seal: revision is its id, and every name the store +// lists that Owns accepts is kept, so Sync and Withdraw delete none that +// Desired pushes no value for, an absent name (Missing) and a released one +// (Release) included, whatever the State they are given records of the +// revision's reading, while Sync still pushes every value Desired holds. +// +// Which names such a revision references is what nobody can read, and it is +// not what the engine's nodes run: each runs the revision it applied before, +// which it reads again from the store when it restarts, so not even a record +// of how this revision reads a name says what they need. A name deleted while +// such a revision is the active one is a value a node may start without at +// its next boot, read as required by a revision the operator cannot see. The +// store stays as it is until a revision somebody can read runs, which the +// write that replaces this one is; the sync after that write removes what it +// lets go. +func (d *Desired) KeepListed(revision string) { + d.running = revision + d.keepsListed = true +} + +// Running is the revision Keep or KeepListed named, empty for none. func (d Desired) Running() string { return d.running } @@ -214,13 +243,30 @@ func (d Desired) Unread() []string { return slices.Sorted(maps.Keys(d.unread)) } -// Holders are the revisions kept (Keep, KeepServed) that reference name and -// read it as required, as prev records their reading (State.LetGo), sorted: -// those that keep an absent name in the store. +// Holders are the revisions kept (Keep, KeepServed, KeepListed) that +// reference name, an absent name the store holds (Result.Retained), and read +// it as required, as prev records their reading (State.LetGo), sorted: those +// that keep it in the store. The revision KeepListed named keeps every name +// the store lists that Owns accepts, and reads each as required whatever prev +// records, since nobody can read how it reads one, so it holds every such +// name. func (d Desired) Holders(name string, prev State) []string { + out := d.holders(name, prev) + if d.keepsListed && Owns(name) && !slices.Contains(out, d.running) { + out = append(out, d.running) + slices.Sort(out) + } + return out +} + +// holders is Holders by what d keeps alone: the revision KeepListed named +// holds only the names withListed kept for it, those the store lists, so that +// a name it no longer lists, which a run that stopped before saving removed, +// is not taken for one it keeps. +func (d Desired) holders(name string, prev State) []string { var out []string for revision, names := range d.kept { - if _, ok := names[name]; ok && !prev.letsGo(revision, name) { + if _, ok := names[name]; ok && (d.unreadable(revision) || !prev.letsGo(revision, name)) { out = append(out, revision) } } @@ -228,6 +274,29 @@ func (d Desired) Holders(name string, prev State) []string { return out } +// unreadable reports whether revision is the one KeepListed named, whose +// document nobody can read. +func (d Desired) unreadable(revision string) bool { + return d.keepsListed && revision == d.running +} + +// withListed is d with every name listed that Owns accepts kept for the +// revision KeepListed named, where it named one, and d itself otherwise. It +// copies what it changes (keep), so the caller's Desired keeps what it was. +func (d Desired) withListed(listed []engine.SecretInfo) Desired { + if !d.keepsListed { + return d + } + names := make([]string, 0, len(listed)) + for _, info := range listed { + if Owns(info.Name) { + names = append(names, info.Name) + } + } + d.keep(d.running, names) + return d +} + // Release names the references the revision the engine runs lets go without // whose Secret or key does not exist, as Absent reports them of // State.LetGoKeys of that revision: Sync and Withdraw remove the entry of @@ -253,11 +322,12 @@ func (d *Desired) Release(absent ...*MissingError) { } // retains reports whether name, absent, stays in the store: a revision kept -// references it (Keep, KeepServed) and reads it as required, which it does -// unless prev records it as letting the name go without (State.LetGo), or a -// node runs a revision the operator does not know (KeepUnread). +// references it (Keep, KeepServed, KeepListed) and reads it as required, +// which it does unless prev records it as letting the name go without +// (State.LetGo), or a node runs a revision the operator does not know +// (KeepUnread). func (d Desired) retains(name string, prev State) bool { - return len(d.unread) > 0 || len(d.Holders(name, prev)) > 0 + return len(d.unread) > 0 || len(d.holders(name, prev)) > 0 } // keeps reports whether a revision kept references name, or a node runs a @@ -276,10 +346,11 @@ func (d Desired) keeps(name string) bool { // letGo reports whether a revision kept references name and, as prev // records its reading, lets it go: what makes a released name one to remove, -// where no other revision kept retains it. +// where no other revision kept retains it. The revision KeepListed named lets +// nothing go: nobody can read how it reads a name, whatever prev records. func (d Desired) letGo(name string, prev State) bool { for revision, names := range d.kept { - if _, ok := names[name]; ok && prev.letsGo(revision, name) { + if _, ok := names[name]; ok && !d.unreadable(revision) && prev.letsGo(revision, name) { return true } } diff --git a/internal/secretsync/sync.go b/internal/secretsync/sync.go index 37eb075d..4cd3c5eb 100644 --- a/internal/secretsync/sync.go +++ b/internal/secretsync/sync.go @@ -120,7 +120,9 @@ type Overwrite struct { // a value somebody else put there for its own and never notice. Then it // deletes every name the engine lists that Owns accepts and desired neither // pushes nor keeps, every absent name (Desired.Missing) but one a revision -// kept reads as required (Desired.Keep, Desired.KeepServed), which it leaves +// kept reads as required (Desired.Keep, Desired.KeepServed, and every name the +// store lists for a revision nobody can read, Desired.KeepListed), which it +// leaves // (Result.Retained, State.Retained), and every name a revision kept lets go // whose Secret or key is gone (Desired.Release) that no other revision kept // reads as required; it records each absent or released name it deleted as @@ -184,6 +186,7 @@ func Sync( if err != nil { return prev, res, fmt.Errorf("listing the engine's secret store: %w", err) } + desired = desired.withListed(listing) listed := make(map[string]engine.SecretInfo, len(listing)) for _, info := range listing { listed[info.Name] = info @@ -444,6 +447,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat if err != nil { return prev, res, fmt.Errorf("listing the engine's secret store: %w", err) } + desired = desired.withListed(listing) next := prev.carried(maps.Clone(prev.fingerprints)) listed := make(map[string]bool, len(listing)) var removing []string diff --git a/internal/secretsync/sync_test.go b/internal/secretsync/sync_test.go index eb07b15f..0c1f6b03 100644 --- a/internal/secretsync/sync_test.go +++ b/internal/secretsync/sync_test.go @@ -589,6 +589,85 @@ func TestSyncKeepsWhatTheRunningDocumentReferences(t *testing.T) { } } +// TestKeepListedDeletesNothingTheStoreLists: over a revision nobody can read +// (one the engine stores without a seal), Sync and Withdraw keep every owned +// name the store lists, whatever the next document reads of it and whatever +// the State records of the revision's reading: an absent name, a released +// one whose Secret is gone, and one nothing references. Which of them the +// nodes need is what nobody can read, since they run, and boot again, the +// revision before it. Values are still pushed, a foreign entry is left alone +// as always, and the caller's Desired keeps none of the listing, yet names +// the revision as what keeps each absent name the store retained (Holders), +// which is how the operator says why it keeps one. +func TestKeepListedDeletesNothingTheStoreLists(t *testing.T) { + ctx := context.Background() + keyD := secretref.Key{Namespace: "ns", Name: "d", Key: tokenKey} + nameD := StoreName(keyD) + eng := newFakeEngine() + eng.entries[foreignName] = fakeEntry{value: foreignValue, by: access} + // The State records revision2 letting nameB and nameC go: the engine + // stored it, as the operator activated it, without a seal. + state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1", nameB: "b", nameC: "c", nameD: "d"}), + State{}.Activated(revision2, []secretref.Key{keyB, keyC})) + eng.takeCalls() + + next := desiredOf(map[string]string{nameA: "a2"}) + next.absent = map[string]*MissingError{nameB: {Key: keyB, Reason: SecretNotFound}} + next.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + next.Keep(revision2) + next.KeepListed(revision2) + synced, res, err := Sync(ctx, eng, next, state, self, discard) + if err != nil { + t.Fatalf("Sync: %v", err) + } + assertCalls(t, eng, "list", "put "+nameA) + assertStore(t, eng, map[string]fakeEntry{ + nameA: {value: "a2", by: self}, nameB: {value: "b", by: self}, nameC: {value: "c", by: self}, + nameD: {value: "d", by: self}, foreignName: {value: foreignValue, by: access}, + }) + if len(res.Deleted) != 0 || !slices.Equal(res.Retained, []string{nameB}) || len(synced.Withdrawn()) != 0 { + t.Errorf("Deleted = %v, Retained = %v and Withdrawn() = %v; want none, [%s] and none", + res.Deleted, res.Retained, synced.Withdrawn(), nameB) + } + if len(next.kept[revision2]) != 0 { + t.Errorf("the caller's Desired keeps %v for %s after Sync, want what it was given, nothing", + next.kept[revision2], revision2) + } + if got := next.Holders(nameB, synced); !slices.Equal(got, []string{revision2}) { + t.Errorf("Holders(%s) = %v after Sync, want [%s], which the State records letting it go", nameB, got, revision2) + } + + held := Desired{partial: true, absent: map[string]*MissingError{ + nameB: {Key: keyB, Reason: SecretNotFound}, nameD: {Key: keyD, Reason: KeyNotFound}, + }} + held.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + held.KeepListed(revision2) + withdrawn, res, err := Withdraw(ctx, eng, held, synced) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + assertCalls(t, eng, "list") + if len(res.Deleted) != 0 || !slices.Equal(res.Retained, []string{nameB, nameD}) || len(withdrawn.Withdrawn()) != 0 { + t.Errorf("Deleted = %v, Retained = %v and Withdrawn() = %v; want none, [%s %s] and none", + res.Deleted, res.Retained, withdrawn.Withdrawn(), nameB, nameD) + } + if got := held.Holders(nameD, withdrawn); !slices.Equal(got, []string{revision2}) { + t.Errorf("Holders(%s) = %v after Withdraw, want [%s]", nameD, got, revision2) + } + + // Kept as a revision whose document names nothing, the same Desired + // deletes all three: nameB and nameC as revision2 lets them go, and + // nameD as nothing references it. + plain := desiredOf(map[string]string{nameA: "a2"}) + plain.absent = next.absent + plain.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + plain.Keep(revision2) + _, res = mustSync(t, eng, plain, synced) + if !slices.Equal(res.Deleted, []string{nameB, nameC, nameD}) { + t.Errorf("Deleted = %v without KeepListed, want [%s %s %s]", res.Deleted, nameB, nameC, nameD) + } +} + // TestSyncWritesBeforeItDeletes: a document switch must find every name it // references written before anything it no longer references is removed. func TestSyncWritesBeforeItDeletes(t *testing.T) {