diff --git a/Makefile b/Makefile index 0d627fbd..cc225458 100644 --- a/Makefile +++ b/Makefile @@ -85,12 +85,16 @@ gateway-api-crds: ## Copy the Gateway API CRDs of the release go.mod requires in # test/docs) runs the crewlet binary `make crewlet` installed # (internal/engine/enginetest), named explicitly so a CREWLET_BIN in the # caller's environment cannot swap in another engine; test/chart renders the -# chart with the pinned helm. +# chart with the pinned helm. The timeout is per test binary and sized to the +# longest: internal/controller, whose envtest suite runs every config spec +# against a real engine and takes about eight minutes on a 4-CPU runner under +# the race detector, which go test's default of ten would fail on any slower +# one. Twenty still stops a hung suite. .PHONY: test test: manifests generate fmt vet setup-envtest crewlet helm ## Run the unit, envtest, engine and chart suites of both modules. KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" \ CREWLET_BIN="$(CREWLET)" HELM="$(HELM)" \ - go test -race $$(go list ./... | grep -v /test/e2e) -coverprofile cover.out + go test -race -timeout 20m $$(go list ./... | grep -v /test/e2e) -coverprofile cover.out cd api && go test -race ./... -coverprofile cover.out KIND_CLUSTER ?= crewlet-operator-test-e2e diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 8aa2350d..2522e47f 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -483,6 +483,28 @@ const ( // what to change. ReasonAddressRefused = "AddressRefused" + // ReasonDataVolumeTooSmall is RuntimeReady=False when the company's data + // volume gives the engine an event stream (three quarters of it, Crewlet's + // stream.store_max_bytes) under what a fresh node of Crewlet reserves on + // it: its nine state logs at their floors, 8120172544 bytes, and with the + // files in the stream's own bucket that bucket beside them, 10826896725 + // bytes in all. The EngineClass API admits no data.storage.size under + // 11Gi, or 14Gi with the files in the bucket, so what reaches this is a + // volume claimed under an older floor: a running company's, whose claim + // template cannot change, or one a class stored before the floor rose + // stamps on a new company. Crewlet refuses to boot on such a stream + // wherever it has a state log to create, a fresh node's every log and the + // chat log an engine adds to a stream made before it, so the operator + // holds the pods back as they are rather than roll an engine that cannot + // boot; an engine whose image has already moved still cannot run on the + // volume, and what the hold changes is that the cause is named. The + // message names the volume, the stream it gives, what a fresh node + // reserves and the least volume that holds it, and what clears it: + // growing the claim (docs/concepts/runtime.md, "Growing a company's data + // volume"), or, for a new company with no claim yet, raising its class's + // data.storage.size, at which the operator then creates its volume. + ReasonDataVolumeTooSmall = "DataVolumeTooSmall" + // ReasonNoAlarms is EngineHealthy=True: the engine raises no alarm. ReasonNoAlarms = "NoAlarms" @@ -635,6 +657,29 @@ const ( // still exists and is held: one whose kind turned from Human to Agent. ReasonSeatHeld = "SeatHeld" + // ReasonRevisionUnreadable is ConfigApplied=False when the engine cannot + // ReasonRevisionUnreadable is ConfigApplied=False when the engine cannot + // open the fleet's revision of the company document (409 + // unreadable_revision or unsealed_revision, or crewlet#142's + // unreadable_fleet_revision, unknown_seal_revision or + // unsealed_fleet_revision): it is sealed under a key the keyring does not + // hold, sealed in a form the engine's build does not read, or stored + // without a seal. The engine then refuses to show the document and to take + // any write built from it, and no wait changes that. A sealed revision is + // replaced by no write, so the operator writes nothing, and somebody has + // to give the engine what opens it: the key put back in the Secret + // -keyring, or the build that sealed it. A revision without a + // seal is replaced by a whole document, which reads nothing from it: under + // Enforce the operator replaces it with the rendered document as it reads + // it, recording a Warning event of this reason, and reports this there + // only for one that lands after that read, until its next look replaces + // it, ten seconds on; while the rendered document is held, the hold's own + // reason is reported and the revision stays. Under Seed and Observe, which + // write no such document, this is reported for as long as the engine runs + // it. The message names the revision and carries the engine's account and + // remedy. + ReasonRevisionUnreadable = "RevisionUnreadable" + // ReasonInvalid is Applied=False (or ConfigApplied=False) when the // engine's dry run refuses the rendered document; the message carries // the engine's own words. diff --git a/api/v1alpha1/company_types.go b/api/v1alpha1/company_types.go index f65492ac..267f3c07 100644 --- a/api/v1alpha1/company_types.go +++ b/api/v1alpha1/company_types.go @@ -111,6 +111,75 @@ type KnowledgeSettings struct { Vectors *bool `json:"vectors,omitempty"` } +// ChatBackend is which chat surface a company runs. It maps onto Crewlet's +// chat.backend (Chatlet is `chatlet`, Vendor is `vendor`, None is `none`). +// +kubebuilder:validation:Enum=Chatlet;Vendor;None +type ChatBackend string + +// The chat backends. +const ( + // ChatChatlet is Chatlet, Crewlet's own chat: rooms the engine keeps, a + // room for each unit's channel, read on the dashboard. + ChatChatlet ChatBackend = "Chatlet" + // ChatVendor is the Slack or Mattermost the company's Integrations (or + // a seat's Slack app) connect it to. + ChatVendor ChatBackend = "Vendor" + // ChatNone is a company with no chat. + ChatNone ChatBackend = "None" +) + +// ChatletSettings is the policy of Chatlet. It maps onto Crewlet's +// chat.chatlet. +// +kubebuilder:validation:MinProperties=1 +type ChatletSettings struct { + // messageRetentionDays is how long a message lives before the engine + // prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + // days, since under a month somebody away could miss a decision before + // reading it. Unset takes Crewlet's default of 365. Crewlet's + // `message_retention_days`. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=3650 + // +kubebuilder:validation:XValidation:rule="self == 0 || self >= 30",message="messageRetentionDays must be 0, which keeps messages for ever, or 30 to 3650" + MessageRetentionDays *int32 `json:"messageRetentionDays,omitempty"` + + // defaultChannelPrivate makes a room created without a stated + // visibility private; unset, such a room is public, readable by the + // company's seats. A room created with a visibility keeps it. Crewlet's + // `default_channel_private`. + // +optional + DefaultChannelPrivate *bool `json:"defaultChannelPrivate,omitempty"` + + // typingStatus is when a seat shows in a room that it is working on a + // message. Unset takes Crewlet's default, Always. Crewlet's + // `typing_status`. + // +optional + TypingStatus WorkingStatus `json:"typingStatus,omitempty"` +} + +// ChatSettings is which chat surface a company runs and the policy of +// Chatlet. It maps onto Crewlet's chat block. +// +kubebuilder:validation:XValidation:rule="!has(self.chatlet) || !has(self.backend) || self.backend == 'Chatlet'",message="chatlet applies to backend Chatlet only, or to no backend, which Crewlet derives Chatlet for a company with no Slack or Mattermost" +// +kubebuilder:validation:MinProperties=1 +type ChatSettings struct { + // backend is which chat surface the company runs. Unset derives it, as + // Crewlet does: Vendor for a company with a Slack or Mattermost + // Integration or a seat with a Slack app, Chatlet for every other. So + // there is no default here: a Chatlet written beside such an + // Integration, or a Vendor without one, is refused by the engine's dry + // run, reported on the Company, since a company's conversations cannot + // live in two places. + // +optional + Backend ChatBackend `json:"backend,omitempty"` + + // chatlet is the policy of Chatlet: for backend Chatlet, or for no + // backend on a company Crewlet derives Chatlet for. On a company whose + // chat is a vendor's nothing would read it, so the engine's dry run + // refuses it there, reported on the Company. + // +optional + Chatlet ChatletSettings `json:"chatlet,omitzero"` +} + // LLMProviderKind is the kind of model provider a reference names. // +kubebuilder:validation:Enum=LLMProvider;PlatformLLMProvider type LLMProviderKind string @@ -1201,6 +1270,11 @@ type CompanySpec struct { // +optional Knowledge KnowledgeSettings `json:"knowledge,omitzero"` + // chat is which chat surface the company runs: Chatlet, Crewlet's own, + // unless it is connected to Slack or Mattermost. Crewlet's `chat`. + // +optional + Chat ChatSettings `json:"chat,omitzero"` + // tokenBudget caps the tokens all seats together may spend per window. // Each window binds every agent seat, so Crewlet refuses one below // turn_engine.executor_min_budget_tokens (25000 unless spec.engineConfig @@ -1237,27 +1311,26 @@ type CompanySpec struct { // engineConfig is passed through into the company document as written, // for the engine's tuning blocks that have no field here: turn_engine, - // learning, scheduling, workers, skill_variables and the + // learning, scheduling, workers, skill_variables, artifacts and the // notification_* settings, spelled as Crewlet spells them. Each entry is // one top-level key of the document and its value. The operator does not // interpret it; the engine validates it in the dry run before every // apply, and a refusal is reported on the Company. // // It may not set what another field or object of this API owns: name, - // mission, vision, policies, timezone, tracker, knowledge, token_budget, - // assistant, providers, integrations, mcp_servers, roles, units or - // seat_access_migrated, which the operator writes with every - // integration kind so the engine acts on each Seat's - // spec.integrations. Every - // position Crewlet holds a credential in lives under one of those keys, so - // engineConfig can never carry a credential: the API server refuses a - // Company whose engineConfig names one of them, before anything is - // stored. + // mission, vision, policies, timezone, tracker, knowledge, chat, + // token_budget, assistant, providers, integrations, mcp_servers, roles, + // units or seat_access_migrated, which the operator writes with every + // integration kind so the engine acts on each Seat's spec.integrations. + // Every position Crewlet holds a credential in lives under one of those + // keys, so engineConfig can never carry a credential: the API server + // refuses a Company whose engineConfig names one of them, before anything + // is stored. // +optional // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=64 - // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated']))",message="engineConfig may not set name, mission, vision, policies, timezone, tracker, knowledge, token_budget, assistant, providers, integrations, mcp_servers, roles, units or seat_access_migrated: those are fields and objects of this API, among them every block that holds a credential" + // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated']))",message="engineConfig may not set name, mission, vision, policies, timezone, tracker, knowledge, token_budget, assistant, providers, integrations, mcp_servers, roles, units or seat_access_migrated: those are fields and objects of this API, among them every block that holds a credential" EngineConfig []EngineConfigEntry `json:"engineConfig,omitempty"` // config is how the operator manages the company's document. @@ -1451,16 +1524,21 @@ type ConfigStatus struct { Epoch *int64 `json:"epoch,omitempty"` // appliedHash is the hash of the rendered document and of the values - // of the Secret keys it references, as last applied: a change to - // either is applied as a new revision, since the engine reads a - // rotated value only when a revision activates. The values enter it - // only as keyed fingerprints, so it reveals nothing about them. + // of the Secret keys it references, as the revision recorded runs + // them: the operator's last write, or a revision it found running that + // document after every value it wrote. The values enter it only as + // keyed fingerprints, so it reveals nothing about them. // +optional // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=128 AppliedHash string `json:"appliedHash,omitempty"` - // appliedAt is when the operator last applied a revision. + // appliedAt is when the operator activated the revision this records, + // or found the engine running it where it adopted a revision of the + // rendered document it did not write (a reload), or a write of its own + // whose activation the engine could not confirm (unconfirmed), which may + // have landed any time since it was written: what the operator's wait + // for the engine to apply its epoch counts from. // +optional AppliedAt *metav1.Time `json:"appliedAt,omitempty"` @@ -1468,6 +1546,45 @@ type ConfigStatus struct { // +optional Objects AppliedObjectCounts `json:"objects,omitzero"` + // unconfirmed are the revisions of the rendered document the operator + // wrote, which the engine stored, and whose activation the engine sent + // to its fleet and could not confirm (a 503 naming the revision it + // stored): the fleet may run one of them or none. Before it writes + // again, the operator reads which revision the engine runs. If it runs + // one of these, that write landed: status.config records it as the + // operator's write, as though the engine had confirmed it. If it runs the + // revision one of them was built on, that one has not landed and still + // may, so it stays, and the write is sent again on the same condition, + // which the engine refuses once one of them lands. Any other can no + // longer land, and goes, to overtaken where it was written into an + // engine that ran no document. A reload the operator asks for, to + // activate a revision of its own again, is recorded here too when the + // engine stores it and cannot confirm activating it, as a write of that + // revision's document, whether or not a read after it could say which + // revision runs. They are oldest first, and a seventeenth drops the + // oldest. + // +optional + // +listType=map + // +listMapKey=revisionID + // +kubebuilder:validation:MaxItems=16 + Unconfirmed []UnconfirmedRevision `json:"unconfirmed,omitempty"` + + // overtaken are writes from unconfirmed into an engine that ran no + // document that another revision the engine activated has overtaken. + // None of them can land any more, and under Seed one of them is taken as + // the seed whatever revision runs now: the engine stored it while it had + // none, so either it landed and was replaced since, or another writer's + // create reached the engine first, which nothing the engine says tells + // apart. Each is kept until a reconcile decides with it, and they go + // once status.config records a revision of the rendered document in + // sync, a write of it, or the seed. They are oldest first, and a + // seventeenth drops the oldest. + // +optional + // +listType=map + // +listMapKey=revisionID + // +kubebuilder:validation:MaxItems=16 + Overtaken []UnconfirmedRevision `json:"overtaken,omitempty"` + // placement is where the revision the engine runs pins its agent seats, // read from the engine's document under every management policy. The // operator keeps a seat pool's pods running while the running revision @@ -1480,6 +1597,93 @@ type ConfigStatus struct { Placement RevisionPlacement `json:"placement,omitzero"` } +// UnconfirmedRevision is a write of the company document whose activation the +// engine could not confirm. +type UnconfirmedRevision struct { + // revisionID is the revision the engine stored for the write. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + RevisionID string `json:"revisionID,omitempty"` + + // baseRevisionID is the revision the write was built on, which its + // activation was conditioned on. Empty for a write into an engine that + // ran no document, conditioned on there being none. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + BaseRevisionID string `json:"baseRevisionID,omitempty"` + + // appliedHash is the appliedHash of the write: the rendered document's + // and its values' hash as they were when it was written, which + // status.config.appliedHash records once the engine is seen running the + // revision. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + AppliedHash string `json:"appliedHash,omitempty"` + + // writtenAt is when the operator wrote it. A write the engine is seen + // running is recorded as applied when the operator found it, since + // nothing says when its activation landed; a seed taken from overtaken, + // which nothing waits on, as applied when it was written. + // +required + WrittenAt *metav1.Time `json:"writtenAt,omitempty"` + + // objects counts the objects the revision was rendered from, which + // status.config.objects records once the engine is seen running it. + // +optional + Objects AppliedObjectCounts `json:"objects,omitzero"` + + // replaced says what the write replaced where that was more than the + // operator's own document, which ConfigApplied and a Warning event report + // once the engine is seen running the revision, as they report a write the + // engine confirmed: Unsealed, a revision the engine stores without a seal, + // which the write replaces reading nothing from it; Drifted, a revision + // the operator did not write whose document differed from the rendered + // one, at driftedAt. Unset for any other base, and for a reload: its base + // is a write of the operator's, and where that write is recorded here too, + // the revision found running reports what that write replaced, once, + // whichever of the two it is. + // +optional + Replaced ReplacedRevision `json:"replaced,omitempty"` + + // driftedAt names where the Drifted revision the write replaced differed + // from the rendered document, as ConfigApplied names it: up to five + // places, and how many more there were. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=32768 + DriftedAt string `json:"driftedAt,omitempty"` + + // documentSHA256 is set for a reload: the SHA-256, in hex, of the + // document of the revision it activates again, as the engine answered + // it. The engine activates again whichever revision is active when the + // request lands, so the revision it stored is the operator's only where + // its document is that one; one found running another document was a + // reload of a revision somebody activated in between, and goes. + // +optional + // +kubebuilder:validation:MinLength=64 + // +kubebuilder:validation:MaxLength=64 + // +kubebuilder:validation:Pattern=`^[0-9a-f]{64}$` + DocumentSHA256 string `json:"documentSHA256,omitempty"` +} + +// ReplacedRevision is what a write of the company document replaced where +// that was more than the operator's own document. +// +kubebuilder:validation:Enum=Unsealed;Drifted +type ReplacedRevision string + +const ( + // ReplacedUnsealed is a revision the engine stores without a seal, which + // nobody can read and a whole document replaces. + ReplacedUnsealed ReplacedRevision = "Unsealed" + + // ReplacedDrifted is a revision the operator did not write whose + // document differed from the rendered one. + ReplacedDrifted ReplacedRevision = "Drifted" +) + // RevisionPlacement is where a revision of the company document pins its // agent seats: the values of the two placement labels its roles name. Only // values that can name one of the company's pods are recorded: a pool name diff --git a/api/v1alpha1/integration_types.go b/api/v1alpha1/integration_types.go index 164558da..8acf2fb5 100644 --- a/api/v1alpha1/integration_types.go +++ b/api/v1alpha1/integration_types.go @@ -16,7 +16,12 @@ const ( // WorkingStatusAlways shows it on every turn a chat message started. WorkingStatusAlways WorkingStatus = "Always" // WorkingStatusAddressed shows it only when a person is plausibly - // waiting: a direct message, a mention, or a thread the seat follows. + // waiting on the seat: a direct message, a mention of it by name, and + // on Slack or Mattermost a thread it is in because it was named there, + // on Chatlet a reply in a thread it started or a person's message in a + // unit's room that reached nobody else and so falls to it as the unit's + // lead. Never a message it reached through a room-wide address, or only + // by having spoken in the thread before. WorkingStatusAddressed WorkingStatus = "Addressed" ) diff --git a/api/v1alpha1/seat_types.go b/api/v1alpha1/seat_types.go index 6525dc18..30fa3aa0 100644 --- a/api/v1alpha1/seat_types.go +++ b/api/v1alpha1/seat_types.go @@ -973,14 +973,19 @@ type SeatSpec struct { // +kubebuilder:validation:Pattern=`^[A-Z][A-Z0-9]{1,9}$` Space string `json:"space,omitempty"` - // contact is how agents reach the person in a Human seat on a chat - // surface, and how what the person does there reaches the engine as - // theirs. It applies to Human seats only, and is optional: a person who - // works only through the engine's dashboard has no account to name, and - // signs in on the seat through the engine's own identity directory. The - // engine warns of a Human seat without one, since nothing addressed to it - // on a chat surface reaches anybody, and the operator records the warning - // as an EngineWarnings event on the Company. + // contact is how agents reach the person in a Human seat on a vendor's + // chat surface or tool, and how what the person does there reaches the + // engine as theirs. It applies to Human seats only, and is optional: a + // person who works only through the engine's dashboard has no account to + // name, and signs in on the seat through the engine's own identity + // directory. On a company that talks on Slack or Mattermost (an + // Integration of either, or a seat's Slack app) the engine warns of a + // Human seat with no account there, a slackUserID or a + // mattermostUsername, since nothing addressed to it there reaches + // anybody; a GitHub, GitLab or Atlassian account does not count, and the + // operator records the warning as an EngineWarnings event on the + // Company. On Chatlet, which a company with neither talks on, the person + // reads the rooms on the dashboard as their seat, and nothing is warned. // +optional Contact SeatContact `json:"contact,omitzero"` diff --git a/api/v1alpha1/unit_types.go b/api/v1alpha1/unit_types.go index bdd3afcb..68f48d06 100644 --- a/api/v1alpha1/unit_types.go +++ b/api/v1alpha1/unit_types.go @@ -156,7 +156,9 @@ type UnitSpec struct { // whose members follow the org chart. Crewlet takes lowercase letters, // digits, - and _, starting with a letter or a digit, up to 64 // characters: the name itself, never #platform or a vendor's channel id. - // Crewlet's `channel`. + // On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + // naming the same channel; a child unit that should share its parent's + // room names none. Crewlet's `channel`. // +optional // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=64 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 160059d0..ca8f87b3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -131,6 +131,47 @@ func (in *CLIAgentAuth) DeepCopy() *CLIAgentAuth { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChatSettings) DeepCopyInto(out *ChatSettings) { + *out = *in + in.Chatlet.DeepCopyInto(&out.Chatlet) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChatSettings. +func (in *ChatSettings) DeepCopy() *ChatSettings { + if in == nil { + return nil + } + out := new(ChatSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChatletSettings) DeepCopyInto(out *ChatletSettings) { + *out = *in + if in.MessageRetentionDays != nil { + in, out := &in.MessageRetentionDays, &out.MessageRetentionDays + *out = new(int32) + **out = **in + } + if in.DefaultChannelPrivate != nil { + in, out := &in.DefaultChannelPrivate, &out.DefaultChannelPrivate + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChatletSettings. +func (in *ChatletSettings) DeepCopy() *ChatletSettings { + if in == nil { + return nil + } + out := new(ChatletSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClassEnvVar) DeepCopyInto(out *ClassEnvVar) { *out = *in @@ -387,6 +428,7 @@ func (in *CompanySpec) DeepCopyInto(out *CompanySpec) { } out.Tracker = in.Tracker in.Knowledge.DeepCopyInto(&out.Knowledge) + in.Chat.DeepCopyInto(&out.Chat) out.TokenBudget = in.TokenBudget in.LLM.DeepCopyInto(&out.LLM) in.Assistant.DeepCopyInto(&out.Assistant) @@ -500,6 +542,20 @@ func (in *ConfigStatus) DeepCopyInto(out *ConfigStatus) { *out = (*in).DeepCopy() } in.Objects.DeepCopyInto(&out.Objects) + if in.Unconfirmed != nil { + in, out := &in.Unconfirmed, &out.Unconfirmed + *out = make([]UnconfirmedRevision, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Overtaken != nil { + in, out := &in.Overtaken, &out.Overtaken + *out = make([]UnconfirmedRevision, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Placement.DeepCopyInto(&out.Placement) } @@ -3341,6 +3397,26 @@ func (in *TrackerSettings) DeepCopy() *TrackerSettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UnconfirmedRevision) DeepCopyInto(out *UnconfirmedRevision) { + *out = *in + if in.WrittenAt != nil { + in, out := &in.WrittenAt, &out.WrittenAt + *out = (*in).DeepCopy() + } + in.Objects.DeepCopyInto(&out.Objects) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnconfirmedRevision. +func (in *UnconfirmedRevision) DeepCopy() *UnconfirmedRevision { + if in == nil { + return nil + } + out := new(UnconfirmedRevision) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Unit) DeepCopyInto(out *Unit) { *out = *in diff --git a/config/crd/bases/crewlet.ai_companies.yaml b/config/crd/bases/crewlet.ai_companies.yaml index 047a7951..86a906f5 100644 --- a/config/crd/bases/crewlet.ai_companies.yaml +++ b/config/crd/bases/crewlet.ai_companies.yaml @@ -240,6 +240,72 @@ spec: it runs on, one whose webTools is Denied' rule: '!has(self.enabled) || !self.enabled || (has(self.llm) && size(self.llm) > 0)' + chat: + description: |- + chat is which chat surface the company runs: Chatlet, Crewlet's own, + unless it is connected to Slack or Mattermost. Crewlet's `chat`. + minProperties: 1 + properties: + backend: + description: |- + backend is which chat surface the company runs. Unset derives it, as + Crewlet does: Vendor for a company with a Slack or Mattermost + Integration or a seat with a Slack app, Chatlet for every other. So + there is no default here: a Chatlet written beside such an + Integration, or a Vendor without one, is refused by the engine's dry + run, reported on the Company, since a company's conversations cannot + live in two places. + enum: + - Chatlet + - Vendor + - None + type: string + chatlet: + description: |- + chatlet is the policy of Chatlet: for backend Chatlet, or for no + backend on a company Crewlet derives Chatlet for. On a company whose + chat is a vendor's nothing would read it, so the engine's dry run + refuses it there, reported on the Company. + minProperties: 1 + properties: + defaultChannelPrivate: + description: |- + defaultChannelPrivate makes a room created without a stated + visibility private; unset, such a room is public, readable by the + company's seats. A room created with a visibility keeps it. Crewlet's + `default_channel_private`. + type: boolean + messageRetentionDays: + description: |- + messageRetentionDays is how long a message lives before the engine + prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + days, since under a month somebody away could miss a decision before + reading it. Unset takes Crewlet's default of 365. Crewlet's + `message_retention_days`. + format: int32 + maximum: 3650 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: messageRetentionDays must be 0, which keeps messages + for ever, or 30 to 3650 + rule: self == 0 || self >= 30 + typingStatus: + description: |- + typingStatus is when a seat shows in a room that it is working on a + message. Unset takes Crewlet's default, Always. Crewlet's + `typing_status`. + enum: + - Always + - Addressed + type: string + type: object + type: object + x-kubernetes-validations: + - message: chatlet applies to backend Chatlet only, or to no backend, + which Crewlet derives Chatlet for a company with no Slack or Mattermost + rule: '!has(self.chatlet) || !has(self.backend) || self.backend + == ''Chatlet''' companyName: description: |- companyName is the company's name. Crewlet derives every seat's @@ -409,22 +475,21 @@ spec: description: |- engineConfig is passed through into the company document as written, for the engine's tuning blocks that have no field here: turn_engine, - learning, scheduling, workers, skill_variables and the + learning, scheduling, workers, skill_variables, artifacts and the notification_* settings, spelled as Crewlet spells them. Each entry is one top-level key of the document and its value. The operator does not interpret it; the engine validates it in the dry run before every apply, and a refusal is reported on the Company. It may not set what another field or object of this API owns: name, - mission, vision, policies, timezone, tracker, knowledge, token_budget, - assistant, providers, integrations, mcp_servers, roles, units or - seat_access_migrated, which the operator writes with every - integration kind so the engine acts on each Seat's - spec.integrations. Every - position Crewlet holds a credential in lives under one of those keys, so - engineConfig can never carry a credential: the API server refuses a - Company whose engineConfig names one of them, before anything is - stored. + mission, vision, policies, timezone, tracker, knowledge, chat, + token_budget, assistant, providers, integrations, mcp_servers, roles, + units or seat_access_migrated, which the operator writes with every + integration kind so the engine acts on each Seat's spec.integrations. + Every position Crewlet holds a credential in lives under one of those + keys, so engineConfig can never carry a credential: the API server + refuses a Company whose engineConfig names one of them, before anything + is stored. items: description: |- EngineConfigEntry is one top-level key of the company document that @@ -458,7 +523,7 @@ spec: those are fields and objects of this API, among them every block that holds a credential' rule: self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', - 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', + 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated'])) exposure: @@ -1730,16 +1795,22 @@ spec: minProperties: 1 properties: appliedAt: - description: appliedAt is when the operator last applied a revision. + description: |- + appliedAt is when the operator activated the revision this records, + or found the engine running it where it adopted a revision of the + rendered document it did not write (a reload), or a write of its own + whose activation the engine could not confirm (unconfirmed), which may + have landed any time since it was written: what the operator's wait + for the engine to apply its epoch counts from. format: date-time type: string appliedHash: description: |- appliedHash is the hash of the rendered document and of the values - of the Secret keys it references, as last applied: a change to - either is applied as a new revision, since the engine reads a - rotated value only when a revision activates. The values enter it - only as keyed fingerprints, so it reveals nothing about them. + of the Secret keys it references, as the revision recorded runs + them: the operator's last write, or a revision it found running that + document after every value it wrote. The values enter it only as + keyed fingerprints, so it reveals nothing about them. maxLength: 128 minLength: 1 type: string @@ -1789,6 +1860,139 @@ spec: minimum: 0 type: integer type: object + overtaken: + description: |- + overtaken are writes from unconfirmed into an engine that ran no + document that another revision the engine activated has overtaken. + None of them can land any more, and under Seed one of them is taken as + the seed whatever revision runs now: the engine stored it while it had + none, so either it landed and was replaced since, or another writer's + create reached the engine first, which nothing the engine says tells + apart. Each is kept until a reconcile decides with it, and they go + once status.config records a revision of the rendered document in + sync, a write of it, or the seed. They are oldest first, and a + seventeenth drops the oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. + enum: + - Unsealed + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map placement: description: |- placement is where the revision the engine runs pins its agent seats, @@ -1839,6 +2043,146 @@ spec: maxLength: 128 minLength: 1 type: string + unconfirmed: + description: |- + unconfirmed are the revisions of the rendered document the operator + wrote, which the engine stored, and whose activation the engine sent + to its fleet and could not confirm (a 503 naming the revision it + stored): the fleet may run one of them or none. Before it writes + again, the operator reads which revision the engine runs. If it runs + one of these, that write landed: status.config records it as the + operator's write, as though the engine had confirmed it. If it runs the + revision one of them was built on, that one has not landed and still + may, so it stays, and the write is sent again on the same condition, + which the engine refuses once one of them lands. Any other can no + longer land, and goes, to overtaken where it was written into an + engine that ran no document. A reload the operator asks for, to + activate a revision of its own again, is recorded here too when the + engine stores it and cannot confirm activating it, as a write of that + revision's document, whether or not a read after it could say which + revision runs. They are oldest first, and a seventeenth drops the + oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. + enum: + - Unsealed + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map type: object endpoints: description: endpoints is where the company is reached. diff --git a/config/crd/bases/crewlet.ai_seats.yaml b/config/crd/bases/crewlet.ai_seats.yaml index 96afad5a..a4248a67 100644 --- a/config/crd/bases/crewlet.ai_seats.yaml +++ b/config/crd/bases/crewlet.ai_seats.yaml @@ -124,14 +124,19 @@ spec: rule: self == oldSelf contact: description: |- - contact is how agents reach the person in a Human seat on a chat - surface, and how what the person does there reaches the engine as - theirs. It applies to Human seats only, and is optional: a person who - works only through the engine's dashboard has no account to name, and - signs in on the seat through the engine's own identity directory. The - engine warns of a Human seat without one, since nothing addressed to it - on a chat surface reaches anybody, and the operator records the warning - as an EngineWarnings event on the Company. + contact is how agents reach the person in a Human seat on a vendor's + chat surface or tool, and how what the person does there reaches the + engine as theirs. It applies to Human seats only, and is optional: a + person who works only through the engine's dashboard has no account to + name, and signs in on the seat through the engine's own identity + directory. On a company that talks on Slack or Mattermost (an + Integration of either, or a seat's Slack app) the engine warns of a + Human seat with no account there, a slackUserID or a + mattermostUsername, since nothing addressed to it there reaches + anybody; a GitHub, GitLab or Atlassian account does not count, and the + operator records the warning as an EngineWarnings event on the + Company. On Chatlet, which a company with neither talks on, the person + reads the rooms on the dashboard as their seat, and nothing is warned. minProperties: 1 properties: atlassianAccount: diff --git a/config/crd/bases/crewlet.ai_units.yaml b/config/crd/bases/crewlet.ai_units.yaml index 18735e6e..c2b86a09 100644 --- a/config/crd/bases/crewlet.ai_units.yaml +++ b/config/crd/bases/crewlet.ai_units.yaml @@ -75,7 +75,9 @@ spec: whose members follow the org chart. Crewlet takes lowercase letters, digits, - and _, starting with a letter or a digit, up to 64 characters: the name itself, never #platform or a vendor's channel id. - Crewlet's `channel`. + On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + naming the same channel; a child unit that should share its parent's + room names none. Crewlet's `channel`. maxLength: 64 minLength: 1 pattern: ^[a-z0-9][a-z0-9_-]{0,63}$ diff --git a/dist/chart/templates/crd/companies.crewlet.ai.yaml b/dist/chart/templates/crd/companies.crewlet.ai.yaml index 5fd22561..6d984914 100644 --- a/dist/chart/templates/crd/companies.crewlet.ai.yaml +++ b/dist/chart/templates/crd/companies.crewlet.ai.yaml @@ -243,6 +243,72 @@ spec: it runs on, one whose webTools is Denied' rule: '!has(self.enabled) || !self.enabled || (has(self.llm) && size(self.llm) > 0)' + chat: + description: |- + chat is which chat surface the company runs: Chatlet, Crewlet's own, + unless it is connected to Slack or Mattermost. Crewlet's `chat`. + minProperties: 1 + properties: + backend: + description: |- + backend is which chat surface the company runs. Unset derives it, as + Crewlet does: Vendor for a company with a Slack or Mattermost + Integration or a seat with a Slack app, Chatlet for every other. So + there is no default here: a Chatlet written beside such an + Integration, or a Vendor without one, is refused by the engine's dry + run, reported on the Company, since a company's conversations cannot + live in two places. + enum: + - Chatlet + - Vendor + - None + type: string + chatlet: + description: |- + chatlet is the policy of Chatlet: for backend Chatlet, or for no + backend on a company Crewlet derives Chatlet for. On a company whose + chat is a vendor's nothing would read it, so the engine's dry run + refuses it there, reported on the Company. + minProperties: 1 + properties: + defaultChannelPrivate: + description: |- + defaultChannelPrivate makes a room created without a stated + visibility private; unset, such a room is public, readable by the + company's seats. A room created with a visibility keeps it. Crewlet's + `default_channel_private`. + type: boolean + messageRetentionDays: + description: |- + messageRetentionDays is how long a message lives before the engine + prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650 + days, since under a month somebody away could miss a decision before + reading it. Unset takes Crewlet's default of 365. Crewlet's + `message_retention_days`. + format: int32 + maximum: 3650 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: messageRetentionDays must be 0, which keeps messages + for ever, or 30 to 3650 + rule: self == 0 || self >= 30 + typingStatus: + description: |- + typingStatus is when a seat shows in a room that it is working on a + message. Unset takes Crewlet's default, Always. Crewlet's + `typing_status`. + enum: + - Always + - Addressed + type: string + type: object + type: object + x-kubernetes-validations: + - message: chatlet applies to backend Chatlet only, or to no backend, + which Crewlet derives Chatlet for a company with no Slack or Mattermost + rule: '!has(self.chatlet) || !has(self.backend) || self.backend + == ''Chatlet''' companyName: description: |- companyName is the company's name. Crewlet derives every seat's @@ -412,22 +478,21 @@ spec: description: |- engineConfig is passed through into the company document as written, for the engine's tuning blocks that have no field here: turn_engine, - learning, scheduling, workers, skill_variables and the + learning, scheduling, workers, skill_variables, artifacts and the notification_* settings, spelled as Crewlet spells them. Each entry is one top-level key of the document and its value. The operator does not interpret it; the engine validates it in the dry run before every apply, and a refusal is reported on the Company. It may not set what another field or object of this API owns: name, - mission, vision, policies, timezone, tracker, knowledge, token_budget, - assistant, providers, integrations, mcp_servers, roles, units or - seat_access_migrated, which the operator writes with every - integration kind so the engine acts on each Seat's - spec.integrations. Every - position Crewlet holds a credential in lives under one of those keys, so - engineConfig can never carry a credential: the API server refuses a - Company whose engineConfig names one of them, before anything is - stored. + mission, vision, policies, timezone, tracker, knowledge, chat, + token_budget, assistant, providers, integrations, mcp_servers, roles, + units or seat_access_migrated, which the operator writes with every + integration kind so the engine acts on each Seat's spec.integrations. + Every position Crewlet holds a credential in lives under one of those + keys, so engineConfig can never carry a credential: the API server + refuses a Company whose engineConfig names one of them, before anything + is stored. items: description: |- EngineConfigEntry is one top-level key of the company document that @@ -461,7 +526,7 @@ spec: those are fields and objects of this API, among them every block that holds a credential' rule: self.all(e, !(e.name in ['name', 'mission', 'vision', 'policies', - 'timezone', 'tracker', 'knowledge', 'token_budget', 'assistant', + 'timezone', 'tracker', 'knowledge', 'chat', 'token_budget', 'assistant', 'providers', 'integrations', 'mcp_servers', 'roles', 'units', 'seat_access_migrated'])) exposure: @@ -1733,16 +1798,22 @@ spec: minProperties: 1 properties: appliedAt: - description: appliedAt is when the operator last applied a revision. + description: |- + appliedAt is when the operator activated the revision this records, + or found the engine running it where it adopted a revision of the + rendered document it did not write (a reload), or a write of its own + whose activation the engine could not confirm (unconfirmed), which may + have landed any time since it was written: what the operator's wait + for the engine to apply its epoch counts from. format: date-time type: string appliedHash: description: |- appliedHash is the hash of the rendered document and of the values - of the Secret keys it references, as last applied: a change to - either is applied as a new revision, since the engine reads a - rotated value only when a revision activates. The values enter it - only as keyed fingerprints, so it reveals nothing about them. + of the Secret keys it references, as the revision recorded runs + them: the operator's last write, or a revision it found running that + document after every value it wrote. The values enter it only as + keyed fingerprints, so it reveals nothing about them. maxLength: 128 minLength: 1 type: string @@ -1792,6 +1863,139 @@ spec: minimum: 0 type: integer type: object + overtaken: + description: |- + overtaken are writes from unconfirmed into an engine that ran no + document that another revision the engine activated has overtaken. + None of them can land any more, and under Seed one of them is taken as + the seed whatever revision runs now: the engine stored it while it had + none, so either it landed and was replaced since, or another writer's + create reached the engine first, which nothing the engine says tells + apart. Each is kept until a reconcile decides with it, and they go + once status.config records a revision of the rendered document in + sync, a write of it, or the seed. They are oldest first, and a + seventeenth drops the oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. + enum: + - Unsealed + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map placement: description: |- placement is where the revision the engine runs pins its agent seats, @@ -1842,6 +2046,146 @@ spec: maxLength: 128 minLength: 1 type: string + unconfirmed: + description: |- + unconfirmed are the revisions of the rendered document the operator + wrote, which the engine stored, and whose activation the engine sent + to its fleet and could not confirm (a 503 naming the revision it + stored): the fleet may run one of them or none. Before it writes + again, the operator reads which revision the engine runs. If it runs + one of these, that write landed: status.config records it as the + operator's write, as though the engine had confirmed it. If it runs the + revision one of them was built on, that one has not landed and still + may, so it stays, and the write is sent again on the same condition, + which the engine refuses once one of them lands. Any other can no + longer land, and goes, to overtaken where it was written into an + engine that ran no document. A reload the operator asks for, to + activate a revision of its own again, is recorded here too when the + engine stores it and cannot confirm activating it, as a write of that + revision's document, whether or not a read after it could say which + revision runs. They are oldest first, and a seventeenth drops the + oldest. + items: + description: |- + UnconfirmedRevision is a write of the company document whose activation the + engine could not confirm. + properties: + appliedHash: + description: |- + appliedHash is the appliedHash of the write: the rendered document's + and its values' hash as they were when it was written, which + status.config.appliedHash records once the engine is seen running the + revision. + maxLength: 128 + minLength: 1 + type: string + baseRevisionID: + description: |- + baseRevisionID is the revision the write was built on, which its + activation was conditioned on. Empty for a write into an engine that + ran no document, conditioned on there being none. + maxLength: 128 + minLength: 1 + type: string + documentSHA256: + description: |- + documentSHA256 is set for a reload: the SHA-256, in hex, of the + document of the revision it activates again, as the engine answered + it. The engine activates again whichever revision is active when the + request lands, so the revision it stored is the operator's only where + its document is that one; one found running another document was a + reload of a revision somebody activated in between, and goes. + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-f]{64}$ + type: string + driftedAt: + description: |- + driftedAt names where the Drifted revision the write replaced differed + from the rendered document, as ConfigApplied names it: up to five + places, and how many more there were. + maxLength: 32768 + minLength: 1 + type: string + objects: + description: |- + objects counts the objects the revision was rendered from, which + status.config.objects records once the engine is seen running it. + minProperties: 1 + properties: + integrations: + description: integrations is the number of Integrations. + format: int32 + minimum: 0 + type: integer + llmProviders: + description: |- + llmProviders is the number of model providers, platform providers + included. + format: int32 + minimum: 0 + type: integer + mcpServers: + description: mcpServers is the number of MCPServers. + format: int32 + minimum: 0 + type: integer + schedules: + description: schedules is the number of Schedules. + format: int32 + minimum: 0 + type: integer + seats: + description: seats is the number of Seats. + format: int32 + minimum: 0 + type: integer + units: + description: units is the number of Units. + format: int32 + minimum: 0 + type: integer + type: object + replaced: + description: |- + replaced says what the write replaced where that was more than the + operator's own document, which ConfigApplied and a Warning event report + once the engine is seen running the revision, as they report a write the + engine confirmed: Unsealed, a revision the engine stores without a seal, + which the write replaces reading nothing from it; Drifted, a revision + the operator did not write whose document differed from the rendered + one, at driftedAt. Unset for any other base, and for a reload: its base + is a write of the operator's, and where that write is recorded here too, + the revision found running reports what that write replaced, once, + whichever of the two it is. + enum: + - Unsealed + - Drifted + type: string + revisionID: + description: revisionID is the revision the engine stored + for the write. + maxLength: 128 + minLength: 1 + type: string + writtenAt: + description: |- + writtenAt is when the operator wrote it. A write the engine is seen + running is recorded as applied when the operator found it, since + nothing says when its activation landed; a seed taken from overtaken, + which nothing waits on, as applied when it was written. + format: date-time + type: string + required: + - appliedHash + - revisionID + - writtenAt + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - revisionID + x-kubernetes-list-type: map type: object endpoints: description: endpoints is where the company is reached. diff --git a/dist/chart/templates/crd/seats.crewlet.ai.yaml b/dist/chart/templates/crd/seats.crewlet.ai.yaml index 28cd51e0..08cd2ade 100644 --- a/dist/chart/templates/crd/seats.crewlet.ai.yaml +++ b/dist/chart/templates/crd/seats.crewlet.ai.yaml @@ -127,14 +127,19 @@ spec: rule: self == oldSelf contact: description: |- - contact is how agents reach the person in a Human seat on a chat - surface, and how what the person does there reaches the engine as - theirs. It applies to Human seats only, and is optional: a person who - works only through the engine's dashboard has no account to name, and - signs in on the seat through the engine's own identity directory. The - engine warns of a Human seat without one, since nothing addressed to it - on a chat surface reaches anybody, and the operator records the warning - as an EngineWarnings event on the Company. + contact is how agents reach the person in a Human seat on a vendor's + chat surface or tool, and how what the person does there reaches the + engine as theirs. It applies to Human seats only, and is optional: a + person who works only through the engine's dashboard has no account to + name, and signs in on the seat through the engine's own identity + directory. On a company that talks on Slack or Mattermost (an + Integration of either, or a seat's Slack app) the engine warns of a + Human seat with no account there, a slackUserID or a + mattermostUsername, since nothing addressed to it there reaches + anybody; a GitHub, GitLab or Atlassian account does not count, and the + operator records the warning as an EngineWarnings event on the + Company. On Chatlet, which a company with neither talks on, the person + reads the rooms on the dashboard as their seat, and nothing is warned. minProperties: 1 properties: atlassianAccount: diff --git a/dist/chart/templates/crd/units.crewlet.ai.yaml b/dist/chart/templates/crd/units.crewlet.ai.yaml index 746f79bb..2b06b20b 100644 --- a/dist/chart/templates/crd/units.crewlet.ai.yaml +++ b/dist/chart/templates/crd/units.crewlet.ai.yaml @@ -78,7 +78,9 @@ spec: whose members follow the org chart. Crewlet takes lowercase letters, digits, - and _, starting with a letter or a digit, up to 64 characters: the name itself, never #platform or a vendor's channel id. - Crewlet's `channel`. + On Chatlet a room belongs to one unit, so Crewlet refuses a second unit + naming the same channel; a child unit that should share its parent's + room names none. Crewlet's `channel`. maxLength: 64 minLength: 1 pattern: ^[a-z0-9][a-z0-9_-]{0,63}$ diff --git a/docs/concepts/applying.md b/docs/concepts/applying.md index f31487dc..db8e6f94 100644 --- a/docs/concepts/applying.md +++ b/docs/concepts/applying.md @@ -99,7 +99,10 @@ flowchart TD revision the engine's activation pointer names, which every engine pod answers alike (`GET /health`, then `GET /config`), reads which revisions each engine pod runs and would start on (`GET /fleet`), and lists the - engine's encrypted secret store. Each referenced key is one entry in the store, named as the + engine's encrypted secret store. A revision the engine cannot open ends the + reconcile here (`RevisionUnreadable`), but for one it stores without a + seal, which under `Enforce` the rendered document replaces. Each referenced + key is one entry in the store, named as the document refers to it (`K8S___`). Only values that changed since the last push are written, and any entry the store lost or the engine lists as last written with a credential other than the @@ -116,11 +119,11 @@ flowchart TD values are one activation: every write but the last asks the engine to activate nothing (`defer_reload`), and the last activates the revision once, with every value in the store. A sync whose last write fails, or - that stops before it, records the values it wrote deferred as unwritten, - so the next sync writes them again and its last write activates them; - where the last write activates nothing (the revision reads none of its - values) or its activation fails, the operator applies the document - itself. + that stops before it, records no fingerprint for the values it wrote + deferred, so the next sync writes them again and its last write + activates them; where the last write activates nothing (the revision + reads none of its values) or its activation fails, the operator applies + the document itself. The entries the running document still references are kept until the next document replaces it, and so are those of every other revision an engine pod runs or would start on: each pod applies the fleet's revision on its @@ -194,6 +197,13 @@ flowchart TD document somebody else activated, whose reading the operator does not know, keeps the value it holds for as long as it runs, as the last good revision keeps every value it requires while the next document is held. + A revision the engine stores without a seal keeps every entry the store + holds: nobody can read which values it references, and the engine's nodes + go on running, and boot again when they restart, the revision they + applied before it, which reads the store as it finds it. So nothing is + deleted before the write that replaces it, a value whose Secret no longer + exists included; step 7 deletes what the replacement lets go once it + runs. The Company's `SecretsSynced` and the part's `ResolvedRefs` message say so wherever the reconcile read the Secrets the document references, and a `SecretKept` event, told once, when the reconcile ends with the value @@ -201,50 +211,134 @@ flowchart TD without, and is told by the event alone. Step 7 deletes the value once the next document runs. Whenever the reconcile ends without activating a revision after it deleted a value, because the document is held at any - step, the operator activates the revision the engine runs again, - unchanged, so the engine reads its store without the deleted value - rather than starting a tool server or an executor run with it (a - `Reactivated` event): where that revision read the value, the engine did - so itself as the operator removed it, and the operator takes that - activation for its own; otherwise, or where the engine's failed, it asks - for one (`POST /config/reload`). It keeps what it owes in its record of - the store until a revision has activated. It activates again only a - revision that goes without the value, never one that keeps it, and - records the revision that activates it again as its own only once it has - read back that it is that document: the engine activates again whichever - revision is active when the request lands, so one somebody else activated - in between is reported with a Warning `Reactivated`, and read as a - revision whose reading the operator does not know. Nothing makes the - engine's reload or its deletion conditional on the revision the operator - read, so a revision somebody activates in that moment goes without a - value it may require until the Secret exists again. + step (the engine's own refusal of it included: a document it finds + invalid, a human seat somebody holds that the document takes away, or + the operator's token refused the change), or because the engine already + runs it with every value it references (step 4), the revision the engine + runs is activated again, unchanged, so the engine reads its store without + the deleted value rather than starting a tool server or an executor run + with it (a `Reactivated` event): where that revision read the value, the + engine did so itself as the operator removed it, and the operator takes + that activation for its own; otherwise, or where the engine's failed, it + asks for one (`POST /config/reload`). It keeps what it owes in its record + of the store until the engine runs a revision activated after the + deletion, whoever activated it ([What the engine has read](#what-the-engine-has-read)). + It activates again only a revision that goes without the value, never one + that keeps it, and records the revision that activates it again as its + own only once it has read back that it is that document: the engine + activates again whichever revision is active when the request lands, so + one somebody else activated in between is reported with a Warning + `Reactivated`, and read as a revision whose reading the operator does not + know. A request of the operator's that fails is read back too: a revision + the engine runs that activated after the deletion read the store without + the value, so nothing is owed and nothing is activated again; where it is + the running document again (the request landed although its answer did + not, or somebody reloaded the document), it is recorded as the + reactivation, and where it is another document, a Warning `Reactivated` + says the operator activated nothing again. Where the engine still runs + the revision read, and answered that it stored the request's revision and + could not confirm activating it, that revision may land yet: the deletion + stays owed and is reported, and, where the revision read is the + operator's, the stored one is recorded as an unconfirmed write of its + document (step 6), so that once it runs it is the operator's. Nothing + makes the engine's reload or its deletion conditional on the revision the + operator read, so a revision somebody activates in that moment goes + without a value it may require until the Secret exists again. Over a + revision the engine stores without a seal, which the engine never + activates again, no activation is asked for: what is owed stays owed + until the write that replaces it activates. 4. **Compare.** The operator compares the engine's active document with the - rendered one, and compares a hash of the rendered document and - of the values it references with the one it recorded when it last applied - (`status.config.appliedHash`). If both match, there is nothing to write. - A changed value alone needs a new revision: the engine reads a rotated - value only when a revision activates. Where the engine runs the rendered - document, the write in step 3 made that revision, since the engine - activates the revision that reads a value it is written, and the - operator records it as its own apply (`status.config`), once it has read - back that the engine runs it, rather than activating the same document a - second time; its summary is the engine's, `reload: secret - written`. Where the engine did not activate one (nothing it runs reads - the value, or its activation failed or was overtaken by another write), - the operator applies the same document itself, with a summary that names - the Secret. So with a value written back in step 3, for the same reason: - since it was replaced, a reload may have handed the engine the other - value. + rendered one, and checks that the revision the engine runs activated + after every value it wrote into the store. If both hold, there is nothing + to write, and the revision is recorded with a hash of the rendered + document and of the values it references (`status.config.appliedHash`) + whoever activated it. A changed value alone needs a new revision: the + engine reads a rotated value only when a revision activates. Where the + engine runs the rendered document, the write in step 3 made that + revision, since the engine activates the revision that reads a value it + is written, and the operator records it as its own apply + (`status.config`), once it has read back that the engine runs it, rather + than activating the same document a second time; its summary is the + engine's, `reload: secret written`. Where the engine did not + activate one (nothing it runs reads the value, or its activation failed + or was overtaken by another write), the operator applies the same + document itself, with a summary that names the Secret. So with a value + written back in step 3, for the same reason: since it was replaced, a + reload may have handed the engine the other value. A value stays a reason + to write until the engine runs a revision activated after it is written + ([What the engine has read](#what-the-engine-has-read)), not just in the + reconcile that wrote it: a write that fails, or loses its race to another + writer, leaves the next sync nothing to write while the engine may still + run a revision that read the store before it. Where no write is coming, + because the engine refuses the document or it is held, the revision the + engine runs is activated again for a value that revision references + (`Reactivated`), so that a credential somebody rotated reaches the + running engine whatever else is wrong with the document: by the engine + itself at the write, or, where its activation failed, by the operator; a + Warning `SecretRefreshFailed` says when neither could. A value the + running revision does not reference waits for the write that does. 5. **Dry-run** the document. The engine checks everything the API server cannot: rules that span objects, such as a Seat that asks for a sandbox the Company does not offer. Each problem the engine reports is traced back, by its place in the document, to the part that rendered that place. 6. **Apply** the document as one revision, on the condition that the engine - still runs the revision the dry run checked against (`If-Match`; into an - engine with no document yet, on the condition that it still has none). - The revision's summary is `crewlet-operator:` followed by the objects - whose part of the document changed. A write that loses a race with - another writer is started over once from a fresh read. + still runs the revision step 3 read (`If-Match`, a revision stored without + a seal included; into an engine with no document yet, on the condition + that it still has none), the one everything before was decided against: + a revision another writer activated since, which the dry run may have been + checked against, is never written over unseen. The revision's summary is + `crewlet-operator:` followed by the objects whose part of the document + changed, and, over a revision stored without a seal, that it replaced it. + A write that loses a race with another writer is started over once from a + fresh read. When the engine + stores the revision and then can't confirm that it activated it (a `503` + naming the revision it stored), the fleet may run it or may not, so the + operator records the write in `status.config.unconfirmed` and writes + nothing more until it has read which revision the engine runs: that one, + and it is recorded and reported as the operator's write, as though the + engine had confirmed it, what it replaced included: a revision somebody + else wrote (`DriftCorrected`, with the `Drifted` event recorded then) or + one the engine stores without a seal (`Applied` naming it, with the + `RevisionUnreadable` event recorded then); at + the epoch the fleet's activation names and as applied when the operator + found it, since nothing says when its activation landed; the one the + write was built on, and the write is sent again on the same condition, + which the engine refuses if the first one landed meanwhile; any other, + and the write goes, since it can no longer land. If it landed and was + activated again since (the engine's own reload after it mints a + credential, a `crewlet config reload`, or the operator's own activation + of it again), the revision that runs activated after every value the + write's sync wrote and withdrew ([What the engine has read](#what-the-engine-has-read)), + so, where it is the rendered document, it is in sync and nothing is + written. Under `Seed`, a write into an engine that had no document is + kept in `status.config.overtaken` once overtaken, and is the seed, + whatever revision runs now: nothing the engine says tells a seed that + landed and was replaced from another writer's create that reached the + engine first. It stays recorded until a reconcile decides with it, so one + that fails first does not lose it. Before the write is sent, the operator + also records, under the write's `appliedHash`, which Secret keys its + document lets go without (`pending-activations`), since nothing else says + it once the render has moved on: a write the engine is found running that + `status.config` names as the operator's, confirmed or not, is recorded as + reading the store that way, even where the reconcile that made it did not + get as far as saving the record of the store after it, so that the value of + a deleted Secret it lets go is taken out of the store rather than kept for + it. Where the `status.config` record of the write is lost too, the revision + reads as one the operator has no record of activating, requiring every + value it references. A reconcile that holds its document reads the engine + while a write is unconfirmed, records how a write that runs reads the + store, and leaves `status.config` to the next one that applies a document; + where it activates that write again, the write's record moves on to the + revision that activates it, which the next reconcile then confirms. A + request to activate the running revision again that the engine stores and + cannot confirm activating is recorded the same way, as a write of the + operator's of that revision's document, from the answer, whatever the + operator can read back after it (step 3), with a digest of that document + (`documentSHA256`): the engine activates again whichever revision is active + when the request lands, so a revision found running another document was a + reload of one somebody activated in between, and is not the operator's. + What a write replaced is reported once, by whichever of the write and such + a reload of it is found running. 7. **Clean up** the store. The operator records the new revision in its record of the store, with the Secret keys its document lets go without, before it deletes anything, so that a deletion whose own record a failed @@ -288,7 +382,7 @@ sequenceDiagram O->>E: DELETE /secrets/NAME that no document references, or of a deleted Secret every revision a pod runs or would start on goes without O->>E: PUT /config?dry_run=true with the rendered document E-->>O: the revision it checked against, or each problem at its place - O->>E: PUT /config, If-Match the revision it checked + O->>E: PUT /config, If-Match the revision it read E-->>O: the new revision and its activation epoch Note over O: records the new revision and the keys it lets go without O->>E: GET /health, then GET /config, the fleet's document now @@ -310,7 +404,8 @@ sequenceDiagram > `stuck` only once it has concluded that it did not: an attempt ended in > failure, or the apply has run past the engine's 45-second grace while a peer > applied the epoch. On such a posture, or on `wait` still 72 seconds after the -> write (four of its 18-second polls), the Company reports +> write (four of its 18-second polls), or after the operator found a revision it +> adopts (below), the Company reports > `ConfigApplied=False ApplyFailed` with the engine's status, posture and > account of the failure (from its record of the attempt in `/fleet`), and an > `ApplyFailed` event. The fleet's revision stays the one the write activated @@ -331,18 +426,23 @@ dry-runs and compares, and reads no Secret value and writes nothing; under never again (see [Management policies](#management-policies)). The operator remembers what it pushed as keyed fingerprints of the values, in -a Secret of its own, `-secret-sync`, owned by the Company: the -engine never returns a value, so this is how the operator knows which ones -changed. A fingerprint without the key that Secret also holds says nothing -about a value. The same Secret holds the operator's record of the store's -other facts: the revision whose reading of the values it knows, with the -Secret keys that revision lets go without (`optional-keys`), the values it -deleted that a revision has yet to activate without, and the ones it said it -keeps. It records nothing of the engine's pods: what each runs and would -start on is what `GET /fleet` says of it now, since a pod applies revisions, -and a data pod moves the revision its volume starts it on, whether or not -the operator is looking, and a pod the fleet does not list keeps every value -it may need instead (step 3). +a Secret of its own, `-secret-sync`, owned by the Company: the engine +never returns a value, so this is how the operator knows which ones changed. A +fingerprint without the key that Secret also holds says nothing about a value. +The same Secret holds the operator's record of the store's other facts: the +revision whose reading of the values it knows, with the Secret keys that +revision lets go without (`optional-keys`), the values it deleted that a +revision has yet to activate without (`withdrawn-names`), the ones it wrote +that a revision has yet to activate with (`written-names`), each of those two +with the activation epoch it waits past and the revisions it awaits +(`awaited-revisions`), the ones it said it keeps, and, for each write of the +document the engine may yet be found running before the operator's record of +the store has followed it, the Secret keys that write's document lets go +without, under its `appliedHash` (`pending-activations`). It records nothing +of the engine's pods: what each runs and would start on is what `GET /fleet` +says of it now, since a pod applies revisions, and a data pod moves the +revision its volume starts it on, whether or not the operator is looking, and +a pod the fleet does not list keeps every value it may need instead (step 3). The engine is reached at its first data pod, `http://-data-0.-data..svc:8080`, with the @@ -355,7 +455,58 @@ reports `ConfigApplied=False` with the reason `EngineUnavailable`, and the operator tries again every ten seconds, or sooner where the engine's `Retry-After` says so. Because the address is the cluster's DNS name of the pod, an operator running outside -the cluster (`make run`) cannot reach any engine. +the cluster (`make run`) cannot reach any engine. When the engine answers but +cannot open the fleet's revision, sealed under a key its keyring lacks or in a +form its build does not read, the Company reports `RevisionUnreadable` +instead, which no retry clears, and the operator looks again when the data +pods restart and every five minutes besides +([The engine cannot open its company document](../guides/troubleshooting.md#the-engine-cannot-open-its-company-document)). +A revision the engine stores without a seal is the exception: a whole +document replaces it, and under `Enforce` the operator writes the rendered one +over it (below). + +### What the engine has read + +The engine reads its secret store only when a revision activates: every node +reads it at its attempt to apply the activation, a node that refuses the +revision included, whose running revision resolves its references from what +that attempt read. So a value written or deleted reaches nothing the engine +runs until a revision activates after it. The fleet's activation pointer +(`GET /fleet`, `activation`) names the fleet's revision at its epoch, and the +epoch only grows. So the operator reads the pointer after every change it +makes to the store, before it asks for the activation that is to read it, and +marks each change with the epoch read (`withdrawn-names` and `written-names` +keep it after the name): any revision the pointer names at a greater epoch +activated after the change. A change is settled once the pointer is read +naming such a revision, whoever activated it: the operator's own write or +reload, a person's `crewlet config reload`, or the engine's own reload after +it mints a credential. Only a revision the operator's read of `GET /config` +returned counts, as one it saw opened: both read the pointer, and a revision +a write activated between the two reads may be one no node can open. A value +is marked as written before it is +written, and the marks are saved before the activation is asked for, so a +save the operator loses after either, or a restart between them, leaves the +change owed, or settled by the next read of the engine, never settled by a +revision that activated before it. A write of the document that the engine +runs is therefore never sent again because the record of its activation was +lost: the next reconcile finds the revision running above the epoch every +value was marked by, and takes it for in sync. That needs a read of the +pointer between the change and the activation. A change the operator could not +follow with one, because the fleet could not be read or the engine had no +document yet, is settled by the activation the operator sees answered; where +the engine answers that it stored a revision for the activation and could not +confirm activating it, which is when the pointer is likeliest to be +unreadable, the change awaits that revision (`awaited-revisions`) and is +settled once the engine is read running it, however the pointer reads then. +Only where the engine confirmed the activation and the operator lost the +record of that answer as well does the next reconcile activate a revision +again for such a change: one write or reload of the document more, with the +same values. So is a change whose activation the engine made itself: the +last write or removal of a sync activates the revision inside the same +request, before any read of the pointer can follow the change, so the +operator settles it from that request's answer once it has read back that +the engine runs the revision it names, and a save lost after that costs one +activation more. --- @@ -394,7 +545,31 @@ write. A revision the engine activates on its own with the same content — a reload after a credential was minted — is not drift: the operator adopts its id in -`status.config.revisionID` and writes nothing. +`status.config.revisionID`, and in `status.config.epoch` the epoch the +fleet's activation names for it (`GET /fleet`), and writes nothing. It +reports `ConfigApplied=True` once the engine has applied that epoch, not the +epoch of the revision it replaced, and `ApplyFailed` when the engine has not +72 seconds after the operator found it, which `status.config.appliedAt` +records: the operator's last write is another revision's, and may be days +old. A fleet whose activation names another revision by then had another +write activate one between the two reads, and the operator records nothing and +starts over once from a fresh read, as it does a write that lost its race. + +A revision the engine stores without a seal, which it shows to nobody and +neither applies nor activates again, is not compared, since nobody can read +it: the engine's remedy for one is a whole document, which reads nothing from +it, and the operator writes the rendered document over it, conditioned on that +revision. It reports `ConfigApplied=True` with the reason `Applied` naming the +revision it replaced, records a Warning `RevisionUnreadable` event saying so, +and summarises the new revision as one that replaced it, the same once a read +shows the engine runs a replacement it could not confirm activating. Nothing +of the old revision is read, so nothing in the secret store is deleted before +the write: the engine's nodes run, and boot again, the revision they applied +before it, which may read any value the store holds, and the sync after the +write removes what the replacement lets go. A revision sealed under a key the +keyring lacks, or in a form the engine's build does not read, is never written +over: it is somebody's document, and the Company reports `RevisionUnreadable` +until the engine can open it. The comparison is over what the engine serves back, which is its own encoding of the document rather than the bytes it was given. It disregards the @@ -416,6 +591,7 @@ secret store (`SecretsSynced` is `Unknown` with the reason `NotManaged`). | the rendered document | `True`, `InSync` | `True`, `InRevision` | | another one | `False`, `Drifted`, naming the places that differ | `False`, `Drifted`, on each part whose place differs; `True` on the others | | no document | `False`, `Drifted` | `False`, `Drifted` | +| a revision it cannot open | `False`, `RevisionUnreadable`, naming it | `False`, `Blocked`, but where the part's current generation already runs | A `Drifted` event is recorded when the difference first appears or changes. Observe is how to hand the objects a company the engine already runs (one @@ -433,7 +609,13 @@ values it references, and from then on leaves the engine alone: the Company reports `ConfigApplied=True` with the reason `Seeded`, and every part `Applied=False` with the reason `NotManaged`, since a change to it is no longer applied. An engine that already had a document when the Company was created -is never written to; the Company reports `Seeded` all the same, saying so. +is never written to; the Company reports `Seeded` all the same, saying so. One +whose fleet's revision it cannot open is not written to either, a revision it +stores without a seal included: the Company reports `RevisionUnreadable` +with the remedy for its cause (the key put back into the keyring, or the +engine build that sealed the revision run), and for a revision stored +without a seal, that an admin of the company replaces it with a whole +document, or that `Enforce` replaces it with the rendered one. --- @@ -515,12 +697,12 @@ takes no work for a person that a pause could hold. | Object | Condition | When | |---|---|---| -| Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know (it names those, and which of the two it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | -| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine) or `EngineUnavailable`. | +| Company | `SecretsSynced` | `True`, `Synced`, once the store holds every referenced value, counting the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which it does not hold, or holds as it was for the revision the engine runs where that revision requires one, or is one whose reading the operator does not know, or for the engine's active revision where that is one stored without a seal, which nobody can read and its nodes do not run (it names those, and which of the three it is), the values it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions it does not know (naming the pods), and, the Company having no `ResolvedRefs`, naming the values of its own setup steps a run goes without and a value of theirs that holds nothing under a key their `optionalKeys` does not list ([What blocks a document](company-document.md#what-blocks-a-document)). `False`, `SecretNotFound` or `SecretTooLarge`, naming each Secret and key; `False`, `EngineUnavailable` when a write to the store failed; `Unknown`, `NotManaged` under Observe, and under Seed once seeded. | +| Company | `ConfigApplied` | `True` with `Applied`, `DriftCorrected`, `InSync` or `Seeded`, once the engine reports the revision's epoch applied. `False` with `Applying` while it applies it, `ApplyFailed` when it did not; otherwise with the reason of whatever blocks the document ([What blocks a document](company-document.md#what-blocks-a-document)): `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `PerSeatNotAllowed`, `PoolCapacityExceeded`, `SeatPoolsNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `SandboxRequired`, `OptionalKeyUnused`, `SecretNotFound`, `SecretTooLarge`, `NameConflict` (the Secret `-secret-sync` is someone else's), `Invalid` (the engine's own words), `Drifted` (Observe), `WriteRefused` (the engine refuses the operator's own token a change to the document), `NotGranted` (the engine does not let the operator's token make a request it needs), `SeatHeld` (the document takes away a human seat a person, a service account or an open invitation holds in the engine), `RevisionUnreadable` (the engine cannot open the fleet's revision, so it neither shows the document nor takes a write built from it; a revision it stores without a seal blocks under `Seed` and `Observe`; under `Enforce` the operator replaces it, and reports it only for one that lands after the operator's read, until the next look ten seconds on replaces it) or `EngineUnavailable`. | | Company | `EngineHealthy` | The engine's alarm table, read each time the engine is reached: `True`, `NoAlarms`; `False`, `Alarms`, naming the one standing longest; `Unknown`, `AlarmsNotReported`, before the engine has evaluated them. Not part of `Ready`. | | Company | `IdentityClaimed` | Whether anybody has joined the company, read from the engine's `/health` each time the engine is reached, and every 30 seconds while nobody has, since a person joins where no watch sees it: `True`, `PersonEnrolled`, once its identity directory holds a person; `False`, `Unclaimed`, while nobody has joined, naming up to five of the company's Human seats and the access token to bring the first person in with; `False`, `NoHumanSeat`, when nobody can, for want of a Human seat; `Unknown`, `IdentityNotReported`, when the engine could not read its directory, did not say, or could not be asked. Not part of `Ready`: a company nobody has joined runs its agent seats. | | every part | `Accepted` | `True` once its Company exists; `False` with `CompanyNotFound`, or `Conflict` when it collides with another object of its company. | -| every part | `ResolvedRefs` | `True`, `Resolved`, its message naming any value it gives a tool server or a seat's executor run that the engine starts with and goes without, under a key an `optionalKeys` list lets go (the message names it: the MCPServer's for a tool server's value, a Seat's or Unit's included), whose Secret or key does not exist or holds an empty value, also while a Secret another part requires holds the document, and saying where the engine keeps the value it held for the revision it runs, which requires it, or whose reading the operator does not know. `False` with `RefNotFound`, `Cycle`, `HumanSeat`, `ReferenceNotAllowed`, `SecretNotFound`, `SecretTooLarge`, `OptionalKeyUnused`, `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty` or `NotAnAccountID`, saying what it references that is missing or cannot be there. Most hold the document, and the part reports the same reason on `Applied`; `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty`, `NotAnAccountID`, and `HumanSeat` on a GitLab `accessLevels` entry hold nothing, because only the part at fault suffers them: it is applied (`Applied=True`) and reported, with a Warning event when the condition turns so. | +| every part | `ResolvedRefs` | `True`, `Resolved`, its message naming any value it gives a tool server or a seat's executor run that the engine starts with and goes without, under a key an `optionalKeys` list lets go (the message names it: the MCPServer's for a tool server's value, a Seat's or Unit's included), whose Secret or key does not exist or holds an empty value, also while a Secret another part requires holds the document, and saying where the engine keeps the value it held for the revision it runs, which requires it, or whose reading the operator does not know, or for its active revision, where that is one stored without a seal, which nobody can read and its nodes do not run. `False` with `RefNotFound`, `Cycle`, `HumanSeat`, `ReferenceNotAllowed`, `SecretNotFound`, `SecretTooLarge`, `OptionalKeyUnused`, `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty` or `NotAnAccountID`, saying what it references that is missing or cannot be there. Most hold the document, and the part reports the same reason on `Applied`; `PoolNotFound`, `NoProvisioner`, `SecretValueEmpty`, `NotAnAccountID`, and `HumanSeat` on a GitLab `accessLevels` entry hold nothing, because only the part at fault suffers them: it is applied (`Applied=True`) and reported, with a Warning event when the condition turns so. | | every part | `Applied` | `True`, `InRevision`, naming the revision, with the generation that revision was rendered from as its `observedGeneration`, once the engine has applied it. `False` with the part's own problem (`Invalid`, `RefNotFound`, a Seat's `PerSeatNotAllowed`, `PoolCapacityExceeded`, `CLIAuthNotPortable`, `NoExecutorCLI`, or `SeatHeld` when it turns a held human seat into an agent's, an LLMProvider's `SandboxPlacementNotOffered`, ...), a Seat's `PlacementPending` while it stays where it runs until a pod of the pool it moves to is ready (or for good, while the Company does not declare that pool), `Blocked` when another object keeps the document from being applied, `Applying` while the engine applies the revision, or `ApplyFailed` when it did not. A part whose current generation already runs keeps `True` meanwhile: only what changed is held back. | | Seat (agent) | `Ready` | Whether the seat runs: see [A Seat's `Ready` condition](#a-seats-ready-condition). A Human seat has none. | | Seat (Agent) | `Paused` | Whether the engine holds the seat paused, once it runs the seat: `True`, `SeatPaused`, naming who paused it, when and why; `False`, `SeatRunning`; with `PauseNotManaged` instead when the pause differs from `spec.paused` under `Observe` or `Seed`. `Unknown`, `PauseUnknown`, while a pause or resume the engine may or may not have recorded is retried; `Unknown`, `PauseRefused`, with the engine's words when it refused one, which is asked for again ten seconds later. None while the engine runs no seat of the Seat's (not in its revision, or a handle another Seat derives too). | @@ -529,24 +711,35 @@ takes no work for a person that a pause could hold. Besides the conditions: - **Company** `status.config`: the revision the engine runs the rendered - document as, its activation epoch, `appliedHash`, when the operator last - applied, and how many objects of each kind it was rendered from (platform - model providers included). `status.config.placement` is where the document - the engine runs pins its agent seats: every value of `crewlet.ai/pool` - (`data` and the Shared pools that run a seat) and of `crewlet.ai/seat` (each - seat of a PerSeat pool). It is read from the engine's own document under - every policy, so under Observe and Seed too. A new revision's pins join - the record before the operator's write that activates it, and while the - engine applies it the record is the previous revision's pins and the new - one's together, because the engine keeps running the previous revision - until it has applied the new one, and for good if that apply fails. Once - the engine reports the new revision's epoch applied, the record is its - pins alone; a document the engine owns (Observe, Seed) is narrowed to only - once the engine's own record of its last apply names that revision and - says it ended well. - The runtime reconciler reads it to keep a seat pool's pods running for as - long as the running revision pins a seat to them ([A pool removed while - seats still run in it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). + document as, its activation epoch, `appliedHash`, when the operator + activated it or found it running (`appliedAt`), and how many objects of each + kind it was rendered from (platform model providers included). + `status.config.unconfirmed` lists the operator's writes the engine stored + and couldn't confirm activating, at most 16, the oldest dropped first, until + a read of the revision the engine runs settles them (step 6 above), each + with what it replaced where that was more than the operator's own document + (`replaced: Unsealed` for a revision the engine stores without a seal; + `replaced: Drifted`, and `driftedAt`, where it differed, for somebody + else's), for the report made once the engine is seen running it. + `status.config.overtaken` lists the ones written into an engine that had no + document that another revision overtook, which can no longer land, and of + which `Seed` takes one for the seed, at most 16, until a reconcile records a + revision of the rendered document in sync, a write of it, or the seed. + `status.config.placement` is where the document the engine runs pins its + agent seats: every value of `crewlet.ai/pool` (`data` and the Shared pools + that run a seat) and of `crewlet.ai/seat` (each seat of a PerSeat pool). It + is read from the engine's own document under every policy, so under Observe + and Seed too. A new revision's pins join the record before the operator's + write that activates it, and while the engine applies it the record is the + previous revision's pins and the new one's together, because the engine + keeps running the previous revision until it has applied the new one, and + for good if that apply fails. Once the engine reports the new revision's + epoch applied, the record is its pins alone; a document the engine owns + (Observe, Seed) is narrowed to only once the engine's own record of its last + apply names that revision and says it ended well. The runtime reconciler + reads it to keep a seat pool's pods running for as long as the running + revision pins a seat to them ([A pool removed while seats still run in + it](runtime.md#a-pool-removed-while-seats-still-run-in-it)). - **Seat**: its handle, its identity in the engine (`seatID`, derived from the company name and the handle), the pool that runs it and the StatefulSet whose pods do (`status.placement.workload`; unset while the @@ -566,16 +759,28 @@ Besides the conditions: Every event the config reconciler records is on the Company, but a seat's pause, which is recorded on the Seat (`SeatPaused`, `SeatResumed`, and `Drifted` when it undid a change made in the engine): `Activated` -with the revision, its epoch and its summary, `Applied` once the engine has -applied it, `ApplyFailed` when it did not, `SecretsSynced` with the store +with the revision, its epoch and its summary (for a write the engine couldn't +confirm, with the revision and its epoch once a read shows the engine runs +it), `Applied` once the engine has +applied it, `ApplyFailed` when it did not, a Warning `RevisionUnreadable` +when a document is held because the engine cannot open the fleet's revision, +or when the rendered document replaced a revision the engine stores without a +seal (for a write the engine couldn't confirm, once a read shows it runs), +`SecretsSynced` with the store entries it wrote and deleted, a Warning `SecretOverwritten` naming each entry it wrote back and who had replaced it, `SecretKept` naming each value of a deleted Secret it keeps in the store for the revision the engine runs, which -requires it or whose reading the operator does not know, `Reactivated` when +requires it or whose reading the operator does not know, or for the engine's +active revision where that is one stored without a seal, which nobody can +read, `Reactivated` when it activated the running revision again so the engine reads its store -without the value of a deleted Secret, a Warning `Reactivated` when the -revision activated again is not the document it read, a Warning -`SecretWithdrawalFailed` when it could not, +without the value of a deleted Secret, or with a value it wrote while no +write of the document was coming, a Warning `Reactivated` when the revision +activated again is not the document it read, or when a request that failed +finds another document running that activated after the change, a Warning +`SecretWithdrawalFailed` when it could not have the engine read its store +without a deleted Secret's value, a Warning `SecretRefreshFailed` when it +could not have it read a value it wrote, `Drifted`, `Invalid` with the engine's problems, and `EngineWarnings` when the engine applied a revision it has something to say about. Events, conditions and logs name Secrets, keys and store entries, diff --git a/docs/concepts/company-document.md b/docs/concepts/company-document.md index cbf7cd90..cad2135c 100644 --- a/docs/concepts/company-document.md +++ b/docs/concepts/company-document.md @@ -37,7 +37,7 @@ handle, and model providers and tool servers by name. | Object | Becomes | |---|---| -| `Company` | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `token_budget`, `providers.embeddings`, `providers.sandbox`, `providers.llm_order`, `integrations.check_interval_seconds`, `seat_access_migrated` (every integration kind, so the engine acts on each Seat's `spec.integrations`), and one top-level key per `spec.engineConfig` entry | +| `Company` | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `chat`, `token_budget`, `providers.embeddings`, `providers.sandbox`, `providers.llm_order`, `integrations.check_interval_seconds`, `seat_access_migrated` (every integration kind, so the engine acts on each Seat's `spec.integrations`), and one top-level key per `spec.engineConfig` entry | | `EngineClass` | `providers.llm.` for each of its `platformLLMProviders`, when the Company sets `spec.llm.usePlatformProviders`; its `state_dir` is `/var/lib/crewlet/llm-cli-platform/`, on the data volume | | `LLMProvider` | `providers.llm.`; its `state_dir` is `/var/lib/crewlet/llm-cli/`, on the data volume | | `MCPServer` | one entry of `mcp_servers`, named by its `metadata.name` | @@ -165,7 +165,7 @@ The engine does not report when a schedule last fired in a form a status field could carry: its record of the fires it dispatched is each node's own table, which the answering node serves, while the fleet's claim on each fire is internal to its coordination store. A Schedule therefore has no -`lastScheduleTime`; the engine's dashboard (**Agents › Schedules**) shows +`lastScheduleTime`; the engine's dashboard (**Company › Schedules**) shows each schedule's next fire and its history. ### Placement diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index c882dc83..ca673cfb 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -31,7 +31,7 @@ the [API reference](../reference/api.md). | Kind | Scope | What it is | Becomes in the company document | |---|---|---|---| | `EngineClass` | Cluster | A platform's defaults and guard rails: the engine image, the size of the data pod, how companies are exposed, limits, models the platform provides | Nothing directly; it shapes the pods, the routes and the limits of every Company that names it | -| `Company` | Namespace | The company itself, and the engine that runs it | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `token_budget`, `providers.embeddings`, `providers.sandbox`, and the engine tuning blocks in `engineConfig` | +| `Company` | Namespace | The company itself, and the engine that runs it | `name`, `mission`, `vision`, `policies`, `timezone`, `tracker`, `knowledge`, `chat`, `token_budget`, `providers.embeddings`, `providers.sandbox`, and the engine tuning blocks in `engineConfig` | | `Unit` | Namespace | One node of the org chart: a division, a team, a squad | One entry of `units`, nested under its parent | | `Seat` | Namespace | One seat: an AI agent, or a person agents work with, tied to their own accounts ([A person's accounts](../guides/integrations.md#a-persons-accounts)) | One entry of `roles`, inside its unit | | `LLMProvider` | Namespace | A model a company's seats can run on: a coding CLI the engine drives ([Running seats on a coding CLI](../guides/cli-agents.md)) | One entry of `providers.llm` | diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index 88d83aee..f5037830 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -521,8 +521,18 @@ A company's event stream is three quarters of its data volume, and the engine refuses to boot on a stream smaller than what a fresh node reserves ([the class's floor](company-document.md)). A volume made under an older floor, or before an engine that keeps more on its stream, can fall short, and -a company that keeps more than its volume holds needs room too. Grow the -claim, not the class: +a company that keeps more than its volume holds needs room too. The operator +holds the pods of a company whose volume falls short as they are, with +`RuntimeReady=False`, reason `DataVolumeTooSmall`, naming the volume and its +claim, rather than roll them onto an engine that cannot reserve its state +logs; a pod whose image has already moved to such an engine restarts all the +same, and the condition says why. A new company made from a class stored +before the floor rose, which the API server lets keep a size it no longer +admits until the size itself is changed, has no claim yet: the condition +names the class instead, and setting the class's `data.storage.size` to at +least its floor (`14Gi`, or `11Gi` with the files in S3) clears it, since the +operator then creates the company's volume at that size. A company that has +its claim grows the claim, not the class: 1. The claim's StorageClass must allow it (`allowVolumeExpansion: true`); the API server refuses an expansion otherwise. @@ -542,11 +552,14 @@ claim, not the class: The operator reads the size the claim has, the smaller of what it asks for and what it was given, and never less than the size it was made with, so an expansion counts once it has landed and a provisioner that rounded a claim up -moves nothing. The class's `data.storage.size` is not involved, and the -StatefulSet keeps the template it was created with. A StorageClass that -cannot expand leaves only a new volume: take a backup and keep the keyring -Secret with it ([Backups](#backups)), then restore it onto a large enough -volume as Crewlet's backup guide describes. +moves nothing. A claim that asks for less than the StatefulSet's template was +not made from it: it is one a `Retain` deletion kept, which a Company created +again under the same name runs on as it is, so it counts at what it asks for, +or what it was given if less. The class's `data.storage.size` is not +involved, and the StatefulSet keeps the template it was created with. A +StorageClass that cannot expand leaves only a new volume: take a backup and +keep the keyring Secret with it ([Backups](#backups)), then restore it onto a +large enough volume as Crewlet's backup guide describes. The engine migrates its data forward only. The operator records the image the data pod last ran *and was ready on* in `status.engine.image`, and its tag in @@ -596,7 +609,10 @@ engine would be told it is reached at that Crewlet refuses to start on class states it, so this is in practice the class's API route made of a Company's namespace or name, whose last label is a number as `{{name}}.{{namespace}}`'s is in a namespace named `4711` or `0x1f` ([Reach -the API from elsewhere](../guides/exposure.md#reach-the-api-from-elsewhere)). +the API from elsewhere](../guides/exposure.md#reach-the-api-from-elsewhere)), +or a data volume whose stream is under what a fresh node reserves on it +(`DataVolumeTooSmall`, [Growing a company's data +volume](#growing-a-companys-data-volume)). Suspending still scales the pod to zero meanwhile. --- @@ -824,6 +840,8 @@ operator: A Company created again with the same name and namespace after a `Retain` deletion adopts the kept keyring and finds its volume, and with it its estate. +Its event stream is sized from that volume as the claim has it, whatever its +class gives a new volume now. To discard a retained company for good, delete the claims first and the keyring Secret after them: a Company that finds its claims without its keyring holds its pod back rather than generate a keyring that cannot open @@ -899,11 +917,17 @@ the start, which `EngineHealthy=False Alarms` shows. Take backups through the engine, which is the only process that can read its estate. `crewlet backup` writes a verified copy into a directory on the data -pod's volume, without stopping the engine: +pod's volume, without stopping the engine. Like every command that reaches a +running node it takes the node's address with `-api` and its token from +`CREWLET_API_TOKEN` alone, never a flag, so pipe in the access token the +Company publishes ([Reaching the engine](#reaching-the-engine)); the +directory is named on your machine, so the timestamp is its clock's, in UTC: ```bash -kubectl exec -n acme acme-data-0 -- crewlet backup -config /etc/crewlet/crewlet.yaml \ - -url http://127.0.0.1:8080 -dir /var/lib/crewlet/backups/$(date -u +%Y-%m-%dT%H-%M) +dir=/var/lib/crewlet/backups/$(date -u +%Y-%m-%dT%H-%M) +kubectl get secret acme-api-access -n acme -o jsonpath='{.data.token}' | base64 -d | + kubectl exec -i -n acme acme-data-0 -- sh -c 'CREWLET_API_TOKEN=$(cat) exec crewlet backup \ + -config /etc/crewlet/crewlet.yaml -api http://127.0.0.1:8080 -dir "$1"' sh "$dir" ``` Run it at least daily. Copy each backup off the volume (`kubectl cp`), since diff --git a/docs/getting-started.md b/docs/getting-started.md index 35ac0859..91604d3a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -323,7 +323,7 @@ bin/kubectl port-forward -n acme svc/acme-engine 8080:8080 Open exactly : the engine is told it is reached there, and accepts a sign-in from that address and no other. While nobody has joined, the sign-in page puts its token form first: paste the token and -choose **Sign in with token**. Then open **Agents › Org chart**, open the +choose **Sign in with token**. Then open **Company › Org chart**, open the founder seat, choose **Add person** with your email address and **Admin** ticked, and open the password link it shows you once. Choose a password, sign in with your address, and set up the authenticator app the engine asks for. diff --git a/docs/guides/credentials.md b/docs/guides/credentials.md index d198b885..d3b45488 100644 --- a/docs/guides/credentials.md +++ b/docs/guides/credentials.md @@ -423,6 +423,15 @@ kubectl get company acme -n acme -o jsonpath='{.status.config.revisionID}{"\n"}' A change that leaves every referenced value as it was (a label, an annotation, another key nothing references) writes nothing. +The new value reaches the engine even while it refuses the rendered +document (`ConfigApplied=False`, say for a seat somebody holds): no revision +of the document is written then, but the revision the engine runs is +activated again, unchanged, with a `Reactivated` event naming the store +entry, as long as that revision uses the value: the engine does so itself +as the value is written, and the operator asks for it where the engine's +activation failed. A Warning `SecretRefreshFailed` says it could not, and +when it tries again. + To rotate a provider's key without a moment where the vendor refuses the one in use, mind the order: write the new key into the Secret, wait for the revision that carries it, and only then revoke the old key at the vendor. diff --git a/docs/guides/integrations.md b/docs/guides/integrations.md index aee5d999..86ff7127 100644 --- a/docs/guides/integrations.md +++ b/docs/guides/integrations.md @@ -15,6 +15,22 @@ Every credential is a Secret reference ([Supplying credentials](credentials.md)) [Credential Positions](../concepts/credentials.md) lists every position and the field that fills it. +A company with no Slack or Mattermost Integration talks on Chatlet instead, +Crewlet's own chat, which needs no Integration: the engine opens a room for +each unit's `channel`, whose members follow the org chart, and its seats and +people talk there, a person on the dashboard as their seat +([Chatlet](https://github.com/crewlet/crewlet/blob/main/docs/concepts/chatlet.md)). +Connecting Slack or Mattermost moves the company onto that vendor's chat. The +Company's `spec.chat` names the backend instead of leaving the engine to +derive it (`None` turns chat off), and sets the policy of Chatlet under +`chatlet`, beside `Chatlet` or no backend at all: how long a message is +kept (`messageRetentionDays`, 365 by default, `0` for ever), whether a room +is private unless its creator says otherwise (`defaultChannelPrivate`), and +when a seat shows it is working (`typingStatus`). The engine's dry run +refuses, on the Company, `Chatlet` beside a Slack or Mattermost Integration, +`Vendor` without one, and a `chatlet` block on a company whose chat is a +vendor's, which nothing would read. + An Integration connects the engine to the vendor. An agent acts there, its chat replies included, through a tool server: an `MCPServer` its Seat gives its own credentials. Most of those servers are programs the engine starts in diff --git a/docs/guides/management-policies.md b/docs/guides/management-policies.md index f77c8301..43fff532 100644 --- a/docs/guides/management-policies.md +++ b/docs/guides/management-policies.md @@ -53,6 +53,14 @@ it stays in the engine's revision history. kubectl get events -n acme --field-selector involvedObject.name=acme,reason=Drifted ``` +A revision the engine stores without a seal, which no engine reads, is +replaced the same way, without a comparison: nobody can read it, and a whole +document is what replaces it. The operator reports `ConfigApplied=True Applied` +naming the revision it replaced, with a Warning `RevisionUnreadable` event. A +revision the engine cannot open because it is sealed, under a key its keyring +lacks or in a form its build does not read, is never written over; see +[The engine cannot open its company document](troubleshooting.md#the-engine-cannot-open-its-company-document). + ## Observe The operator renders the document, has the engine dry-run it, and compares @@ -101,7 +109,10 @@ the Secret values it references, and from then on leaves the engine alone. The Company reports `ConfigApplied=True Seeded`; every object reports `Applied=False NotManaged`, because a change to it is no longer applied, and `SecretsSynced` is `Unknown`. An engine that already had a document is never -written to. +written to, even one it cannot open: a revision it stores without a seal is +reported as `ConfigApplied=False RevisionUnreadable`, for an admin of the +company to replace with a whole document, or for `Enforce` to replace with the +rendered one. > **Under Seed, rotate in the engine.** Rotating a Secret is not applied under > Seed either: rotate the credential through the engine's own secret store diff --git a/docs/guides/people.md b/docs/guides/people.md index 3a779c07..a39d987b 100644 --- a/docs/guides/people.md +++ b/docs/guides/people.md @@ -44,8 +44,10 @@ spec: - `contact` is optional. It names the person's chat accounts, so that an agent can mention or message them on Slack or Mattermost; a person who - works only through the dashboard needs none. Crewlet warns of a Human seat - without one, which the operator reports as an `EngineWarnings` event + works only through the dashboard needs none. On a company that talks on + either, Crewlet warns of a Human seat with no account there, which the + operator reports as an `EngineWarnings` event; on Chatlet, which a company + with neither talks on, it warns of nothing ([A person's accounts](integrations.md#a-persons-accounts)). - `email` is optional too, and unique: two Seats of one company may not declare one address, compared as Crewlet compares them (case aside, and @@ -167,7 +169,7 @@ kubectl get secret acme-api-access -n acme -o jsonpath='{.data.token}' | base64 In the dashboard, the sign-in page puts its token form first while nobody has joined (**Sign in with token**; once somebody has, it is folded under -**Use an API token instead**). Paste the token, open **Agents › Org chart**, +**Use an API token instead**). Paste the token, open **Company › Org chart**, open your seat (drawn **Open seat**) and choose: - **Invite**: your address, and **Admin** ticked. The invitation link is @@ -202,8 +204,8 @@ wait for it: a company nobody has joined runs its agents all the same. ## Everyone after -People with `admin` invite and create the others in the dashboard (**Agents › -Org chart**, on the person's seat; **Settings › People & access** to change, +People with `admin` invite and create the others in the dashboard (**Company › +Org chart**, on the person's seat; **Settings › People** to change, suspend or remove them), or with `crewlet iam`. Each needs a vacant Human seat: add the Seat first. Under `Enforce` the company document is read-only in the dashboard, so a seat is added as a `Seat` object, not there diff --git a/docs/guides/tool-servers.md b/docs/guides/tool-servers.md index 562d726e..7ffc1476 100644 --- a/docs/guides/tool-servers.md +++ b/docs/guides/tool-servers.md @@ -134,9 +134,11 @@ operator's under the `Enforce` policy, under which it keeps the store in step with the Secrets ([Management policies](../concepts/applying.md#management-policies)), and it holds where the revision the engine runs goes without the value too. A revision that reads the key as required, as the one from before you listed -the key does, or another document somebody else activated, whose reading the -operator does not know, keeps the value it holds until the next document -runs, as the last good revision keeps every value it requires; the Company's +the key does, another document somebody else activated, whose reading the +operator does not know, or a revision the engine stores without a seal, which +nobody can read while its nodes run the one before it, keeps the value it +holds until the next document runs, as the last good revision keeps every +value it requires; the Company's `SecretsSynced`, the object's `ResolvedRefs` message and a `SecretKept` event say so (a document the render holds is told by the event alone). Once no engine pod runs, or would start on, a revision that requires it, the operator @@ -512,9 +514,15 @@ Site URL's path. `mcp-server-mattermost` sends every call to that address with `httpx`, which follows no redirect unless told to, and it never tells it, so at an address Mattermost redirects no agent can post. -The prefix makes its posting tool `mattermost_post_message`, the name -Crewlet's [mentions skill](https://github.com/crewlet/crewlet/blob/main/examples/tool-skills/platform-mentions.md) -triggers on. Each seat gives it `MATTERMOST_TOKEN`, the bot token of its +The prefix makes the server's tools `mattermost_post_message`, +`mattermost_update_message`, `mattermost_list_my_channels`, +`mattermost_list_public_channels` and `mattermost_search_users`, the names +Crewlet's bundled [tool skills](https://github.com/crewlet/crewlet/tree/main/examples/tool-skills) +trigger on: its [mentions skill](https://github.com/crewlet/crewlet/blob/main/examples/tool-skills/platform-mentions.md) +among them, which teaches a seat Mattermost's `@username` mentions and thread +replies before its first post. Under another prefix, or a release that +renames one of those tools, the skills go quiet without a word. Each seat +gives it `MATTERMOST_TOKEN`, the bot token of its `spec.integrations.mattermost` ([Mattermost](integrations.md#mattermost); Crewlet's [Mattermost](https://github.com/crewlet/crewlet/blob/main/docs/integrations/mattermost.md#configure-in-yaml)). @@ -577,6 +585,15 @@ spec: toolPrefix: slack_ ``` +The prefix does here what it does on Mattermost: it makes the server's tools +`slack_conversations_add_message`, `slack_channels_list`, +`slack_channels_me` and `slack_users_search`, the names the same bundled +skills trigger on, so a seat posting on Slack is taught Slack's +`<@member-id>` mentions and thread replies before its first post. Under +another prefix, or a release that renames one of those tools, they go quiet +without a word (Crewlet's +[Slack](https://github.com/crewlet/crewlet/blob/main/docs/integrations/slack.md#configure-in-yaml)). + `slack-mcp-server` registers `conversations_add_message`, its only tool that posts, when `SLACK_MCP_ADD_MESSAGE_TOOL` is set, or when `SLACK_MCP_ENABLED_TOOLS`, the list of tools it offers, names it; a list that diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 20ff8af7..429c9734 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -33,6 +33,7 @@ flowchart TD EX -->|True| CA{"ConfigApplied"} CA -->|EngineUnavailable| UN["The operator cannot reach the engine"] CA -->|"WriteRefused, NotGranted, SeatHeld"| WR["The engine refuses a change
to the document"] + CA -->|RevisionUnreadable| RU["The engine cannot open
its company document"] CA -->|"Applying, ApplyFailed"| AF["The engine did not apply a revision"] SYM -->|"IdentityClaimed=False"| NS["Nobody can sign in"] CA -->|"any other reason"| NA["Changes are not applied"] @@ -78,6 +79,7 @@ with your last change yet. | | False | `APITokensInvalid` | `-api-tokens`, as a platform supplied it, lacks a token or holds one shorter than 43 characters or outside base64, base64url and hex. Repair it, or delete it to have the operator generate it. The pod is held as it is. | | | False | `InsecureSignInNotAcknowledged` | The class lets people sign in with a password alone (`spec.signIn.secondFactor: Optional`), and the Company is reached at an address Crewlet does not read as loopback, which the message names: Crewlet refuses to start so without the class's `spec.signIn.acceptInsecure: true`. The platform acknowledges it on the class or requires a second factor again ([Two-step verification](people.md#two-step-verification)). The pod is held as it is. | | | False | `AddressRefused` | An address the engine would be told it is reached at is one Crewlet refuses to start on; the message names it, its key in `crewlet.yaml` and where it came from. In practice it is the class's API route: its `api.hostname` template's last label holds `{{namespace}}` or `{{name}}`, and the Company's namespace is a number, such as `4711` or `0x1f`, or its namespace or name makes one with the characters beside it (`0x{{name}}` for a Company named `cafe`), so the host's last label is a number, which a browser reads as an IPv4 address. The platform ends the template in a domain of its own (`{{name}}.{{namespace}}.crewlet.example.com`), or the Company runs in a namespace of another name ([Reach the API from elsewhere](exposure.md#reach-the-api-from-elsewhere)). An address the Company states is refused by the API when it is written instead, unless it was admitted before the API refused it: the message then says to correct the Company's field. An artifact origin that meets the dashboard's is refused here too: give artifacts a domain of their own. The pod is held as it is. | +| | False | `DataVolumeTooSmall` | The data volume gives the engine an event stream, three quarters of it, under what a fresh node reserves on it, its state logs at their floors and, with the files in the stream's own bucket, that bucket beside them. The class's `data.storage.size` admits no less than `14Gi` (`11Gi` with the files in S3), so this is a volume sized under an older floor, and the message names what to grow. A company that has its claim (one that kept it, or one created again over the claim a company deleted before kept) is told the volume and the claim: [grow the claim](../concepts/runtime.md#growing-a-companys-data-volume) to at least the size the message names, and the pod runs once the expansion lands. Until then it is held as it is, which keeps an engine that still starts on the volume running, though not one whose image has already moved to an engine that cannot reserve its state logs there. A new company made from a class stored before the floor rose has no claim yet and is told the class: set its `spec.data.storage.size` to at least that size, and the operator creates the company's volume at that size. | | `Exposed` | True | `RouteAccepted` | Every route the class asks for was accepted. When the message adds that the engine is told people reach it at `http://localhost:8080`, every link it gives a person, an invitation's included, leads there through a port-forward: set the Company's `spec.exposure.dashboardBaseURL`, or route the API ([Where people reach the dashboard](exposure.md#where-people-reach-the-dashboard)). | | | True | `NoRoutes` | The class routes nothing (webhooks `None` or `External`, API `ClusterInternal`, artifacts `None`). The message says the same about the dashboard's address as under `RouteAccepted`. | | | False | `RouteNotAccepted` | A Gateway has not attached the HTTPRoute, or the Ingress controller has not published an address. The message carries the controller's own reason; a Gateway listener's `allowedRoutes` that does not admit the tenant's namespace is the usual cause. | @@ -85,7 +87,7 @@ with your last change yet. | | False | `GatewayAPINotInstalled` | The class routes through Gateways and the cluster had no Gateway API when the operator started. Install it and restart the operator. | | | False | `PublicBaseURLMissing` | The class's webhooks mode is `External` and the Company has no `spec.exposure.publicBaseURL`. | | | False | `RouteNotHTTPS` | No Gateway listener the artifacts route attaches to is HTTPS, so the data pod opens no artifact origin; add one with a certificate for the route's host. | -| `SecretsSynced` | True | `Synced` | The engine's secret store holds every referenced value. The message counts the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which the store does not hold, so the engine leaves each out ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)); names, in a sentence of its own, those the store keeps as they were for the revision the engine runs, because that revision reads them as required or is one whose reading the operator does not know (another document somebody else activated), each to go once a revision that lets it go runs; names, in another, those it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions the operator does not know, with the pods, each to go once no pod runs or would start on a revision that references it ([The engine did not apply a revision](#the-engine-did-not-apply-a-revision)); and, the Company having no `ResolvedRefs`, names the values of its own setup steps a run goes without, under a key a step's `optionalKeys` lists, and a value of theirs that holds nothing under a key none lists, which every executor run starts without. | +| `SecretsSynced` | True | `Synced` | The engine's secret store holds every referenced value. The message counts the values a tool server or a seat's executor run may go without whose Secret or key does not exist, which the store does not hold, so the engine leaves each out ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)); names, in a sentence of its own, those the store keeps as they were for the revision the engine runs, because that revision reads them as required or is one whose reading the operator does not know (another document somebody else activated), or for the engine's active revision, where that is one stored without a seal, which nobody can read while its nodes run the revision before it, each to go once a revision that lets it go runs; names, in another, those it keeps only for a revision an engine pod runs or would start on other than the fleet's, or for a pod whose revisions the operator does not know, with the pods, each to go once no pod runs or would start on a revision that references it ([The engine did not apply a revision](#the-engine-did-not-apply-a-revision)); and, the Company having no `ResolvedRefs`, names the values of its own setup steps a run goes without, under a key a step's `optionalKeys` lists, and a value of theirs that holds nothing under a key none lists, which every executor run starts without. | | | False | `SecretNotFound` | A referenced Secret or key is missing; the message names each. | | | False | `SecretTooLarge` | A value is over the 64 KiB the store accepts. | | | False | `EngineUnavailable` | Writing to the store failed; the message says why. Retried. | @@ -109,19 +111,20 @@ with your last change yet. | | True | `DriftCorrected` | Under `Enforce`, someone changed the document in the engine and the operator applied the rendered one over it. A `Drifted` event names what differed. | | | True | `InSync` | Under `Observe`, the engine runs exactly the rendered document. | | | True | `Seeded` | Under `Seed`, the engine has a document and owns it from now on. | -| | False | `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `SecretNotFound`, `SecretTooLarge`, `PerSeatNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `OptionalKeyUnused` | An object of the company keeps the document from rendering. The object at fault reports the same reason on `Applied`; fix it there ([What blocks a document](../concepts/company-document.md#what-blocks-a-document)). Meanwhile, under `Enforce`, the operator still takes out of the engine's secret store the value of a deleted Secret the document lets go without, and activates the running revision again so the engine reads the store without it, with a `Reactivated` event, where the running revision lets the value go too, or a Warning `Reactivated` when the revision activated again is not the document it read; one that reads it as required, or another document somebody else activated, keeps it until the next document runs, with a `SecretKept` event, the only report of it here, since the Secrets are not read ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). | +| | False | `RefNotFound`, `Cycle`, `Conflict`, `ReferenceNotAllowed`, `HumanSeat`, `SecretNotFound`, `SecretTooLarge`, `PerSeatNotAllowed`, `CLIAuthNotPortable`, `NoExecutorCLI`, `SandboxPlacementNotOffered`, `OptionalKeyUnused` | An object of the company keeps the document from rendering. The object at fault reports the same reason on `Applied`; fix it there ([What blocks a document](../concepts/company-document.md#what-blocks-a-document)). Meanwhile, under `Enforce`, the operator still takes out of the engine's secret store the value of a deleted Secret the document lets go without, and activates the running revision again so the engine reads the store without it, with a `Reactivated` event, where the running revision lets the value go too, or a Warning `Reactivated` when the revision activated again is not the document it read, or when its request failed and another document, activated after the deletion, runs instead, and it activated nothing again; one that reads it as required, or another document somebody else activated, keeps it until the next document runs, with a `SecretKept` event, the only report of it here, since the Secrets are not read ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). | | | False | `NameConflict` | The Secret `-secret-sync`, where the operator keeps what it pushed to the engine's store, exists and is someone else's; `SecretsSynced` says the same. Rename or delete it; nothing is applied meanwhile. | | | False | `SeatPoolsNotAllowed` | The Company declares `spec.topology.seatPools`, and its class's `seatPools.policy` is not `Allowed`. Remove the pools, or run the company with a class that allows them. | | | False | `SandboxRequired` | The Company has an agent seat and a model provider, and no `spec.sandbox`: every agent seat's executor runs in a box, and Crewlet refuses a company that places none on every write. Set `spec.sandbox.defaultRunIn` to `Direct`, or to `E2B` with `spec.sandbox.e2b` ([Where a seat's executor runs](cli-agents.md#where-a-seats-executor-runs)). | | | False | `PoolCapacityExceeded` | The company's seat pods would number more than its class's `seatPools.maxPods`: its Shared pools' replicas alone (the message says so; lower them), or with the pods of its PerSeat seats, each of which reports it too. | | | False | `Invalid` | The engine's dry run refused the document, in its own words, every problem in this message. A problem at a place a part rendered (a Seat, Unit, LLMProvider, MCPServer, Integration or Schedule) is also reported on that part, as `Applied=False Invalid`; one at a place the Company renders itself (its own fields, `spec.embeddings`, `spec.sandbox` and its setup steps) is in this message alone. | | | False | `Drifted` | Under `Observe`, the engine runs another document; the message names the places that differ. | -| | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write. | +| | False | `Applying` | The operator activated the revision and the engine has not applied it yet; it runs the previous one meanwhile. Usually gone within seconds; it turns into `ApplyFailed` when the engine fails the revision or has not applied it 72 seconds after the write, or after the operator found it running where it adopts a revision it did not write. | | | False | `ApplyFailed` | The engine's first data pod, the one the operator reaches, did not apply the revision: it keeps running the one it ran before, and starts on the one its volume names if it restarts, the same one unless the engine is still moving it (the message says which), and the engine's secret store keeps every value of either meanwhile. The message carries the engine's status, its posture (`isolated` on a single node, `shed` or `stuck` beside peers that applied it) and its own account of the failure; see [The engine did not apply a revision](#the-engine-did-not-apply-a-revision). | -| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine asked to be asked again later (`503`: it is starting, or cannot read the coordination store that holds the fleet's revision). Retried every ten seconds, or sooner when the engine says when; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | +| | False | `EngineUnavailable` | The operator cannot reach the engine, the engine does not know its token, or the engine answered `503`: it asked to be asked again later (it is starting, cannot read the coordination store that holds the fleet's revision, or met a condition of its own that clears, such as a store busy with other writes), met something on its side that waiting doesn't clear, or stored the operator's write and couldn't confirm activating it. Looked at again every ten seconds, or sooner when the engine says when, and a write it couldn't confirm is not sent again until a read shows it did not land; see [The operator cannot reach the engine](#the-operator-cannot-reach-the-engine). | | | False | `WriteRefused` | The engine refuses the operator's own token a change to the document (`403 config_managed`): its `api.auth.company_writers` names other tokens. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `NotGranted` | The engine knows the operator's token and will not let it make a request the operator needs (`403`): the token lacks the grant the message names, or the engine's identity directory binds it to a seat the company no longer has. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | | | False | `SeatHeld` | The document takes a human seat out of the company, or makes it an agent's, while a person, a service account or an open invitation holds it in the engine (`409 seat_held`). The message names each seat and what holds it; the engine keeps running the revision it runs. See [The engine refuses a change to the document](#the-engine-refuses-a-change-to-the-document). | +| | False | `RevisionUnreadable` | The engine cannot open the fleet's revision of the company document (`409 unreadable_revision`, `unreadable_fleet_revision`, `unknown_seal_revision`, `unsealed_revision` or `unsealed_fleet_revision`), so the engine shows the document to nobody and takes no write built from it. No wait changes that: the operator writes nothing, and looks again when the data pods restart and every five minutes besides. A revision stored without a seal is reported under `Seed` and `Observe`; under `Enforce` the operator replaces it with the rendered document, and reports it only for one that lands after the operator's read, until the next look ten seconds on replaces it. The message names the revision and quotes the engine's account and remedy; see [The engine cannot open its company document](#the-engine-cannot-open-its-company-document). | ### EngineClass @@ -145,7 +148,7 @@ conditions. `kubectl get engineclasses` shows it. | | False | `CompanyNotFound` | `spec.companyRef` names no Company in the namespace. | | | False | `CompanyNotAccepted` | Its Company is not `Accepted` (its class is gone, or it is over its class's limits); the message carries why. Nothing of the company is applied until it is. | | | False | `Conflict` | Crewlet would read it and another object as one: two Seats with one handle, email address or contact account; two Integrations for one vendor; an LLMProvider named like a platform provider of the class; a Seat's header for an HTTP tool server spelled unlike its Unit spells it (`authorization` beside `Authorization`), which Crewlet would send either of. The message names the other object. | -| `ResolvedRefs` | True | `Resolved` | Every object and Secret key it references exists; or the message names the values it gives a tool server or a seat's executor run that it goes without, whose Secret or key does not exist or holds an empty value, under a key an `optionalKeys` list lets the engine leave out, and that list: the MCPServer's for a value of a tool server's, a Seat's or Unit's included ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). A line that adds that the engine keeps the value it held for the revision it runs names a deleted Secret's value that revision reads as required, or whose reading the operator does not know; it goes once a revision that lets it go runs. It says so while a Secret another object requires holds the document too; while another object keeps the document from rendering, the Secrets are not read, and the message says only that every object it references exists. | +| `ResolvedRefs` | True | `Resolved` | Every object and Secret key it references exists; or the message names the values it gives a tool server or a seat's executor run that it goes without, whose Secret or key does not exist or holds an empty value, under a key an `optionalKeys` list lets the engine leave out, and that list: the MCPServer's for a value of a tool server's, a Seat's or Unit's included ([A key that may go without its value](tool-servers.md#a-key-that-may-go-without-its-value)). A line that adds that the engine keeps the value it held for the revision it runs names a deleted Secret's value that revision reads as required, or whose reading the operator does not know; one that says it keeps it for its active revision names one kept for a revision stored without a seal, which nobody can read while the nodes run the revision before it. Either goes once a revision that lets it go runs. It says so while a Secret another object requires holds the document too; while another object keeps the document from rendering, the Secrets are not read, and the message says only that every object it references exists. | | | False | `RefNotFound` | It references an object that does not exist (a Unit, a Seat, an LLMProvider, an MCPServer); the message names it. | | | False | `Cycle` | A Unit's `parentRef` chain leads back to itself. | | | False | `PoolNotFound` | A Seat's `spec.placement.pool` is neither `data` nor one of its Company's `spec.topology.seatPools`. The seat is applied all the same, pinned to the label `crewlet.ai/undeclared-pool`, which no pod carries: it runs nowhere (`Ready=False`, `PoolNotFound`), or keeps running where it ran (`Applied=False`, `PlacementPending`), until the Company declares the pool or the Seat names another. Nothing else is held. | @@ -207,7 +210,9 @@ kubectl logs acme-data-0 -n acme | grep -E 'api_bound|engine_recovered|api_liste message names the figure it needed and breaks it down. The class's `data.storage.size` admits no less than `14Gi` (`11Gi` with the files in S3), so this is a company whose volume was made smaller: under an older - floor, or before an engine that keeps more on its stream. + floor, or before an engine that keeps more on its stream. The operator holds such a company's pods as they + are (`RuntimeReady=False DataVolumeTooSmall`, above), which cannot stop a + pod whose image moved under it from restarting. [Grow its volume](../concepts/runtime.md#growing-a-companys-data-volume). - **`config_apply_failed`**: the revision was stored but this node could not start it; the message says why. The engine keeps running the previous @@ -367,6 +372,42 @@ requests to `http://-data-0.-data..svc:8080` fail: document it cannot vouch for, and the engine logs `config_request_unavailable`. It clears once the engine's broker is back; the data pod's log says why it is not. +- The engine met a condition of its own that clears: its store busy with + other writes, closed while the node adopts a peer's snapshot or shuts + down, or a request out of time. `/config` and `/secrets` answer + `503 unavailable` with a `Retry-After` the operator follows, and the + condition in the engine's words, which the message quotes. +- The engine met something on its side that waiting doesn't clear. It + answers `503` with no `Retry-After`, saying waiting won't change that and, + where it can tell, who has to act, and the condition says so before it + quotes the engine, rather than promise a retry that changes nothing. The + engine's log names what it met. The operator keeps asking every ten + seconds, so `ConfigApplied` turns once that is cleared in the engine. +- The engine stored the operator's write and couldn't confirm that it + activated it: the message names the revision. The engine may run it or may + not. Nothing needs doing: the operator keeps the write in + `status.config.unconfirmed`, reads which revision the engine runs before it + writes again, and records the write as its own if the engine runs it, with a + `Normal` event `Activated` and, where it replaced a revision somebody else + wrote, `ConfigApplied=True DriftCorrected` and a Warning `Drifted`, or one + the engine stores without a seal, `ConfigApplied=True Applied` naming it and + a Warning `RevisionUnreadable`, as for a write the engine confirmed, dated + when it found it running. The message of the write it could not confirm + names such a revision already. It never writes the document a second time + on top of one that landed, nor on top of one that landed and was activated + again before it looked (a reload): that revision activated after every + value the write was made with, so where it is the rendered document the + engine runs it with them, and it is in sync. Under `Seed` a write into an + engine that had no document is kept in `status.config.overtaken` once + another revision runs, and reported as the seed, whatever revision runs + now. A request to activate the operator's own revision again, for the + value of a deleted Secret or one written since, that the engine stored and + couldn't confirm is kept the same way, whatever the operator could read + back after it, beside the `SecretWithdrawalFailed` or + `SecretRefreshFailed` Warning, and recorded as the operator's once it runs + that revision's document; one found running another document reloaded a + revision somebody activated before the request landed, and is not the + operator's. ## Nobody can sign in @@ -430,29 +471,113 @@ while something in the engine's identity directory holds that seat. A person holds a human seat for as long as they are in the company, an open invitation holds the seat it invites to, and a service account may be bound to one. The engine keeps running the revision it runs, and the operator tries again every -ten seconds. The message names each seat and what kind of thing holds it, -never who; a Seat that still exists and is held reports `Applied=False -SeatHeld` too. An admin of the company frees each seat first, in the engine's +ten seconds. When it removes a value from the store meanwhile, the value of a +deleted Secret that revision lets go without, it activates that revision +again so that the engine reads the store without it, as for any document the +engine refuses (step 3 of +[What one reconcile does](../concepts/applying.md#what-one-reconcile-does)). +The message names each seat and what kind of thing holds it, never who; a +Seat that still exists and is held reports `Applied=False SeatHeld` too. An admin of the company frees each seat first, in the engine's dashboard or with [`crewlet iam`](https://github.com/crewlet/crewlet/blob/main/docs/concepts/identity-and-access.md): moves the person to another human seat or removes them, unbinds the service account, or cancels the invitation. `crewlet iam seats` lists what holds each seat. The operator does neither for you: whoever holds a seat is a person, or something a person set up. Or keep the seat: restore its Seat as a Human seat. +## The engine cannot open its company document + +`ConfigApplied=False RevisionUnreadable` means the engine cannot open the +fleet's revision of the company document. The engine refuses every read of the +document with `409`, and every write built from it too; it keeps running what +it applied, and shows no document until it can open that revision. No wait +changes that. A revision that is sealed, under a key the keyring lacks or in a +form the engine's build does not read, is somebody's document, which no write +replaces, since that would discard it unread: nothing the operator sends opens +it, so the operator writes nothing and does not ask again every ten seconds. +It looks again when the data pods restart, and every five minutes besides. The +message names the revision, quotes the engine's account and its remedy, and +says which of five it is: + +- `unreadable_revision`: the revision is sealed under a key the data pod's + keyring does not hold. The keyring is the Secret `-keyring`, which + the operator generates once and never rewrites, so a key gone from it was + taken out by somebody else, or the Secret was restored from an older backup + than the volumes. Put the key back in that Secret from the backup kept + with the volumes, with its `crewlet.ai/active-key-id` annotation as it was: + the data pods restart with it, and their restart runs the reconcile again. +- `unreadable_fleet_revision`: the revision the fleet's activation pointer + carries is sealed under a key the data pod's keyring was never given. + Every pod of the company has its keyring from `-keyring`, so the + data pod lacks a key the Secret gained after it started, and takes it when + it restarts (the pods roll on their own once the operator sees the + change), or a pod that held the key sealed a revision under it and the key + has since gone from the Secret: put it back as above. +- `unknown_seal_revision`: the revision, the data pod's own copy or the one + the fleet's activation pointer carries, is sealed in a form the engine's + build does not read, most likely by a newer build of Crewlet than the data + pod runs. The keyring is not at fault, and putting a key back changes + nothing: run the data pods on the build that sealed it, the engine image + their EngineClass names (with the Company's `spec.runtime.version` as its + tag where the class allows that). A data pod still rolling onto that image + takes it when it restarts; one taken back to an older image than it ran + needs the newer one again. The engine's hint says which. +- `unsealed_revision`: the revision is stored without a seal, which no keyring + opens and no engine reads; only an engine older than the keyring every + engine now requires stored one. +- `unsealed_fleet_revision`: the revision on the fleet's activation pointer + carries no seal. Every engine seals what it activates, so it was put there + from outside the fleet. + +A revision without a seal is the one the engine lets a write replace: a whole +document (`PUT /config`), which reads nothing from it, as the engine's hint +says. Under `Enforce` the operator does exactly that, as soon as nothing holds +the rendered document: it writes the rendered document over the revision, +conditioned on it, and reports `ConfigApplied=True Applied` naming the +revision it replaced, with a Warning `RevisionUnreadable` event saying so. A +document held for a missing Secret, refused by the engine or by the render +keeps the revision in place, and `ConfigApplied` carries the hold's own +reason. Nothing is read from the revision, so nothing in the engine's secret +store is deleted before the write: the engine's nodes go on running, and boot +again when they restart, the revision they applied before it, which may read +any value the store holds; the sync after the write removes what the +replacement lets go. So under `Enforce`, `RevisionUnreadable` reports a +revision without a seal only where one lands after the operator's read, until +its next look ten seconds on replaces it. Under `Seed` and `Observe`, which +write no document over the fleet's revision, it stays reported: an admin of +the company replaces it with a whole document, or the Company's +`managementPolicy` is set to `Enforce`, under which the operator replaces it +with the rendered one. Should the rendered document drop a human seat somebody +holds in the engine, the replacement is refused as any write is +([`SeatHeld`](#the-engine-refuses-a-change-to-the-document)), naming every +seat the document does not keep as a human seat that a person, a service +account or an open invitation holds, since nobody can read which seats the +revision it replaces had. + +To see which keys the keyring holds without printing them: + +```bash +kubectl get secret acme-keyring -n acme -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' +kubectl get events -n acme --field-selector involvedObject.name=acme,reason=RevisionUnreadable +kubectl logs acme-data-0 -n acme | grep config_revision_unreadable | tail +``` + ## The engine did not apply a revision -`ConfigApplied=False ApplyFailed` means the engine stored and activated the -revision the operator wrote and then could not apply it, or never did. The -revision stays the fleet's: `GET /config` answers it, and every pod tries it -again. A pod that did not apply it keeps serving the revision it ran before, -and a data pod starts on the revision its volume names if it restarts, that -same one unless the engine is still moving it (a seat pod starts on nothing -and tries the fleet's revision again); the dry run accepted the new one, -so what failed is something only applying it meets on this engine: a -subsystem it could not start (Worklet, the code sandbox, the inbound -edge), or a rule that needs the data pod's own configuration. The -condition's message carries the engine's account of the failure, its -`status` and its `posture`: +`ConfigApplied=False ApplyFailed` means the engine activated a revision of +the rendered document and then could not apply it, or never did: the revision +the operator wrote, or one with the same document it found running and +adopted without a write (a person's `crewlet config reload`, or the engine's +own reload after it minted a credential). The revision stays the fleet's: +`GET /config` answers it, and every pod tries it again. A pod that did not +apply it keeps serving the revision it ran before, and a data pod starts on +the revision its volume names if it restarts, that same one unless the +engine is still moving it (a seat pod starts on nothing and tries the +fleet's revision again); the dry run accepted the document, so what failed +is something only applying it meets on this engine: a subsystem it could not +start (Worklet, the code sandbox, the inbound edge), a value it resolves +when the revision activates, or a rule that needs the data pod's own +configuration. The condition's message carries the engine's account of the +failure, its `status` and its `posture`: - `isolated`: the only node, or every node, failed the revision. It keeps serving and stays ready, and retries the revision three times at most. @@ -511,7 +636,9 @@ has neither applied the revision nor concluded that it failed within 72 seconds (its posture still reads `wait`), the engine is not polling its activation pointer at all, or an apply hangs with no peer to compare it against; its log says which, and a restart of the data pod is the usual -remedy. +remedy. The 72 seconds run from `status.config.appliedAt`: when the operator +activated the revision, or found one it adopted running, never from its last +write of another revision, which may be days old. ```bash kubectl logs acme-data-0 -n acme | grep -E 'config_apply_failed|reconcile_tick_failed' | tail @@ -753,8 +880,10 @@ Some settings the engine runs are still worth knowing about. Plain `http` on a routable address sends passwords unencrypted. An `https` address with no proxy listed shows every caller as the proxy. A class may let people sign in with a password alone at a routable address, which it acknowledged. A company -may have no human seat anyone can sign in to, or a token ceiling another -leaves idle. None of these stops the engine. It says so in two places: +may have no human seat anyone can sign in to, a token ceiling another leaves +idle, a person on Slack or Mattermost nobody can mention there, or, on +Chatlet, no unit with a `channel`, and so no team with a room the chart +keeps. None of these stops the engine. It says so in two places: - **The Company's events**: the engine's answer to each revision the operator writes, as an `EngineWarnings` event (`The engine runs revision … with diff --git a/docs/reference/api.md b/docs/reference/api.md index ec0aa653..77458032 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -91,6 +91,7 @@ _Validation:_ _Appears in:_ - [ConfigStatus](#configstatus) +- [UnconfirmedRevision](#unconfirmedrevision) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -365,6 +366,65 @@ _Appears in:_ | `Denied` | CLIAgentWebToolsDenied runs the CLI with its web tools off.
| +#### ChatBackend + +_Underlying type:_ _string_ + +ChatBackend is which chat surface a company runs. It maps onto Crewlet's +chat.backend (Chatlet is `chatlet`, Vendor is `vendor`, None is `none`). + +_Validation:_ +- Enum: [Chatlet Vendor None] + +_Appears in:_ +- [ChatSettings](#chatsettings) + +| Field | Description | +| --- | --- | +| `Chatlet` | ChatChatlet is Chatlet, Crewlet's own chat: rooms the engine keeps, a
room for each unit's channel, read on the dashboard.
| +| `Vendor` | ChatVendor is the Slack or Mattermost the company's Integrations (or
a seat's Slack app) connect it to.
| +| `None` | ChatNone is a company with no chat.
| + + +#### ChatSettings + + + +ChatSettings is which chat surface a company runs and the policy of +Chatlet. It maps onto Crewlet's chat block. + +_Validation:_ +- MinProperties: 1 + +_Appears in:_ +- [CompanySpec](#companyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `backend` _[ChatBackend](#chatbackend)_ | backend is which chat surface the company runs. Unset derives it, as
Crewlet does: Vendor for a company with a Slack or Mattermost
Integration or a seat with a Slack app, Chatlet for every other. So
there is no default here: a Chatlet written beside such an
Integration, or a Vendor without one, is refused by the engine's dry
run, reported on the Company, since a company's conversations cannot
live in two places. | | Enum: [Chatlet Vendor None]
Optional: \{\}
| +| `chatlet` _[ChatletSettings](#chatletsettings)_ | chatlet is the policy of Chatlet: for backend Chatlet, or for no
backend on a company Crewlet derives Chatlet for. On a company whose
chat is a vendor's nothing would read it, so the engine's dry run
refuses it there, reported on the Company. | | MinProperties: 1
Optional: \{\}
| + + +#### ChatletSettings + + + +ChatletSettings is the policy of Chatlet. It maps onto Crewlet's +chat.chatlet. + +_Validation:_ +- MinProperties: 1 + +_Appears in:_ +- [ChatSettings](#chatsettings) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `messageRetentionDays` _integer_ | messageRetentionDays is how long a message lives before the engine
prunes it: 0 keeps messages for ever, and a horizon is 30 to 3650
days, since under a month somebody away could miss a decision before
reading it. Unset takes Crewlet's default of 365. Crewlet's
`message_retention_days`. | | Maximum: 3650
Minimum: 0
Optional: \{\}
| +| `defaultChannelPrivate` _boolean_ | defaultChannelPrivate makes a room created without a stated
visibility private; unset, such a room is public, readable by the
company's seats. A room created with a visibility keeps it. Crewlet's
`default_channel_private`. | | Optional: \{\}
| +| `typingStatus` _[WorkingStatus](#workingstatus)_ | typingStatus is when a seat shows in a room that it is working on a
message. Unset takes Crewlet's default, Always. Crewlet's
`typing_status`. | | Enum: [Always Addressed]
Optional: \{\}
| + + #### ClassEnvVar @@ -666,13 +726,14 @@ _Appears in:_ | `timeZone` _string_ | timeZone is the company's clock, an IANA time zone such as
Europe/Berlin: where its days begin for budgets, due dates and
schedules. Unset is UTC. Crewlet's `timezone`. | | MaxLength: 64
MinLength: 1
Pattern: `^[A-Za-z][A-Za-z0-9_+-]*(/[A-Za-z0-9_+-]+)*$`
Optional: \{\}
| | `tracker` _[TrackerSettings](#trackersettings)_ | tracker is which work tracker the company runs. | | MinProperties: 1
Optional: \{\}
| | `knowledge` _[KnowledgeSettings](#knowledgesettings)_ | knowledge is which knowledge base the company runs. | | MinProperties: 1
Optional: \{\}
| +| `chat` _[ChatSettings](#chatsettings)_ | chat is which chat surface the company runs: Chatlet, Crewlet's own,
unless it is connected to Slack or Mattermost. Crewlet's `chat`. | | MinProperties: 1
Optional: \{\}
| | `tokenBudget` _[TokenBudget](#tokenbudget)_ | tokenBudget caps the tokens all seats together may spend per window.
Each window binds every agent seat, so Crewlet refuses one below
turn_engine.executor_min_budget_tokens (25000 unless spec.engineConfig
sets turn_engine): an executor launch needs that much of every window
left, and a seat under a smaller ceiling would never launch one. The
refusal comes from the engine's dry run, Invalid on the Company.
Crewlet's `token_budget`. | | MinProperties: 1
Optional: \{\}
| | `llm` _[CompanyLLMSettings](#companyllmsettings)_ | llm is how the company chooses among its model providers. | | MinProperties: 1
Optional: \{\}
| | `assistant` _[CompanyAssistant](#companyassistant)_ | assistant configures Ask Crewlet. Unset leaves it off.
Crewlet's `assistant`. | | MinProperties: 1
Optional: \{\}
| | `embeddings` _[EmbeddingSettings](#embeddingsettings)_ | embeddings is the provider vectors are computed with. Unset leaves
search keyword-only. Crewlet's providers.embeddings. | | Optional: \{\}
| | `sandbox` _[SandboxSettings](#sandboxsettings)_ | sandbox is where the company's agent seats' executors run. Required
of a company with an Agent seat and a model provider, which Crewlet
refuses without it (SandboxRequired). Crewlet's providers.sandbox. | | Optional: \{\}
| | `integrations` _[CompanyIntegrationSettings](#companyintegrationsettings)_ | integrations is company-wide integration behaviour. | | MinProperties: 1
Optional: \{\}
| -| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, token_budget,
assistant, providers, integrations, mcp_servers, roles, units or
seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's
spec.integrations. Every
position Crewlet holds a credential in lives under one of those keys, so
engineConfig can never carry a credential: the API server refuses a
Company whose engineConfig names one of them, before anything is
stored. | | MaxItems: 64
Optional: \{\}
| +| `engineConfig` _[EngineConfigEntry](#engineconfigentry) array_ | engineConfig is passed through into the company document as written,
for the engine's tuning blocks that have no field here: turn_engine,
learning, scheduling, workers, skill_variables, artifacts and the
notification_* settings, spelled as Crewlet spells them. Each entry is
one top-level key of the document and its value. The operator does not
interpret it; the engine validates it in the dry run before every
apply, and a refusal is reported on the Company.
It may not set what another field or object of this API owns: name,
mission, vision, policies, timezone, tracker, knowledge, chat,
token_budget, assistant, providers, integrations, mcp_servers, roles,
units or seat_access_migrated, which the operator writes with every
integration kind so the engine acts on each Seat's spec.integrations.
Every position Crewlet holds a credential in lives under one of those
keys, so engineConfig can never carry a credential: the API server
refuses a Company whose engineConfig names one of them, before anything
is stored. | | MaxItems: 64
Optional: \{\}
| | `config` _[ConfigSettings](#configsettings)_ | config is how the operator manages the company's document. | \{ managementPolicy:Enforce \} | MinProperties: 1
Optional: \{\}
| | `topology` _[CompanyTopology](#companytopology)_ | topology is how the engine is laid out in pods. | \{ data:map[replicas:1] \} | MinProperties: 1
Optional: \{\}
| | `exposure` _[CompanyExposure](#companyexposure)_ | exposure overrides how the company is reached. | | MinProperties: 1
Optional: \{\}
| @@ -760,9 +821,11 @@ _Appears in:_ | --- | --- | --- | --- | | `revisionID` _string_ | revisionID is the engine's id of the last revision the operator saw
run the rendered document: the one it applied, one the engine
activated with the same content (a reload), or, under Observe, the
one it found in sync. Under Seed it is the revision the document was
seeded as. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| | `epoch` _integer_ | epoch is the engine's activation counter for that revision. The
engine applies a revision asynchronously; it runs it once the applied
epoch its /health reports has reached this. | | Minimum: 0
Optional: \{\}
| -| `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as last applied: a change to
either is applied as a new revision, since the engine reads a
rotated value only when a revision activates. The values enter it
only as keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| -| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator last applied a revision. | | Optional: \{\}
| +| `appliedHash` _string_ | appliedHash is the hash of the rendered document and of the values
of the Secret keys it references, as the revision recorded runs
them: the operator's last write, or a revision it found running that
document after every value it wrote. The values enter it only as
keyed fingerprints, so it reveals nothing about them. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| +| `appliedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | appliedAt is when the operator activated the revision this records,
or found the engine running it where it adopted a revision of the
rendered document it did not write (a reload), or a write of its own
whose activation the engine could not confirm (unconfirmed), which may
have landed any time since it was written: what the operator's wait
for the engine to apply its epoch counts from. | | Optional: \{\}
| | `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from. | | MinProperties: 1
Optional: \{\}
| +| `unconfirmed` _[UnconfirmedRevision](#unconfirmedrevision) array_ | unconfirmed are the revisions of the rendered document the operator
wrote, which the engine stored, and whose activation the engine sent
to its fleet and could not confirm (a 503 naming the revision it
stored): the fleet may run one of them or none. Before it writes
again, the operator reads which revision the engine runs. If it runs
one of these, that write landed: status.config records it as the
operator's write, as though the engine had confirmed it. If it runs the
revision one of them was built on, that one has not landed and still
may, so it stays, and the write is sent again on the same condition,
which the engine refuses once one of them lands. Any other can no
longer land, and goes, to overtaken where it was written into an
engine that ran no document. A reload the operator asks for, to
activate a revision of its own again, is recorded here too when the
engine stores it and cannot confirm activating it, as a write of that
revision's document, whether or not a read after it could say which
revision runs. They are oldest first, and a seventeenth drops the
oldest. | | MaxItems: 16
Optional: \{\}
| +| `overtaken` _[UnconfirmedRevision](#unconfirmedrevision) array_ | overtaken are writes from unconfirmed into an engine that ran no
document that another revision the engine activated has overtaken.
None of them can land any more, and under Seed one of them is taken as
the seed whatever revision runs now: the engine stored it while it had
none, so either it landed and was replaced since, or another writer's
create reached the engine first, which nothing the engine says tells
apart. Each is kept until a reconcile decides with it, and they go
once status.config records a revision of the rendered document in
sync, a write of it, or the seed. They are oldest first, and a
seventeenth drops the oldest. | | MaxItems: 16
Optional: \{\}
| | `placement` _[RevisionPlacement](#revisionplacement)_ | placement is where the revision the engine runs pins its agent seats,
read from the engine's document under every management policy. The
operator keeps a seat pool's pods running while the running revision
pins a seat to them, even after the pool or the seat's placement is
gone from the spec: a pod stopped while its seat is still pinned to
it would leave the seat held by nobody, and a blocked document keeps
the last good revision running. Once a revision that pins nothing to
them runs, the pods are deleted. | | MinProperties: 1
Optional: \{\}
| @@ -2916,6 +2979,25 @@ _Appears in:_ | `Atlassian` | | +#### ReplacedRevision + +_Underlying type:_ _string_ + +ReplacedRevision is what a write of the company document replaced where +that was more than the operator's own document. + +_Validation:_ +- Enum: [Unsealed Drifted] + +_Appears in:_ +- [UnconfirmedRevision](#unconfirmedrevision) + +| Field | Description | +| --- | --- | +| `Unsealed` | ReplacedUnsealed is a revision the engine stores without a seal, which
nobody can read and a whole document replaces.
| +| `Drifted` | ReplacedDrifted is a revision the operator did not write whose
document differed from the rendered one.
| + + #### RetentionSettings @@ -3691,7 +3773,7 @@ _Appears in:_ | `integrations` _[SeatIntegrations](#seatintegrations)_ | integrations is the seat's own identity on the company's
integrations. | | MinProperties: 1
Optional: \{\}
| | `project` _string_ | project is the tracker project a seat outside every unit owns, by its
key; a seat in a unit takes its unit's. Crewlet requires a container
key to be an upper-case letter followed by 1 to 9 upper-case letters or
digits, such as LEAD. Crewlet's `project`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| | `space` _string_ | space is the knowledge container a seat outside every unit owns, by
its key, which follows the same rule as project; a seat in a unit
takes its unit's. Crewlet's `space`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| -| `contact` _[SeatContact](#seatcontact)_ | contact is how agents reach the person in a Human seat on a chat
surface, and how what the person does there reaches the engine as
theirs. It applies to Human seats only, and is optional: a person who
works only through the engine's dashboard has no account to name, and
signs in on the seat through the engine's own identity directory. The
engine warns of a Human seat without one, since nothing addressed to it
on a chat surface reaches anybody, and the operator records the warning
as an EngineWarnings event on the Company. | | MinProperties: 1
Optional: \{\}
| +| `contact` _[SeatContact](#seatcontact)_ | contact is how agents reach the person in a Human seat on a vendor's
chat surface or tool, and how what the person does there reaches the
engine as theirs. It applies to Human seats only, and is optional: a
person who works only through the engine's dashboard has no account to
name, and signs in on the seat through the engine's own identity
directory. On a company that talks on Slack or Mattermost (an
Integration of either, or a seat's Slack app) the engine warns of a
Human seat with no account there, a slackUserID or a
mattermostUsername, since nothing addressed to it there reaches
anybody; a GitHub, GitLab or Atlassian account does not count, and the
operator records the warning as an EngineWarnings event on the
Company. On Chatlet, which a company with neither talks on, the person
reads the rooms on the dashboard as their seat, and nothing is warned. | | MinProperties: 1
Optional: \{\}
| | `availability` _string_ | availability tells colleagues of a Human seat when to expect an
answer, such as "CET business hours". Crewlet's `availability`. | | MaxLength: 256
MinLength: 1
Optional: \{\}
| @@ -4086,6 +4168,30 @@ _Appears in:_ | `worklet` _[WorkletSettings](#workletsettings)_ | worklet is the policy of Worklet, for backend Worklet. | | MinProperties: 1
Optional: \{\}
| +#### UnconfirmedRevision + + + +UnconfirmedRevision is a write of the company document whose activation the +engine could not confirm. + + + +_Appears in:_ +- [ConfigStatus](#configstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `revisionID` _string_ | revisionID is the revision the engine stored for the write. | | MaxLength: 128
MinLength: 1
Required: \{\}
| +| `baseRevisionID` _string_ | baseRevisionID is the revision the write was built on, which its
activation was conditioned on. Empty for a write into an engine that
ran no document, conditioned on there being none. | | MaxLength: 128
MinLength: 1
Optional: \{\}
| +| `appliedHash` _string_ | appliedHash is the appliedHash of the write: the rendered document's
and its values' hash as they were when it was written, which
status.config.appliedHash records once the engine is seen running the
revision. | | MaxLength: 128
MinLength: 1
Required: \{\}
| +| `writtenAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#time-v1-meta)_ | writtenAt is when the operator wrote it. A write the engine is seen
running is recorded as applied when the operator found it, since
nothing says when its activation landed; a seed taken from overtaken,
which nothing waits on, as applied when it was written. | | Required: \{\}
| +| `objects` _[AppliedObjectCounts](#appliedobjectcounts)_ | objects counts the objects the revision was rendered from, which
status.config.objects records once the engine is seen running it. | | MinProperties: 1
Optional: \{\}
| +| `replaced` _[ReplacedRevision](#replacedrevision)_ | replaced says what the write replaced where that was more than the
operator's own document, which ConfigApplied and a Warning event report
once the engine is seen running the revision, as they report a write the
engine confirmed: Unsealed, a revision the engine stores without a seal,
which the write replaces reading nothing from it; Drifted, a revision
the operator did not write whose document differed from the rendered
one, at driftedAt. Unset for any other base, and for a reload: its base
is a write of the operator's, and where that write is recorded here too,
the revision found running reports what that write replaced, once,
whichever of the two it is. | | Enum: [Unsealed Drifted]
Optional: \{\}
| +| `driftedAt` _string_ | driftedAt names where the Drifted revision the write replaced differed
from the rendered document, as ConfigApplied names it: up to five
places, and how many more there were. | | MaxLength: 32768
MinLength: 1
Optional: \{\}
| +| `documentSHA256` _string_ | documentSHA256 is set for a reload: the SHA-256, in hex, of the
document of the revision it activates again, as the engine answered
it. The engine activates again whichever revision is active when the
request lands, so the revision it stored is the operator's only where
its document is that one; one found running another document was a
reload of a revision somebody activated in between, and goes. | | MaxLength: 64
MinLength: 64
Pattern: `^[0-9a-f]\{64\}$`
Optional: \{\}
| + + #### Unit @@ -4132,7 +4238,7 @@ _Appears in:_ | `leadRef` _[LocalObjectReference](#localobjectreference)_ | leadRef names the Seat that leads the unit. Unset inherits the
parent's lead. Crewlet's `lead`. | | Optional: \{\}
| | `purpose` _string_ | purpose is what the unit is for. Crewlet's `purpose`. | | MaxLength: 2048
MinLength: 1
Optional: \{\}
| | `goals` _string array_ | goals are what the unit is working toward. Crewlet's `goals`. | | MaxItems: 32
items:MaxLength: 1024
items:MinLength: 1
Optional: \{\}
| -| `channel` _string_ | channel is the unit's channel on the company's chat surface, by name,
such as platform; child units without one inherit it. On Chatlet, the
engine's own chat and the surface of a company that declares no
Mattermost or Slack Integration, each channel a unit names is a room
whose members follow the org chart. Crewlet takes lowercase letters,
digits, - and _, starting with a letter or a digit, up to 64
characters: the name itself, never #platform or a vendor's channel id.
Crewlet's `channel`. | | MaxLength: 64
MinLength: 1
Pattern: `^[a-z0-9][a-z0-9_-]\{0,63\}$`
Optional: \{\}
| +| `channel` _string_ | channel is the unit's channel on the company's chat surface, by name,
such as platform; child units without one inherit it. On Chatlet, the
engine's own chat and the surface of a company that declares no
Mattermost or Slack Integration, each channel a unit names is a room
whose members follow the org chart. Crewlet takes lowercase letters,
digits, - and _, starting with a letter or a digit, up to 64
characters: the name itself, never #platform or a vendor's channel id.
On Chatlet a room belongs to one unit, so Crewlet refuses a second unit
naming the same channel; a child unit that should share its parent's
room names none. Crewlet's `channel`. | | MaxLength: 64
MinLength: 1
Pattern: `^[a-z0-9][a-z0-9_-]\{0,63\}$`
Optional: \{\}
| | `knowledge` _string array_ | knowledge are free-text references to knowledge the unit relies on.
They are not a read scope. Crewlet's `knowledge`. | | MaxItems: 32
items:MaxLength: 1024
items:MinLength: 1
Optional: \{\}
| | `project` _string_ | project is the tracker project the unit files work in, by its key.
Crewlet requires a container key to be an upper-case letter followed
by 1 to 9 upper-case letters or digits, such as PLAT. Crewlet's
`project`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| | `space` _string_ | space is the knowledge container the unit writes pages in, by its
key, which follows the same rule as project. Crewlet's `space`. | | MaxLength: 10
MinLength: 2
Pattern: `^[A-Z][A-Z0-9]\{1,9\}$`
Optional: \{\}
| @@ -4248,13 +4354,14 @@ _Validation:_ - Enum: [Always Addressed] _Appears in:_ +- [ChatletSettings](#chatletsettings) - [MattermostIntegration](#mattermostintegration) - [SlackIntegration](#slackintegration) | Field | Description | | --- | --- | | `Always` | WorkingStatusAlways shows it on every turn a chat message started.
| -| `Addressed` | WorkingStatusAddressed shows it only when a person is plausibly
waiting: a direct message, a mention, or a thread the seat follows.
| +| `Addressed` | WorkingStatusAddressed shows it only when a person is plausibly
waiting on the seat: a direct message, a mention of it by name, and
on Slack or Mattermost a thread it is in because it was named there,
on Chatlet a reply in a thread it started or a person's message in a
unit's room that reached nobody else and so falls to it as the unit's
lead. Never a message it reached through a room-wide address, or only
by having spoken in the thread before.
| #### WorkletSettings diff --git a/internal/controller/company_runtime.go b/internal/controller/company_runtime.go index cf83b945..08585a0a 100644 --- a/internal/controller/company_runtime.go +++ b/internal/controller/company_runtime.go @@ -177,7 +177,7 @@ func (r *CompanyReconciler) reconcileWorkload( } if len(in.missing) > 0 { return holdBack(crewletaiv1alpha1.ReasonSecretNotFound, - "The engine's pods cannot be given the values they read from Secrets: "+strings.Join(in.missing, "; ")+".") + ended("The engine's pods cannot be given the values they read from Secrets: "+strings.Join(in.missing, "; "))) } return r.runWorkload(ctx, c, class, in, existing, pools) } @@ -234,6 +234,15 @@ func (r *CompanyReconciler) runWorkload( // pods keep the one they run, which named an address it took. return holdBack(crewletaiv1alpha1.ReasonAddressRefused, addressRefusalMessage(c, class, refused)) } + if small, ok := errors.AsType[*render.VolumeError](err); ok { + // The engine could not reserve its state logs on this stream at a + // start with a log to create, so the pods keep the one they run: an + // engine whose logs the stream already holds runs on, and one that + // already refuses it at least has its cause named here rather than + // in a restart loop's previous log. + return holdBack(crewletaiv1alpha1.ReasonDataVolumeTooSmall, + volumeTooSmallMessage(c, class, existing != nil || liveClaim(claims), small)) + } if errors.Is(err, render.ErrSignInNotAcknowledged) { // The engine would refuse this crewlet.yaml at every start, so the // pods keep the one they run, which requires a second factor or @@ -318,7 +327,8 @@ func (r *CompanyReconciler) runWorkload( } return suspendedCondition(""), nil case refusal != nil: - return runtimeCondition(metav1.ConditionFalse, refusal.reason, refusal.message+" The data pods keep running "+image+"."), nil + return runtimeCondition(metav1.ConditionFalse, refusal.reason, + refusal.message+" "+ended("The data pods keep running "+image)), nil case progress.ready: return runtimeCondition(metav1.ConditionTrue, crewletaiv1alpha1.ReasonPodsReady, progress.message), nil default: diff --git a/internal/controller/company_runtime_test.go b/internal/controller/company_runtime_test.go index 94855c8f..d066b321 100644 --- a/internal/controller/company_runtime_test.go +++ b/internal/controller/company_runtime_test.go @@ -1128,6 +1128,78 @@ var _ = Describe("Company runtime", func() { "the startup window covers the recovery of the volume the pod has") }) + It("holds the pods back on a data volume under what a fresh node reserves, and runs them once its claim grows", func() { + class := newClass() + company := reconciled(newCompany(class.Name)) + // A company claimed under an older floor: its StatefulSet as the + // operator made it then, with a 12Gi claim template. A template + // cannot change, so the StatefulSet is made again around one, as + // the operator would have made it. (envtest runs no garbage + // collector, so deleting it leaves nothing else to go with it.) + made := statefulSet() + old := made.DeepCopy() + old.ObjectMeta = metav1.ObjectMeta{ + Namespace: made.Namespace, Name: made.Name, Labels: made.Labels, Annotations: made.Annotations, + OwnerReferences: made.OwnerReferences, + } + old.Status = appsv1.StatefulSetStatus{} + old.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("12Gi") + Expect(k8sClient.Delete(ctx, made)).To(Succeed()) + Eventually(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(made), &appsv1.StatefulSet{})) + }).Should(BeTrue()) + Expect(k8sClient.Create(ctx, old)).To(Succeed()) + running := statefulSet() + bootstrap := &corev1.Secret{} + get(bootstrap, bootstrapName) + tierA := bootstrap.Data[configFile] + + By("holding the pods as they are, naming the volume and what clears it") + company = reconciled(company) + runtimeReady := condition(company, crewletaiv1alpha1.ConditionRuntimeReady) + Expect(runtimeReady.Status).To(Equal(metav1.ConditionFalse)) + Expect(runtimeReady.Reason).To(Equal(crewletaiv1alpha1.ReasonDataVolumeTooSmall)) + Expect(runtimeReady.Message).To(ContainSubstring("The data volume is 12Gi")) + Expect(runtimeReady.Message).To(ContainSubstring("is 9663676416 bytes")) + Expect(runtimeReady.Message).To(ContainSubstring("reserves 10826896725 bytes")) + Expect(runtimeReady.Message).To(ContainSubstring("Grow the claim " + dataClaimName + " to at least 14Gi")) + Expect(statefulSet().Spec.Template).To(Equal(running.Spec.Template), "the pods rolled onto a crewlet.yaml the engine refuses") + get(bootstrap, bootstrapName) + Expect(bootstrap.Data[configFile]).To(Equal(tierA), "the bootstrap Secret was rewritten") + + By("running them on the stream of the claim its owner grew, once the expansion lands") + expandable := &storagev1.StorageClass{ + Name: "expandable-" + namespace, + Provisioner: "example.com/expandable", + AllowVolumeExpansion: new(true), + } + Expect(k8sClient.Create(ctx, expandable)).To(Succeed()) + DeferCleanup(func() { Expect(k8sClient.Delete(ctx, expandable)).To(Succeed()) }) + claim := &corev1.PersistentVolumeClaim{ + Namespace: namespace, + Name: dataClaimName, + Labels: labelsFor(company, componentData), + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &expandable.Name, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("14Gi")}, + }, + }, + } + Expect(k8sClient.Create(ctx, claim)).To(Succeed()) + claim.Status.Phase = corev1.ClaimBound + claim.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("14Gi")} + Expect(k8sClient.Status().Update(ctx, claim)).To(Succeed()) + company = reconciled(company) + Expect(condition(company, crewletaiv1alpha1.ConditionRuntimeReady).Reason).To(Equal(crewletaiv1alpha1.ReasonPodsNotReady)) + get(bootstrap, bootstrapName) + Expect(string(bootstrap.Data[configFile])).To(ContainSubstring("store_max_bytes: 11274289152"), + "the event stream is three quarters of the 14Gi the claim now has") + Expect(statefulSet().Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().String()).To(Equal("12Gi"), + "the template stays as it was made") + }) + It("rolls the pods when a value they read at start changes, and only then", func() { class := newClass() reconciled(newCompany(class.Name)) diff --git a/internal/controller/company_workload.go b/internal/controller/company_workload.go index 30c6a492..8b402b7a 100644 --- a/internal/controller/company_workload.go +++ b/internal/controller/company_workload.go @@ -931,6 +931,42 @@ func applyConfigOf[T any](v any) (*T, error) { return out, nil } +// volumeTooSmallMessage is RuntimeReady's account of a data volume whose +// share, the engine's event stream, is under what a fresh node of Crewlet +// reserves on it (small), with what clears it, which depends on where the volume's size +// comes from (dataVolumeSize). Where the company has a claim, or a data +// StatefulSet that makes one from a template that never changes (claimed), +// the claim is what is grown. Where it has neither, the volume is the one its +// class would give a new company, which a class stored before the floor rose +// still can, since the API server's ratcheting keeps a stored value it no +// longer admits: there is no claim to grow, so the class's size is what is +// raised. +func volumeTooSmallMessage( + c *crewletaiv1alpha1.Company, class *crewletaiv1alpha1.EngineClass, claimed bool, small *render.VolumeError, +) string { + sizes := fmt.Sprintf("three quarters of it, the engine's event stream, is %d bytes: a fresh node of Crewlet "+ + "reserves %d bytes of it for its state logs, and refuses to boot without them, so a data volume needs at "+ + "least %s.", small.Stream, small.Floor, small.Least.String()) + if !claimed { + return fmt.Sprintf("EngineClass %s gives a new company a data volume of %s, and %s Set the class's "+ + "spec.data.storage.size to at least %s, and the operator creates the company's data volume at that size. "+ + "Nothing of the company's data pods is created until then.", + class.Name, small.Volume.String(), sizes, small.Least.String()) + } + return fmt.Sprintf("The data volume is %s, and %s Grow the claim %s to at least %s (its StorageClass must allow "+ + "volume expansion; see \"Growing a company's data volume\" in the operator's docs/concepts/runtime.md), and the "+ + "operator sizes the stream from it once the expansion lands. The pods are held as they are.", + small.Volume.String(), sizes, dataVolume+"-"+dataStatefulSetName(c)+"-0", small.Least.String()) +} + +// liveClaim reports whether claims holds a data volume claim not being +// deleted: one a pod runs on, or will, whatever made it. +func liveClaim(claims []corev1.PersistentVolumeClaim) bool { + return slices.ContainsFunc(claims, func(claim corev1.PersistentVolumeClaim) bool { + return claim.DeletionTimestamp.IsZero() + }) +} + // existingStatefulSet returns the data StatefulSet, or nil when there is none. // The cache holds only the StatefulSets the operator manages (CacheOptions), // so when it has none the API server is asked: a StatefulSet of that name @@ -975,6 +1011,14 @@ func (r *CompanyReconciler) statefulSetNamed(ctx context.Context, namespace, nam // what its engine reserves (made under an older floor, or before an engine // that keeps more on its stream) had no way out: the template cannot change, // and an expanded claim left the stream as small as it was. +// +// A claim that asks for less than the template is the exception, since the +// template did not make it: a StatefulSet makes each claim at its template's +// size, and a claim's request only ever grows. It is one a Company deleted +// under spec.deletionPolicy Retain kept, which a Company created again under +// its name runs on as it is, whatever its class gives a new volume now, so +// it counts at what it asks for, or at what it was given if less. Counted at +// the template, the stream was sized for a volume the pod does not have. func dataVolumeSize( existing *appsv1.StatefulSet, class *crewletaiv1alpha1.EngineClass, claims []corev1.PersistentVolumeClaim, ) resource.Quantity { @@ -997,7 +1041,14 @@ func dataVolumeSize( size := template requested, asked := claim.Spec.Resources.Requests[corev1.ResourceStorage] granted, bound := claim.Status.Capacity[corev1.ResourceStorage] - if asked && bound { + switch { + case asked && requested.Cmp(template) < 0: + // A kept claim, which the template did not make. + size = requested + if bound && granted.Cmp(size) < 0 { + size = granted + } + case asked && bound: grown := requested if granted.Cmp(grown) < 0 { grown = granted diff --git a/internal/controller/company_workload_test.go b/internal/controller/company_workload_test.go index 21632c45..a59d9b57 100644 --- a/internal/controller/company_workload_test.go +++ b/internal/controller/company_workload_test.go @@ -134,11 +134,15 @@ func TestSeatPodStartupSeconds(t *testing.T) { // to the volumes the data pods actually have: the claim template's size (the // existing StatefulSet's, or the class's before there is one), grown by an // expansion of every claim once the expansion has landed, never shrunk by a -// claim given less than it asked for, and the smallest across the pods. +// claim given less than it asked for, and the smallest across the pods; and a +// claim the template did not make, which asks for less than it, at the size +// that claim has. func TestDataVolumeSize(t *testing.T) { t.Parallel() - // kept is the size the existing StatefulSet's claim template asks for. - const kept = "16Gi" + // kept is the size the existing StatefulSet's claim template asks for, + // and older the size of a claim a deleted Company's StatefulSet made from + // a smaller template, which a Company created again under its name runs on. + const kept, older = "16Gi", "12Gi" class := &crewletaiv1alpha1.EngineClass{} class.Spec.Data.Storage.Size = new(resource.MustParse("32Gi")) existing := &appsv1.StatefulSet{} @@ -174,6 +178,12 @@ func TestDataVolumeSize(t *testing.T) { []corev1.PersistentVolumeClaim{bound("64Gi", "64Gi"), bound(kept, kept)}, kept, }, {"a claim being deleted", existing, []corev1.PersistentVolumeClaim{deleting}, kept}, + {"a kept claim under the template", existing, []corev1.PersistentVolumeClaim{bound(older, older)}, older}, + {"a kept claim before there is a StatefulSet", nil, []corev1.PersistentVolumeClaim{bound(older, older)}, older}, + {"a kept claim rounded up", existing, []corev1.PersistentVolumeClaim{bound(older, "14Gi")}, older}, + {"a kept claim given less than it asks", existing, []corev1.PersistentVolumeClaim{bound(older, "10Gi")}, "10Gi"}, + {"a kept claim not bound yet", existing, []corev1.PersistentVolumeClaim{claimRequesting(older)}, older}, + {"a kept claim expanded past the template", existing, []corev1.PersistentVolumeClaim{bound("20Gi", "20Gi")}, "20Gi"}, } { t.Run(c.name, func(t *testing.T) { t.Parallel() @@ -184,6 +194,71 @@ func TestDataVolumeSize(t *testing.T) { } } +// TestAVolumeUnderTheFloorNamesWhatToGrow holds DataVolumeTooSmall's message +// to the remedy that exists. A company with a data claim, or a StatefulSet +// that makes one, is told to grow that claim. One with neither has its volume +// from its class, a class stored before the floor rose (which the API +// server's ratcheting keeps), and is told to raise the class's size: there is +// no claim to grow, and naming one sent its owner looking for a claim that +// does not exist. +func TestAVolumeUnderTheFloorNamesWhatToGrow(t *testing.T) { + t.Parallel() + company := &crewletaiv1alpha1.Company{Namespace: "tenant-a", Name: companyName} + class := &crewletaiv1alpha1.EngineClass{Name: "small"} + small := &render.VolumeError{Volume: resource.MustParse("12Gi"), Stream: 9663676416, Floor: 10826896725, + Least: resource.MustParse("14Gi")} + for _, c := range []struct { + name string + claimed bool + says, saysNot []string + }{ + { + name: "a company with a claim", claimed: true, + says: []string{"The data volume is 12Gi", "Grow the claim data-" + companyName + "-data-0 to at least 14Gi", "held as they are"}, + saysNot: []string{"EngineClass", "spec.data.storage.size"}, + }, + { + name: "a new company of a class under the floor", + says: []string{"EngineClass small gives a new company a data volume of 12Gi", + "spec.data.storage.size to at least 14Gi", "is 9663676416 bytes", "reserves 10826896725 bytes"}, + saysNot: []string{"Grow the claim", "data-" + companyName + "-data-0"}, + }, + } { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + message := volumeTooSmallMessage(company, class, c.claimed, small) + for _, s := range c.says { + if !strings.Contains(message, s) { + t.Errorf("the message does not say %q: %s", s, message) + } + } + for _, s := range c.saysNot { + if strings.Contains(message, s) { + t.Errorf("the message says %q: %s", s, message) + } + } + if !endsOnce(message) { + t.Errorf("the message does not end with one full stop: %s", message) + } + }) + } + deleting := claimRequesting("12Gi") + deleting.DeletionTimestamp = new(metav1.Now()) + for _, c := range []struct { + name string + claims []corev1.PersistentVolumeClaim + live bool + }{ + {"no claim", nil, false}, + {"a claim being deleted", []corev1.PersistentVolumeClaim{deleting}, false}, + {"a claim", []corev1.PersistentVolumeClaim{deleting, claimRequesting("12Gi")}, true}, + } { + if got := liveClaim(c.claims); got != c.live { + t.Errorf("liveClaim of %s = %t, want %t", c.name, got, c.live) + } + } +} + // claimRequesting is a data volume claim that asks for size, not bound. func claimRequesting(size string) corev1.PersistentVolumeClaim { return corev1.PersistentVolumeClaim{ diff --git a/internal/controller/config_apply.go b/internal/controller/config_apply.go index c0ad28f9..43831340 100644 --- a/internal/controller/config_apply.go +++ b/internal/controller/config_apply.go @@ -42,7 +42,11 @@ const enginePollInterval = 18 * time.Second const engineLagGrace = 45 * time.Second // applyDeadline is how long after a write the operator waits for an engine -// whose posture still reads wait. A node reports wait for the whole of an +// whose posture still reads wait, or after it first finds the engine running +// a revision of the rendered document it did not write, which it adopts +// (inSync), or a write of its own whose activation the engine could not +// confirm, which may have landed any time since (confirmUnconfirmed). A node +// reports wait for the whole of an // apply, and turns serve when it succeeds or a diverged posture when an // attempt fails, so wait past this is one of two things the engine itself // never concludes: it is not polling its activation pointer at all, or an @@ -115,7 +119,9 @@ func (r *CompanyConfigReconciler) clock() time.Time { // reports about it: True once the engine has applied the epoch status.config // records and has not diverged since, Applying while its posture reads wait, // and ApplyFailed when its posture reads diverged, or still wait -// applyDeadline after the write. It runs on every look at the engine, the +// applyDeadline after the write, or after the operator found the revision it +// adopted, or a write of its own whose activation the engine could not +// confirm (status.config.appliedAt). It runs on every look at the engine, the // periodic resync included, so an engine that diverges after applying the // operator's revision, on one someone activates later, is reported too. // health and healthErr are what readHealth read; result is what the @@ -133,10 +139,7 @@ func (run *configRun) verifyApplied( return result } key := types.NamespacedName{Namespace: run.company.Namespace, Name: run.company.Name} - config := run.company.Status.Config - if run.config != nil { - config = *run.config - } + config := run.recordedConfig() var target int64 if config.Epoch != nil { target = *config.Epoch @@ -156,8 +159,9 @@ func (run *configRun) verifyApplied( return result case config.AppliedAt != nil && run.clock().Sub(config.AppliedAt.Time) >= applyDeadline: run.applyFailed(config.RevisionID, fmt.Sprintf("The engine has not applied revision %s (epoch %d) %s after the "+ - "operator activated it, and has not concluded that it failed: it reports status %q and posture %q at applied "+ - "epoch %d, and keeps running that. It is not polling its activation pointer, or an apply hangs; its log says which.", + "operator activated it or found it activated, and has not concluded that it failed: it reports status %q and "+ + "posture %q at applied epoch %d, and keeps running that. It is not polling its activation pointer, or an apply "+ + "hangs; its log says which.", config.RevisionID, target, applyDeadline, health.Status, health.Posture, health.AppliedEpoch)) return ctrl.Result{RequeueAfter: engineRetryInterval} } @@ -184,8 +188,8 @@ func (run *configRun) failureAccount(ctx context.Context, eng *engine.Client, he health.Status, health.Posture, health.AppliedEpoch) fleet, err := run.fleetOf(ctx, eng) if err != nil { - return fmt.Sprintf("The engine reports that it did not apply the activated epoch, and its record of the attempt "+ - "could not be read: %v.", err) + runs + return ended(fmt.Sprintf("The engine reports that it did not apply the activated epoch, and its record of the attempt "+ + "could not be read: %v", err)) + runs } record := fleet.LastApply() if record.Serving != "" { @@ -197,7 +201,7 @@ func (run *configRun) failureAccount(ctx context.Context, eng *engine.Client, he if account == "" { account = "the engine gave no account of the failure" } - return fmt.Sprintf("The engine failed to apply epoch %d (revision %s): %s.", record.Epoch, record.RevisionID, account) + runs + return ended(fmt.Sprintf("The engine failed to apply epoch %d (revision %s): %s", record.Epoch, record.RevisionID, account)) + runs } epoch := max(fleet.TargetEpoch, target) return fmt.Sprintf("The engine reports that it did not apply epoch %d and records no failed attempt at it: its apply "+ @@ -233,7 +237,7 @@ func (run *configRun) settled(key types.NamespacedName, target int64, revision s run.recordPlacement(*run.running, true) } current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied) - if run.wrote || (current != nil && + if run.wrote || run.confirmed != "" || (current != nil && (current.Reason == crewletaiv1alpha1.ReasonApplying || current.Reason == crewletaiv1alpha1.ReasonApplyFailed)) { run.event(corev1.EventTypeNormal, crewletaiv1alpha1.ReasonApplied, "ApplyConfig", "The engine applied revision %s (epoch %d).", revision, target) diff --git a/internal/controller/config_apply_test.go b/internal/controller/config_apply_test.go index 75c23f70..2d1549d4 100644 --- a/internal/controller/config_apply_test.go +++ b/internal/controller/config_apply_test.go @@ -23,10 +23,11 @@ import ( // never puts the real engine in. The real engine's ordinary path, an apply // that reaches ConfigApplied=True, is the envtest suite's (config_test.go). -// fakeEngine answers /health and /fleet with the bodies it holds, and counts -// the /fleet reads. +// fakeEngine answers /health and /fleet with the bodies it holds, /fleet +// with fleetStatus where it is set, and counts the /fleet reads. type fakeEngine struct { health, fleet string + fleetStatus int fleetReads int } @@ -39,6 +40,9 @@ func (f *fakeEngine) client(t *testing.T) *engine.Client { _, _ = w.Write([]byte(f.health)) case fleetRoute: f.fleetReads++ + if f.fleetStatus != 0 { + w.WriteHeader(f.fleetStatus) + } _, _ = w.Write([]byte(f.fleet)) default: http.NotFound(w, r) diff --git a/internal/controller/config_fleet.go b/internal/controller/config_fleet.go index 5228ce54..1d8722bf 100644 --- a/internal/controller/config_fleet.go +++ b/internal/controller/config_fleet.go @@ -153,7 +153,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { setCondition(conditions, gen, crewletaiv1alpha1.ConditionReady, false, crewletaiv1alpha1.ReasonSuspended, fmt.Sprintf("Company %s is suspended: no engine node runs its seats, and their messages wait.", run.company.Name)) case p.executor != nil && p.executor.Reason != "": - message := p.executor.Field + ": " + p.executor.Message + "." + message := ended(p.executor.Field + ": " + p.executor.Message) lease, held := engine.SeatLease{}, false if run.fleet != nil { lease, held = run.leaseOf(handle) @@ -174,7 +174,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { s.Status.Placement.Node = lease.Node message := heldMessage(lease, workload) if p.executor != nil { - message += " It " + p.executor.Message + "." + message += " " + ended("It "+p.executor.Message) } setCondition(conditions, gen, crewletaiv1alpha1.ConditionReady, true, crewletaiv1alpha1.ReasonLeaseHeld, message) return @@ -192,7 +192,7 @@ func (run *configRun) decideReady(p *part, s *crewletaiv1alpha1.Seat) { s.Status.Placement.Node = p.base.(*crewletaiv1alpha1.Seat).Status.Placement.Node meta.SetStatusCondition(conditions, metav1.Condition{ Type: crewletaiv1alpha1.ConditionReady, Status: metav1.ConditionUnknown, Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: bounded("The engine's fleet, which says which node holds the seat, could not be read: " + run.fleetErr.Error()), + Message: bounded(ended("The engine's fleet, which says which node holds the seat, could not be read: " + run.fleetErr.Error())), ObservedGeneration: gen, }) default: diff --git a/internal/controller/config_fleet_test.go b/internal/controller/config_fleet_test.go index 0a25677e..9c727faf 100644 --- a/internal/controller/config_fleet_test.go +++ b/internal/controller/config_fleet_test.go @@ -12,11 +12,8 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/engine" @@ -215,19 +212,12 @@ func TestSeatReadyFollowsTheFleet(t *testing.T) { // reconcile that would not have looked again. func TestPlacementResyncPollsOnlyWhileSeatsSettle(t *testing.T) { const workload = fleetWorkload - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } statefulSet := func(ready int32) *appsv1.StatefulSet { return fleetStatefulSet(workload, ready) } start := time.Date(2026, 10, 9, 8, 0, 0, 0, time.UTC) reconciler := &CompanyConfigReconciler{ResyncInterval: time.Minute} look := func(at time.Time, ready int32, fleet *engine.Fleet, result ctrl.Result) time.Duration { t.Helper() - reconciler.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(statefulSet(ready)).Build() + reconciler.Client = fakeClient().WithObjects(statefulSet(ready)).Build() reconciler.now = func() time.Time { return at } run, _ := seatRun("") run.CompanyConfigReconciler = reconciler @@ -305,10 +295,6 @@ func TestSeatsPastAPodsShareAreSettling(t *testing.T) { workload = fleetWorkload seatBo = "bo" ) - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } nodes := func(n int) []engine.FleetNode { var out []engine.FleetNode for i := range n { @@ -340,7 +326,7 @@ func TestSeatsPastAPodsShareAreSettling(t *testing.T) { bo.Name, bo.Status.Handle = seatBo, seatBo run.parts = append(run.parts, newPart(bo)) bo.Status.Placement = ana.obj.(*crewletaiv1alpha1.Seat).Status.Placement - run.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(fleetStatefulSet(workload, c.ready)).Build() + run.Client = fakeClient().WithObjects(fleetStatefulSet(workload, c.ready)).Build() run.fleet = c.fleet got, err := run.settlingSeats(context.Background()) if err != nil { diff --git a/internal/controller/config_move.go b/internal/controller/config_move.go index 59a8546f..a0e39a2b 100644 --- a/internal/controller/config_move.go +++ b/internal/controller/config_move.go @@ -42,7 +42,10 @@ const placementHoldInterval = 5 * time.Second // holdMoves keeps each agent seat that moves to a pool with no ready pod // pinned where the engine runs it, re-rendering the document with those pins // when there is any. active is the engine's document, nil when it has none: -// a seat the engine does not run yet has nowhere to stay. +// a seat the engine does not run yet has nowhere to stay. It is nil too where +// the engine runs a revision stored without a seal, which the write replaces +// (configRun.unsealed): where that revision pins a seat is what nobody can +// read, and the engine shows no other revision in its place. func (run *configRun) holdMoves(ctx context.Context, active *engine.ActiveConfig) error { if active == nil { return nil diff --git a/internal/controller/config_optional.go b/internal/controller/config_optional.go index 6d787214..fa72e64f 100644 --- a/internal/controller/config_optional.go +++ b/internal/controller/config_optional.go @@ -192,7 +192,8 @@ type leftOutValue struct { // leftOutLines are the lines of values, each saying too where the engine's // store keeps the value it held for a revision an engine pod runs or starts -// on (run.retained). +// on, or for the engine's active revision where it stores that one without a +// seal (run.retained). func (run *configRun) leftOutLines(values []leftOutValue) []string { out := make([]string, len(values)) for i, v := range values { @@ -200,8 +201,14 @@ func (run *configRun) leftOutLines(values []leftOutValue) []string { switch { case !slices.Contains(run.retained, v.key): case run.keptPlainly(): - out[i] += "; the engine keeps the value it held for the revision it runs, which " + - strings.TrimPrefix(run.keptBecause(false, "the key"), "the revision the engine runs ") + + held, subject := "the revision it runs", "the revision the engine runs " + if run.keptFor[0].unsealed { + // The engine's active revision, one stored without a seal, + // which its nodes do not run. + held, subject = "its active revision", "the engine's active revision " + } + out[i] += "; the engine keeps the value it held for " + held + ", which " + + strings.TrimPrefix(run.keptBecause(false, "the key"), subject) + ", until a revision that lets it go runs" default: out[i] += "; the engine keeps the value it held, as " + run.keptBecause(false, "the key") + @@ -214,6 +221,6 @@ func (run *configRun) leftOutLines(values []leftOutValue) []string { // leftOutMessage is a part's ResolvedRefs=True message when it gives values // it lets go without whose Secret or key does not exist or holds nothing. func leftOutMessage(leftOut []string) string { - return "Every object it references exists, and every Secret key but these, each a value it gives under a key " + - "an optionalKeys list lets go without it: " + strings.Join(leftOut, "; ") + "." + return ended("Every object it references exists, and every Secret key but these, each a value it gives under a key " + + "an optionalKeys list lets go without it: " + strings.Join(leftOut, "; ")) } diff --git a/internal/controller/config_optional_test.go b/internal/controller/config_optional_test.go index e71f3871..e8655019 100644 --- a/internal/controller/config_optional_test.go +++ b/internal/controller/config_optional_test.go @@ -11,7 +11,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/render" @@ -74,7 +73,7 @@ func optionalRun(t *testing.T, secrets ...*corev1.Secret) (*configRun, *part, *e run := &configRun{ CompanyConfigReconciler: &CompanyConfigReconciler{ Recorder: recorder, - APIReader: fake.NewClientBuilder().WithObjects(objectsOf(secrets)...).Build(), + APIReader: fakeClient().WithObjects(objectsOf(secrets)...).Build(), }, company: company, class: class, parts: []*part{p, newPart(seat)}, } diff --git a/internal/controller/config_parts.go b/internal/controller/config_parts.go index 62903a13..2001088a 100644 --- a/internal/controller/config_parts.go +++ b/internal/controller/config_parts.go @@ -231,15 +231,39 @@ func bounded(message string) string { return message[:cut] + ellipsis } -// sentences joins findings into one message, each ending with a period. +// sentences joins findings into one message, each ending as ended ends it. func sentences(findings []string) string { out := make([]string, len(findings)) for i, f := range findings { - f = capitalize(strings.TrimSpace(f)) - if !strings.HasSuffix(f, ".") { - f += "." - } - out[i] = f + out[i] = ended(capitalize(strings.TrimSpace(f))) } return strings.Join(out, " ") } + +// ended is text as the end of a sentence: with a full stop added where it +// has none, and as it is where it already ends with one, or with "!" or "?", +// or with a parenthesis that closes on one. +// +// THE ONE RULE FOR A MESSAGE THAT ENDS IN WORDS IT QUOTES. A condition or an +// event often ends with what somebody else said: the engine's refusal, its +// account of an apply that failed, or an error that carries either. The +// engine writes those as sentences, each with its full stop, while an error of +// the operator's own or of Go's has none. So a message that added a full stop +// to what it quoted ended "...won't change that.." whenever the quote was the +// engine's, and one that added none ended without a stop whenever it was not. +// The engine ends its own details and hints by the same rule (crewlet's +// internal/api/httpjson, Sentence). The parenthesis is the client's: an +// *engine.APIError quotes the engine's hint in parentheses after its message, +// and the hint is a sentence with its own full stop, so a message quoting one +// already ends inside them and a stop after them would end it twice ("….)."). +// TestQuotedEndingsGoThroughEnded holds every message of the package to it. +func ended(text string) string { + text = strings.TrimSpace(text) + if text == "" { + return text + } + if body := strings.TrimSuffix(text, ")"); body != "" && strings.ContainsRune(".!?", rune(body[len(body)-1])) { + return text + } + return text + "." +} diff --git a/internal/controller/config_pause.go b/internal/controller/config_pause.go index c4163c4b..f93a1b11 100644 --- a/internal/controller/config_pause.go +++ b/internal/controller/config_pause.go @@ -159,8 +159,8 @@ func (run *configRun) syncPause( case errors.Is(err, engine.ErrOutcomeUnknown): return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseUnknown, - message: fmt.Sprintf("The engine may or may not have recorded the operator's %s of the seat; the operator asks again in %s: %s.", - changeWord(*want), engineRetryInterval, err), + message: ended(fmt.Sprintf("The engine may or may not have recorded the operator's %s of the seat; the operator asks again in %s: %s", + changeWord(*want), engineRetryInterval, err)), }, engineRetryInterval case errors.As(err, &refused): // A refusal is often contention (the record lost to other writers @@ -168,15 +168,15 @@ func (run *configRun) syncPause( // for is asked for again soon rather than at the next resync. return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseRefused, - message: fmt.Sprintf("The engine refused the operator's %s of the seat, which it asks for again in %s: %s", - changeWord(*want), engineRetryInterval, refused.Message), + message: ended(fmt.Sprintf("The engine refused the operator's %s of the seat, which it asks for again in %s: %s", + changeWord(*want), engineRetryInterval, refused.Message)), }, engineRetryInterval case err != nil: logf.FromContext(ctx).V(1).Info("Failed to change a seat's pause", "seat", seat.Name, "error", err.Error()) return pauseVerdict{ status: metav1.ConditionUnknown, reason: crewletaiv1alpha1.ReasonPauseUnknown, - message: fmt.Sprintf("The operator's %s of the seat may not have reached the engine; it tries again in %s: %s.", - changeWord(*want), engineRetryInterval, err), + message: ended(fmt.Sprintf("The operator's %s of the seat may not have reached the engine; it tries again in %s: %s", + changeWord(*want), engineRetryInterval, err)), }, engineRetryInterval } if answer.Changed { @@ -221,16 +221,19 @@ func reported(state engine.SeatPause) pauseVerdict { message: "The engine runs the seat: nobody has it paused.", } } - message := fmt.Sprintf("The engine holds the seat paused since %s, paused by %s", state.At.Format(time.RFC3339), state.By) + held := fmt.Sprintf("The engine holds the seat paused since %s, paused by %s", state.At.Format(time.RFC3339), state.By) if state.Reason != "" { - message += ": " + state.Reason + // The reason is whoever paused the seat's own words, which may end + // a sentence of their own. + held += ": " + state.Reason } - message += ". It starts no new turn and its mail waits on its inbox" + waits := "It starts no new turn and its mail waits on its inbox" if state.StopRunning { - message += "; its running work was stopped: the turn it was on, and its executor run at once" + waits += "; its running work was stopped: the turn it was on, and its executor run at once" } return pauseVerdict{ - status: metav1.ConditionTrue, reason: crewletaiv1alpha1.ReasonSeatPaused, message: message + ".", pause: state, + status: metav1.ConditionTrue, reason: crewletaiv1alpha1.ReasonSeatPaused, + message: ended(held) + " " + ended(waits), pause: state, } } diff --git a/internal/controller/config_pause_test.go b/internal/controller/config_pause_test.go index ac9fd949..d19d3c8b 100644 --- a/internal/controller/config_pause_test.go +++ b/internal/controller/config_pause_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -26,16 +27,28 @@ import ( type pauseEngine struct { mu sync.Mutex // roster is what GET /agents answers, by handle; rosterDown makes it - // fail. + // fail as a serving engine's does, which answers 503 only when it + // cannot decide who is asking (its auth.Caller). roster map[string]bool rosterDown bool // record is whether the coordination store holds each seat paused, which - // the tools answer from and change; refuse makes them refuse. - record map[string]bool - refuse bool - calls []string + // the tools answer from and change; refusal, when set, makes them refuse + // with it as the engine's account, and unknown makes them answer that the + // write may or may not have landed, with it as the engine's detail. + record map[string]bool + refusal, unknown string + calls []string } +// The engine's words for a pause call that lost the record to other writers +// every time it tried, and for one whose write got no answer (crewlet's +// internal/agent/builtin, seatpause.go). +const ( + pauseLostRaces = "pause_seat lost 4 races in a row for ana to other changes to its pause. " + + "Read the seat again before retrying." + pauseMayHaveLanded = "the write may or may not have landed; reading the seat again says which" +) + func (f *pauseEngine) client(t *testing.T) *engine.Client { t.Helper() server := mcp.NewServer(&mcp.Implementation{Name: "fake", Version: "1"}, nil) @@ -51,16 +64,22 @@ func (f *pauseEngine) client(t *testing.T) *engine.Client { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, name+" "+args.Handle) - if f.refuse { + if f.refusal != "" { return &mcp.CallToolResult{IsError: true, Content: []mcp.Content{&mcp.TextContent{ - Text: "lost the record to other writers four times in a row", + Text: f.refusal, }}}, nil } - changed := f.record[args.Handle] != paused - f.record[args.Handle] = paused - answer := map[string]any{"handle": args.Handle, "outcome": "applied", "paused": paused, "changed": changed} - if paused { - answer["paused_by"], answer["paused_at"] = engine.TokenLogin(render.TokenOperator), "2026-10-09T08:00:00Z" + answer := map[string]any{"handle": args.Handle} + switch { + case f.unknown != "": + answer["outcome"], answer["detail"] = "unknown", f.unknown + default: + answer["outcome"], answer["paused"] = "applied", paused + answer["changed"] = f.record[args.Handle] != paused + f.record[args.Handle] = paused + if paused { + answer["paused_by"], answer["paused_at"] = engine.TokenLogin(render.TokenOperator), "2026-10-09T08:00:00Z" + } } text, err := json.Marshal(answer) if err != nil { @@ -78,7 +97,11 @@ func (f *pauseEngine) client(t *testing.T) *engine.Client { f.mu.Lock() defer f.mu.Unlock() if f.rosterDown { - http.Error(w, `{"error":"unavailable"}`, http.StatusServiceUnavailable) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"identity_unavailable",` + + `"message":"Can't check who you are right now. Try again in a moment."}`)) return } rows := []map[string]any{} @@ -145,7 +168,8 @@ func TestAPauseThatCannotBeMadeIsAskedForAgainSoon(t *testing.T) { } }) t.Run("the engine refuses the call", func(t *testing.T) { - f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, refuse: true} + f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, + refusal: pauseLostRaces} run, p := pauseRun(new(true)) result := run.syncPauses(context.Background(), f.client(t), ctrl.Result{RequeueAfter: 5 * time.Minute}) if p.pause == nil || p.pause.reason != crewletaiv1alpha1.ReasonPauseRefused { @@ -176,3 +200,51 @@ func TestASeatTheEngineDoesNotRunHasNoPause(t *testing.T) { t.Errorf("the operator called %v for a seat the engine does not run", f.calls) } } + +// TestAPauseVerdictEndsWhatItQuotesOnce holds each pause verdict that quotes +// the engine to one full stop at its end, whether the engine's words end +// with one or not: its refusal is a sentence with its own, and its detail of +// a write that may not have landed is a clause with none, which either could +// become. Each verdict used to add a full stop of its own, or none, whatever +// it quoted, so one of the two shapes ended with two or none. +func TestAPauseVerdictEndsWhatItQuotesOnce(t *testing.T) { + const clause = "the record moved under the call each time it tried" + for _, c := range []struct { + name, refusal, unknown, reason string + }{ + {name: "a refusal the engine ends", refusal: pauseLostRaces, reason: crewletaiv1alpha1.ReasonPauseRefused}, + {name: "a refusal the engine does not end", refusal: clause, reason: crewletaiv1alpha1.ReasonPauseRefused}, + {name: "a detail the engine does not end", unknown: pauseMayHaveLanded, + reason: crewletaiv1alpha1.ReasonPauseUnknown}, + {name: "a detail the engine ends", unknown: "The write may or may not have landed. " + + "Reading the seat again says which.", reason: crewletaiv1alpha1.ReasonPauseUnknown}, + } { + t.Run(c.name, func(t *testing.T) { + f := &pauseEngine{roster: map[string]bool{seatAna: false}, record: map[string]bool{seatAna: false}, + refusal: c.refusal, unknown: c.unknown} + run, p := pauseRun(new(true)) + run.syncPauses(context.Background(), f.client(t), ctrl.Result{}) + if p.pause == nil || p.pause.reason != c.reason { + t.Fatalf("the verdict is %+v, want %s", p.pause, c.reason) + } + if message := p.pause.message; !strings.HasSuffix(message, ".") || strings.HasSuffix(message, "..") { + t.Errorf("the verdict's message does not end with one full stop: %q", message) + } + }) + } +} + +// TestAPauseReasonIsQuotedAsWrittenOnce holds the Paused message to one full +// stop after the reason whoever paused the seat gave, which is their own +// words and may end a sentence of its own, before the operator's next one. +func TestAPauseReasonIsQuotedAsWrittenOnce(t *testing.T) { + at := time.Date(2026, 10, 9, 8, 0, 0, 0, time.UTC) + for _, reason := range []string{"Hold the deploy.", "hold the deploy", ""} { + for _, stop := range []bool{false, true} { + verdict := reported(engine.SeatPause{Paused: true, By: humanJane, At: at, Reason: reason, StopRunning: stop}) + if strings.Contains(verdict.message, "..") || !endsOnce(verdict.message) { + t.Errorf("reason %q, stop %t: the message is %q, want each sentence ended once", reason, stop, verdict.message) + } + } + } +} diff --git a/internal/controller/config_placement_test.go b/internal/controller/config_placement_test.go index 8abb2a64..9abcd1b3 100644 --- a/internal/controller/config_placement_test.go +++ b/internal/controller/config_placement_test.go @@ -6,9 +6,7 @@ import ( "testing" "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" "github.com/crewlet/crewlet-operator/internal/engine" @@ -49,13 +47,9 @@ func TestPlacementRecordKeepsThePreviousPinsUntilApplied(t *testing.T) { // on a client that serves its status subresource. func placementCompany(t *testing.T, recorded crewletaiv1alpha1.RevisionPlacement) (client.Client, *crewletaiv1alpha1.Company) { t.Helper() - scheme := runtime.NewScheme() - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } company := &crewletaiv1alpha1.Company{Namespace: "ns", Name: fleetCompany} company.Status.Config = crewletaiv1alpha1.ConfigStatus{RevisionID: "r1", Placement: recorded} - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(company).WithStatusSubresource(company).Build() + c := fakeClient().WithObjects(company).WithStatusSubresource(company).Build() got := &crewletaiv1alpha1.Company{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(company), got); err != nil { t.Fatal(err) diff --git a/internal/controller/config_reload_test.go b/internal/controller/config_reload_test.go index c1fac7c0..3a3e4235 100644 --- a/internal/controller/config_reload_test.go +++ b/internal/controller/config_reload_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "slices" "strings" "testing" @@ -9,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crewlet/crewlet-operator/internal/engine" "github.com/crewlet/crewlet-operator/internal/secretref" "github.com/crewlet/crewlet-operator/internal/secretsync" ) @@ -75,6 +77,9 @@ func TestARotationIsTheEnginesOneActivation(t *testing.T) { if state := w.state(t); state.Recorded() != "r2" || !slices.Equal(state.LetGo("r2"), []string{w.registry}) { t.Fatalf("the state records %q letting %v go, want r2 read as the rendered document", state.Recorded(), state.LetGo("r2")) } + if owed := w.state(t).Written(); len(owed) != 0 { + t.Errorf("the state owes %v, want nothing: r2 activated after the sync wrote the rotated value", owed) + } if !slices.ContainsFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated Activated revision r2 ") }) { t.Errorf("events %q, want r2's activation", recorded) } @@ -89,33 +94,25 @@ func TestARotationIsTheEnginesOneActivation(t *testing.T) { } // TestARotationIsAppliedWhereTheEngineDidNotReactivate: an engine whose -// write re-activated nothing, or failed to, or whose re-activation somebody -// else's write overtook before the operator read it back, leaves the apply -// to the operator, which writes the rendered document itself. +// write re-activated nothing, or failed to, leaves the apply to the +// operator, which writes the rendered document itself. func TestARotationIsAppliedWhereTheEngineDidNotReactivate(t *testing.T) { for _, tc := range []struct { name string // reload is what the engine answers of its re-activation instead // of making one; nil for one it makes. reload *map[string]any - // overtaken is whether somebody else activates the rendered - // document again right after the engine's re-activation. - overtaken bool - want []string + want []string }{ {name: "nothing read the value", reload: new(map[string]any(nil)), want: []string{"r1", "r2"}}, {name: "the re-activation lost a race", reload: failedReload("revision_advanced", "another configuration write activated first"), want: []string{"r1", "r2"}}, {name: "the re-activation could not be confirmed", reload: failedReload("unavailable", "the coordination store did not answer"), want: []string{"r1", "r2"}}, - {name: "somebody else's write overtook it", overtaken: true, want: []string{"r1", "r2", "r3", "r4"}}, } { t.Run(tc.name, func(t *testing.T) { w := newRotation(t) w.engine.reloadAnswer = tc.reload - if tc.overtaken { - w.engine.afterSecretsReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } - } if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -142,3 +139,55 @@ func failedReload(code, detail string) *map[string]any { failed := refusal(code, detail) return &failed } + +// TestARotationAnotherWriteOvertookIsNotWrittenOver: somebody else activates +// the rendered document again right after the engine's re-activation at the +// rotation, before the operator reads it back. The operator's write is +// conditioned on the revision its reconcile read, so the engine refuses it +// rather than store it over theirs unseen, and the next reconcile, from a +// fresh read, writes the document on their revision: it cannot tell their +// activation came after the rotated value, since it read the fleet after +// both. +func TestARotationAnotherWriteOvertookIsNotWrittenOver(t *testing.T) { + w := newRotation(t) + w.engine.afterSecretsReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + if err := w.enforce(t); !errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("enforce() = %v, want the write refused as a lost race", err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3"}) { + t.Fatalf("the revisions are %v, want r1, the engine's r2 and the other writer's r3: nothing written over r3", got) + } + w.again(t, w.statusWritten()) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r4"}) || + !slices.Equal(conditions, []string{"r3"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r4 on r3", written, conditions) + } + if owed := w.state(t).Written(); len(owed) != 0 { + t.Errorf("the state owes %v, want nothing after r4", owed) + } +} + +// TestAWriteAfterTheEnginesActivationAtTheSyncIsConditionedOnIt: the engine +// runs somebody else's document, and a rotated value has the engine activate +// it again as the sync writes the value. The operator's write of the rendered +// document is conditioned on that activation, read back as the same document, +// not on the revision the reconcile read before the sync: conditioned on that +// one, every rotation that came with a change of the document was refused as +// a lost race and written again by a second reconcile. +func TestAWriteAfterTheEnginesActivationAtTheSyncIsConditionedOnIt(t *testing.T) { + w := newWithdrawal(t, runningOther) + w.rotate(t) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + written, refused := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || refused != 0 || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v (and %d were refused), want r3 alone, conditioned on "+ + "r2, the engine's activation at the sync", written, conditions, refused) + } +} diff --git a/internal/controller/config_run.go b/internal/controller/config_run.go index 63b7df17..74a245c9 100644 --- a/internal/controller/config_run.go +++ b/internal/controller/config_run.go @@ -91,6 +91,11 @@ type configRun struct { config *crewletaiv1alpha1.ConfigStatus // wrote is whether this reconcile activated a revision. wrote bool + // confirmed is the revision, empty for none, that this reconcile found + // the engine running and that the operator wrote and the engine could + // not confirm activating (confirmUnconfirmed): a write an earlier + // reconcile made, of what it rendered then. + confirmed string // objectsChecked and secretsChecked are whether the reconcile got as // far as resolving the parts' references to other objects, and to @@ -118,8 +123,16 @@ type configRun struct { rendered, running *crewletaiv1alpha1.RevisionPlacement runningRevision string // active is the document the engine ran when converge last read it, - // nil before it did or while the engine had none. + // nil before it did, while the engine had none, and while it could not + // open the one it ran. active []byte + // unsealed is the engine's active revision when converge last read it, + // where the engine stores it without a seal and the reconcile replaces + // it (engine.Unsealed, under Enforce): read by nothing, the operator and + // the engine's own nodes included, which run the revision before it, so + // the reconcile goes on with no active document and the write is + // conditioned on it (enforce). Empty for any other revision. + unsealed string // placement is what status.config.placement becomes once this reconcile // writes the status; nil leaves it as it is. placement *placementUpdate @@ -281,11 +294,7 @@ func (run *configRun) reconcileDocument(ctx context.Context) (ctrl.Result, error if isAPIServerError(err) { return ctrl.Result{}, err } - if !run.writeRefused(err) && !run.notGranted(err) && !run.seatHeld(err) { - run.unavailable(err) - } - _, _ = run.fleetOf(ctx, eng) - return ctrl.Result{RequeueAfter: retryAfter(err)}, nil + return run.engineFailed(ctx, eng, err), nil } health, healthErr := run.readHealth(ctx, eng) result = run.verifyApplied(ctx, eng, health, healthErr, result) @@ -382,31 +391,7 @@ func (run *configRun) policy() crewletaiv1alpha1.ManagementPolicy { // error, and the reconcile is retried. func (run *configRun) converge(ctx context.Context, eng *engine.Client) (ctrl.Result, error) { for attempt := 1; ; attempt++ { - active, err := activeConfig(ctx, eng) - if err != nil { - return ctrl.Result{}, err - } - if active != nil { - run.active = active.Document - } - var result ctrl.Result - switch policy := run.policy(); { - case policy == crewletaiv1alpha1.ManagementPolicyObserve: - return run.observe(ctx, eng, active) - case policy == crewletaiv1alpha1.ManagementPolicySeed && active != nil: - if err := run.seeded(active); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: run.resyncInterval()}, nil - default: - if run.desired == nil { - blocked, err := run.resolveSecrets(ctx) - if blocked || err != nil { - return ctrl.Result{}, err - } - } - result, err = run.enforce(ctx, eng, *run.desired, active) - } + result, err := run.convergeOnce(ctx, eng) lost := errors.Is(err, engine.ErrRevisionAdvanced) || errors.Is(err, engine.ErrAlreadyConfigured) if lost && attempt == 1 { logf.FromContext(ctx).V(1).Info("Another writer changed the engine's document during the apply; reading it again") @@ -416,6 +401,96 @@ func (run *configRun) converge(ctx context.Context, eng *engine.Client) (ctrl.Re } } +// convergeOnce is one attempt of converge, from a fresh read of the engine's +// document. +func (run *configRun) convergeOnce(ctx context.Context, eng *engine.Client) (ctrl.Result, error) { + // The fleet is read after this attempt's read of the document, never + // before it: confirmUnconfirmed, which runs before keepServing reads it + // afresh, takes the epoch the fleet's activation names for the revision + // this read returns, and an answer an attempt that lost its write read + // describes the engine from before that revision. + run.fleet, run.fleetErr = nil, nil + // A confirmation is of the revision the read runs: another one runs once + // the reconcile starts over. + run.confirmed = "" + active, err := activeConfig(ctx, eng) + run.active, run.unsealed = nil, "" + if revision, ok := engine.Unsealed(err); ok && run.policy() == crewletaiv1alpha1.ManagementPolicyEnforce { + // A REVISION STORED WITHOUT A SEAL IS REPLACED, under Enforce: the + // engine's own remedy for one is a whole document, which goes on + // over it reading nothing from it, and the rendered document is + // one. Nothing reads what it holds, the operator included, so the + // reconcile goes on as over an engine with no document, but for the + // write's base, which is this revision. Under Seed the operator + // writes only into an engine with no document, and under Observe it + // writes nothing, so there it stays what blocks the document + // (revisionUnreadable), as a revision sealed in a way the engine + // cannot open does under every policy: a write over one of those + // would discard a document somebody sealed, and the engine refuses + // it. + run.unsealed, err = revision, nil + } + if err != nil { + return ctrl.Result{}, err + } + runs := run.unsealed + if active != nil { + run.active = active.Document + runs = active.RevisionID + } + // Before anything is decided from it: the revision the engine runs may be + // a write of the operator's that the engine could not confirm + // (config_unconfirmed.go). + if err := run.settleUnconfirmed(ctx, eng, runs, active); err != nil { + return ctrl.Result{}, err + } + switch policy := run.policy(); { + case policy == crewletaiv1alpha1.ManagementPolicyObserve: + return run.observe(ctx, eng, active) + case policy == crewletaiv1alpha1.ManagementPolicySeed && active != nil: + if err := run.seeded(active); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: run.resyncInterval()}, nil + } + if run.desired == nil { + blocked, err := run.resolveSecrets(ctx) + if blocked || err != nil { + return ctrl.Result{}, err + } + } + return run.enforce(ctx, eng, *run.desired, active) +} + +// activationAhead is inSync's answer, and confirmUnconfirmed's, where the +// fleet's activation names a revision other than the one GET /config answered +// with: both read the fleet's activation pointer, so another write activated +// a revision between the two reads. It is a lost race +// (engine.ErrRevisionAdvanced), which converge reads again from the start +// once, as it reads a write that lost. +type activationAhead struct { + // read is the revision GET /config answered with. + read string + // activation is the fleet's. + activation engine.Activation + // unconfirmed is whether read is a write of the operator's whose + // activation the engine could not confirm, rather than a revision of the + // rendered document the operator has not recorded. + unconfirmed bool +} + +func (e *activationAhead) Error() string { + runs := fmt.Sprintf("the rendered document as revision %s, which the operator has not recorded", e.read) + if e.unconfirmed { + runs = fmt.Sprintf("revision %s, which the operator wrote and the engine could not confirm activating", e.read) + } + return fmt.Sprintf("the fleet runs %s, and its activation named revision %s (epoch %d) by the time the operator "+ + "read the fleet: another write activated a revision between the two reads", runs, e.activation.RevisionID, + e.activation.Epoch) +} + +func (e *activationAhead) Unwrap() error { return engine.ErrRevisionAdvanced } + // activeConfig is the fleet's active configuration, the revision the // engine's activation pointer names, which every node answers whatever it // runs itself (keepServing reads that), or nil when it has none yet. @@ -432,7 +507,9 @@ func activeConfig(ctx context.Context, eng *engine.Client) (*engine.ActiveConfig // enforce makes the engine run the rendered document: under Enforce always, // and under Seed for the one write into an engine with no document (active -// is nil then). +// is nil then). Under Enforce active is nil too where the engine's active +// revision is one it stores without a seal, which the write replaces +// (run.unsealed), and which its nodes do not run. func (run *configRun) enforce( ctx context.Context, eng *engine.Client, desired secretsync.Desired, active *engine.ActiveConfig, ) (ctrl.Result, error) { @@ -450,9 +527,9 @@ func (run *configRun) enforce( if notOwned.Controller == nil || !isPreviousCompany(notOwned.Controller, c) { run.secretsSynced = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionSecretsSynced, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonNameConflict, Message: capitalize(notOwned.Error()) + ".", + Reason: crewletaiv1alpha1.ReasonNameConflict, Message: capitalize(ended(notOwned.Error())), } - run.block(crewletaiv1alpha1.ReasonNameConflict, capitalize(notOwned.Error())+".") + run.block(crewletaiv1alpha1.ReasonNameConflict, capitalize(ended(notOwned.Error()))) return ctrl.Result{}, nil } state, err = secretsync.State{}, nil @@ -460,6 +537,7 @@ func (run *configRun) enforce( if err != nil { return ctrl.Result{}, err } + state = run.settlePending(state, active) run.told = state.Retained() diffs, err := diffWith(run.doc, active) if err != nil { @@ -478,8 +556,22 @@ func (run *configRun) enforce( // the store at every apply attempt, and a data pod starts on the // revision its volume names, which does not start without them // (keepServing). + // + // A revision stored without a seal keeps everything the store holds: + // which names it references is what nobody can read, the operator + // included, and it is not what the engine's nodes run. The engine + // activates it again neither by a reload nor by a revert to it, so each + // node goes on running the revision it applied before it, and boots that + // one again when it restarts (it never adopts a body it cannot open), + // reading the store as it finds it then. A name deleted now is one such a + // node may start without, read as required by a revision nobody sees, for + // as long as the replacement is held: by the engine refusing it, by a + // seat somebody holds, or by a write that has not been sent yet. So + // nothing is deleted before the write; the sync after it decides by the + // revision then running, the replacement, as after any write. pushing := desired - if active != nil { + switch { + case active != nil: state = run.adopt(state, active, diffs) pushing.Keep(active.RevisionID, storeNamesIn(active.Document)...) released, err := secretsync.Absent(ctx, run.APIReader, state.LetGoKeys(active.RevisionID)) @@ -487,10 +579,15 @@ func (run *configRun) enforce( return ctrl.Result{}, err } pushing.Release(released...) + case run.unsealed != "": + pushing.KeepListed(run.unsealed) } if err := run.keepServing(ctx, eng, &pushing); err != nil { return ctrl.Result{}, err } + // What the fleet keepServing read settles or marks, the sync below + // saves, whatever it writes, before any activation is asked for. + state, _ = run.settleRead(ctx, eng, active, state) // SecretKept is said of what the reconcile ends up keeping: a value the // sync below keeps for the running revision goes after the apply. defer run.tellKept() @@ -500,38 +597,42 @@ func (run *configRun) enforce( } hash := appliedHash(run.doc, state, desired.Names()) - recorded := c.Status.Config - // A value written back although no Secret changed (one somebody else - // overwrote, or one the store lost) leaves the hash where it was, but the - // engine reads its store only when a revision activates: since the - // overwrite, a reload may have handed it the other value, which it would - // keep running. So a change of a value is followed by an activation, - // which is how a rotated credential reaches a running engine: the - // engine's own, which re-activates the revision that reads a value it is - // written or removes (reloadedBySync), or else the operator's apply - // below. - if active != nil && len(diffs) == 0 && hash == recorded.AppliedHash && len(pushed.Put) == 0 { - run.inSync(active) - failed, err := run.settle(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) - if err != nil { - return ctrl.Result{}, err - } - if failed { - return ctrl.Result{RequeueAfter: min(engineRetryInterval, run.resyncInterval())}, nil - } - return run.heldResync(ctrl.Result{RequeueAfter: run.resyncInterval()}), nil - } - reloaded, err := run.reloadedBySync(ctx, eng, active, diffs, pushed.Reload) + recorded := run.recordedConfig() + // The engine runs the rendered document with every value only where the + // revision it runs read the store after the last value the sync wrote: + // the engine reads its store only when a revision activates. The state + // says which values no activation is known to have read yet + // (State.Written), settled above by the activation the engine runs where + // that came after them, and every value is in it from before it was + // written: a value rotated, or one written back although no Secret + // changed (one somebody else overwrote, or one the store lost), which + // leaves the hash where it was, and one a write of the document that + // failed or lost its race left the next sync nothing to write. So the + // recorded hash is no part of the question: a revision of the rendered + // document that read every value is in sync whatever the operator + // recorded, a write of it whose record the reconcile that made it never + // saved included, and a withdrawal is all it owes (settle), activated by + // the engine at the removal or else by the operator. One that did not is + // activated again, which is how a rotated credential reaches a running + // engine: by the engine itself, which re-activates the revision that + // reads a value it is written (syncActivation), or else by the + // operator's apply below. + if active != nil && len(diffs) == 0 && len(state.Written()) == 0 { + return run.keepInSync(ctx, eng, active, hash, pushed.Reload, pushed.ReloadAction, state, stateRef) + } + synced, err := run.syncActivation(ctx, eng, active, pushed.Reload) if err != nil { return ctrl.Result{}, err } - if reloaded != nil { + if reloaded := synced; reloaded != nil && run.rendersAs(active, diffs) { // The engine runs the rendered document with every value the sync - // left in its store: applying it again would be a second activation - // of the same thing. It is recorded as the apply would have been, - // and it read the store without every name withdrawn. + // left in its store, activated by the sync's own last request after + // every write and removal: applying it again would be a second + // activation of the same thing. It is recorded as the apply would + // have been, and it read the store without every name withdrawn and + // with every value written. run.recordApply(*reloaded, hash, false, active, diffs, - strings.TrimPrefix(run.summary(active, diffs, pushed.Put), "crewlet-operator: ")+", re-activated by the engine") + strings.TrimPrefix(run.summary(active, diffs, state.Written()), "crewlet-operator: ")+", re-activated by the engine") state = state.Activated(reloaded.RevisionID, run.doc.Optional) if err := secretsync.SaveState(ctx, run.Client, stateRef, c, state); err != nil { return ctrl.Result{}, err @@ -543,12 +644,9 @@ func (run *configRun) enforce( // had before the operator took it over. drifted := active != nil && len(diffs) > 0 && active.RevisionID != recorded.RevisionID - checked, err := eng.ApplyConfig(ctx, engine.ApplyRequest{Document: run.doc.YAML, DryRun: true}) - if err != nil { - if run.invalid(err) { - // The sync above may have withdrawn a value the running - // revision lets go: it is activated again instead. - return run.settleRefused(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) + if _, err := eng.ApplyConfig(ctx, engine.ApplyRequest{Document: run.doc.YAML, DryRun: true}); err != nil { + if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef, err); refused { + return result, saveErr } return ctrl.Result{}, err } @@ -560,23 +658,46 @@ func (run *configRun) enforce( if err := run.widenPlacement(ctx, *run.rendered); err != nil { return ctrl.Result{}, err } - req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, pushed.Put)} - if checked.BaseRevisionID == "" { - req.CreateOnly = true - } else { - // What the dry run checked against: a write in between is lost - // rather than overwritten unseen. - req.IfMatch = checked.BaseRevisionID + // What the write's activation reads is marked before it is asked for, + // so that a save lost after it still leaves what it read settled by the + // next read of the engine; and so is how its document reads every name, + // for the revision the engine stores, which a later read may find + // running although its activation went unconfirmed, or its record was + // never saved (settlePending). + ahead := !state.Pends(hash) + state = state.Pending(hash, run.doc.Optional) + if state, err = run.markChanges(ctx, eng, state, stateRef, ahead); err != nil { + return ctrl.Result{}, err } - if run.policy() == crewletaiv1alpha1.ManagementPolicySeed { - req.IfMatch, req.CreateOnly = "", true + req := engine.ApplyRequest{Document: run.doc.YAML, Summary: run.summary(active, diffs, state.Written())} + // THE WRITE IS CONDITIONED ON THE REVISION THIS RECONCILE READ, which its + // diff, its drift, its summary and the names its sync kept were decided + // against: the engine's document, a revision stored without a seal, which + // the write replaces, or none, which makes it create-only (the one write + // Seed makes, since Seed writes only where the read found no document). + // Not on the revision the dry run names: a write that landed since the + // read is the one the dry run was checked against, and a write + // conditioned on that went over it unseen, reported as replacing the + // revision the reconcile read. Conditioned on the read, the engine + // refuses it before storing anything, and the reconcile starts over from + // a fresh read (converge). + // The engine's own activation of that revision at the sync is the + // same document, read back (syncActivation): the write is conditioned + // on it, or it would be refused over the sync's own activation. + switch { + case synced != nil: + req.IfMatch = synced.RevisionID + case active != nil: + req.IfMatch = active.RevisionID + case run.unsealed != "": + req.IfMatch = run.unsealed + default: + req.CreateOnly = true } applied, err := eng.ApplyConfig(ctx, req) if err != nil { - if run.invalid(err) { - return run.settleRefused(ctx, eng, active, pushed.Reload, pushed.ReloadAction, state, stateRef) - } - return ctrl.Result{}, err + return run.writeFailed(ctx, eng, err, run.writeAttemptOf(req, hash, drifted, diffs), active, pushed.Reload, + pushed.ReloadAction, state, stateRef) } run.recordApply(applied, hash, drifted, active, diffs, req.Summary) @@ -629,6 +750,107 @@ func (run *configRun) cleanUp( return run.heldResync(ctrl.Result{RequeueAfter: applyPollInterval}), nil } +// refusedByEngine ends the reconcile where the engine refused the rendered +// document, its dry run or its write, with err, for what no retry of the +// write changes and the Company's conditions now say: problems with the +// document (invalid), a human seat somebody holds that it takes away +// (seatHeld), or the operator's token refused the change (writeRefused, +// notGranted). It reports false for any other error, which goes back as one +// (engineFailed). +// +// Whatever the refusal, the sync before it may have changed the store under +// the revision the engine runs, and no write is coming to activate one that +// reads the change, so the activation owed is made now (settleRefused, which +// takes the sync's reload and reloadAction, as settle does), +// before the reconcile ends: a refused document went back as an error, which +// nothing settled, and for as long as somebody held the seat the engine went +// on starting tool servers and executor runs with the value of a Secret that no +// longer exists. Neither a held seat nor a managed document stops a reload of +// the running revision, which is admin's. A refusal the engine's own users +// clear is looked at again as engineRetry says, since nothing in the cluster +// changes when they do; one the Company's objects clear, a document the +// engine finds invalid, when one of them changes. +func (run *configRun) refusedByEngine( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, + state secretsync.State, ref types.NamespacedName, err error, +) (ctrl.Result, bool, error) { + switch { + case run.invalid(err): + result, saveErr := run.settleRefused(ctx, eng, active, reload, reloadAction, state, ref) + return result, true, saveErr + case run.seatHeld(err), run.writeRefused(err), run.notGranted(err): + result, saveErr := run.settleRefused(ctx, eng, active, reload, reloadAction, state, ref) + if saveErr != nil { + return result, true, saveErr + } + wait := run.engineRetry(err) + if result.RequeueAfter > 0 { + // The look a failed activation's report promises, or the + // pointer poll a value kept for another revision asks for + // (heldResync), where that is sooner. + wait = min(wait, result.RequeueAfter) + } + return ctrl.Result{RequeueAfter: wait}, true, nil + } + return ctrl.Result{}, false, nil +} + +// writeAttempt is a write of the rendered document: the revision it was +// built on, which it is conditioned on ("" for none), its appliedHash, and +// what it replaced where that was more than the operator's own document +// (replaced, with driftedAt, the places a Drifted one differed at as +// ConfigApplied names them), which a write the engine could not confirm keeps +// for the report made once a read shows it runs. +type writeAttempt struct { + base, hash string + replaced crewletaiv1alpha1.ReplacedRevision + driftedAt string +} + +// writeAttemptOf is req, a write of the rendered document whose appliedHash +// is hash, as a writeAttempt: over a revision stored without a seal +// (run.unsealed), or drifted where the revision it was built on is somebody +// else's, which differed from the rendered document at diffs. +func (run *configRun) writeAttemptOf(req engine.ApplyRequest, hash string, drifted bool, diffs []render.Path) writeAttempt { + attempt := writeAttempt{base: req.IfMatch, hash: hash} + switch { + case run.unsealed != "": + attempt.replaced = crewletaiv1alpha1.ReplacedUnsealed + case drifted: + attempt.replaced, attempt.driftedAt = crewletaiv1alpha1.ReplacedDrifted, placesOf(diffs) + } + return attempt +} + +// writeFailed ends the reconcile where the engine did not take the write, +// failing with err. A document it refused is settled as one (refusedByEngine, +// with the sync's reload and reloadAction). +// A write it stored and could not confirm activating is recorded in +// status.config.unconfirmed, the secret sync's state having recorded what its +// activation reads before it was sent, and settled by the next read of the +// revision the engine runs, never by sending the write again before it +// (config_unconfirmed.go). Any other failure goes back. +func (run *configRun) writeFailed( + ctx context.Context, eng *engine.Client, err error, write writeAttempt, active *engine.ActiveConfig, + reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, +) (ctrl.Result, error) { + if result, refused, saveErr := run.refusedByEngine(ctx, eng, active, reload, reloadAction, state, ref, err); refused { + return result, saveErr + } + if stored, ok := engine.UnconfirmedActivation(err); ok { + // How its document reads every name the state recorded before it + // was sent (markChanges), and the changes it was sent after await + // the revision the engine stored, which settles them once found + // running however the fleet's pointer reads (State.Awaiting). + after := run.unconfirmed(stored, write, err) + if saveErr := secretsync.SaveState(ctx, run.Client, ref, run.company, state.Awaiting(stored)); saveErr != nil { + return ctrl.Result{}, saveErr + } + return ctrl.Result{RequeueAfter: after}, nil + } + return ctrl.Result{}, err +} + // adopt is state recording the reading of active, the revision the engine // runs, where its document is the one this reconcile rendered: diffs, where // the two differ, is empty, and the engine's document references every store @@ -646,40 +868,36 @@ func (run *configRun) adopt(state secretsync.State, active *engine.ActiveConfig, return state.Adopted(active.RevisionID, run.doc.Optional) } -// reloadedBySync is the revision the engine activated when the sync changed -// its store, where that is the rendered document running with every value -// the sync left there: nil where the operator has to apply the document -// itself. +// syncActivation is the engine's own activation of active at the sync, read +// back: the revision it activated when the sync changed its store, where the +// engine runs it with active's document, the one the operator read before +// the sync; nil where the operator has to activate a document itself. // // The engine re-activates its active revision whenever a request writes or // removes a value that revision reads, once for all the values of one write, // and answers what that did; reload, secretsync.Result.Reload, is the answer -// to the last such request of the sync, after which the store held, of every value -// the revision reads, what it holds now. Where that re-activation landed and -// the document the engine ran before it is the rendered one (diffs is empty, -// and both reference every store name as often, so that the rendered -// document's reading of the store is that revision's, as adopt says), the -// engine runs what an apply of the rendered document would activate. It -// names no revision, so as in reactivate it is read back: the engine must run -// the revision it answered, with the document the operator read before the -// sync, or somebody else's write landed in between and the document is -// applied. +// to the last such request of the sync, after which the store held, of every +// value the revision reads, what it holds now. It names no revision it was +// built on, so as in reactivate it is read back: the engine must run the +// revision it answered, with the document the operator read before the +// sync, or somebody else's write landed in between, and what the reconcile +// decided against its read no longer holds. // // A re-activation that failed, one the engine did not make because nothing // the revision reads changed, and a write whose answer is not read, leave the -// apply to the operator, as before the engine re-activated at all. -func (run *configRun) reloadedBySync( - ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, diffs []render.Path, reload *engine.Reload, +// activation to the operator, as before the engine re-activated at all. +func (run *configRun) syncActivation( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, ) (*engine.ApplyResult, error) { - if active == nil || reload == nil || len(diffs) > 0 || - !maps.Equal(storeReferences(run.doc.YAML), storeReferences(active.Document)) { + if active == nil || reload == nil { return nil, nil } log := logf.FromContext(ctx) if !reload.Landed() { // The engine sealed the values and says why it did not - // re-activate: the apply below is the activation instead. - log.Info("The engine did not activate its document again after the secret sync; applying it", + // re-activate: the operator's write or reload is the activation + // instead. + log.Info("The engine did not activate its document again after the secret sync", "error", reload.Error, "detail", reload.Detail) return nil, nil } @@ -688,29 +906,114 @@ func (run *configRun) reloadedBySync( return nil, err } if now == nil || now.RevisionID != reload.RevisionID || !bytes.Equal(now.Document, active.Document) { - log.Info("The engine runs another revision than the one it activated at the secret sync; applying the document", + log.Info("The engine runs another revision than the one it activated at the secret sync", "reloaded", reload.RevisionID) return nil, nil } return &engine.ApplyResult{RevisionID: reload.RevisionID, Epoch: reload.Epoch, Warnings: []string{}}, nil } +// rendersAs reports whether active, the engine's document, is the rendered +// one: diffs is empty, and both reference every store name as often, so that +// the rendered document's reading of the store is that revision's, as adopt +// says. An activation of it again with every value the sync left in the +// store (syncActivation) is then what an apply of the rendered document +// would activate. +func (run *configRun) rendersAs(active *engine.ActiveConfig, diffs []render.Path) bool { + return active != nil && len(diffs) == 0 && + maps.Equal(storeReferences(run.doc.YAML), storeReferences(active.Document)) +} + +// settleRead is state with every change of the store settled that the +// fleet's activation came after (State.Ran), or that awaits the revision +// this attempt read (State.Found: one the engine stored for an activation +// asked for after the change and could not confirm, which settles it however +// the pointer reads, and whether or not the fleet can be read), and every +// change no read of the fleet's activation pointer has followed marked by the +// epoch it names now (State.Observed), and whether that moved a mark, which +// the caller saves before it asks for an activation (markChanges' rule: a +// save lost after the activation must leave a change the activation settles). +// It reads the fleet the reconcile last read (fleetOf), which keepServing +// reads afresh after this attempt's read of the document. active is the +// revision that read returned, which GET /config opened: the activation +// settles only where it is that revision. Both read the fleet's activation +// pointer, so another revision there is one a write activated between the +// two reads, perhaps one no node opens. state holds only what earlier +// reconciles changed, so any read of the fleet this attempt makes followed +// it. A fleet that cannot be read, or answers without its activation, settles +// nothing by the pointer and marks nothing. +func (run *configRun) settleRead( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, +) (secretsync.State, bool) { + next := state + if active != nil { + next = next.Found(active.RevisionID) + } + if len(next.Withdrawn()) == 0 && len(next.Written()) == 0 { + return next, !next.SameMarks(state) + } + fleet, err := run.fleetOf(ctx, eng) + if err != nil || fleet.Activation == nil { + return next, !next.SameMarks(state) + } + if active != nil && fleet.Activation.RevisionID == active.RevisionID { + next = next.Ran(fleet.Activation.Epoch) + } + next = next.Observed(fleet.Activation.Epoch) + return next, !next.SameMarks(state) +} + +// markChanges reads the fleet's activation pointer after the changes of the +// store state records that no read has followed yet, marks each with the +// epoch it names (State.Observed), and saves state, before the caller asks +// the engine for an activation: one the engine is later read running above +// that epoch came after every change, whoever made it, so a save lost after +// the activation the caller asks for leaves what that activation read settled +// by the next read of the engine (settleRead) rather than owed for good. A +// fleet that cannot be read, or answers without its activation, leaves them +// unread, for an activation the operator sees confirmed to settle; a save +// that fails is the caller's error, and no activation is asked for. changed +// says the caller changed state since it was saved, so that it is saved +// whether or not a change of the store was marked. +func (run *configRun) markChanges( + ctx context.Context, eng *engine.Client, state secretsync.State, ref types.NamespacedName, changed bool, +) (secretsync.State, error) { + if state.Unobserved() { + fleet, err := eng.Fleet(ctx) + if err != nil || fleet.Activation == nil { + logf.FromContext(ctx).V(1).Info("Failed to read the fleet's activation pointer after a change of the "+ + "engine's secret store; the change waits for an activation the operator sees confirmed", + "error", fmt.Sprint(err)) + } else { + state, changed = state.Observed(fleet.Activation.Epoch), true + } + } + if !changed { + return state, nil + } + return state, secretsync.SaveState(ctx, run.Client, ref, run.company, state) +} + // sync pushes the desired values into the engine's secret store, deletes // what nothing references and what the revision the engine runs goes -// without, and saves what it did, whether or not it got through. What it -// keeps for that revision is recorded (keep), for the reconcile to say once -// it knows it ends up keeping it (tellKept). +// without, and saves what it did, whether or not it got through, and what it +// is about to write before it writes it (secretsync.Sync). What it keeps for +// that revision is recorded (keep), for the reconcile to say once it knows it +// ends up keeping it (tellKept). func (run *configRun) sync( ctx context.Context, eng *engine.Client, desired secretsync.Desired, state secretsync.State, ref types.NamespacedName, ) (secretsync.State, secretsync.Result, error) { - next, result, syncErr := secretsync.Sync(ctx, eng, desired, state, engine.TokenLogin(render.TokenOperator)) + next, result, syncErr := secretsync.Sync(ctx, eng, desired, state, engine.TokenLogin(render.TokenOperator), + func(ahead secretsync.State) error { + return secretsync.SaveState(ctx, run.Client, ref, run.company, ahead) + }) if err := secretsync.SaveState(ctx, run.Client, ref, run.company, next); err != nil { return next, result, errors.Join(syncErr, err) } if syncErr != nil { run.secretsSynced = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionSecretsSynced, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: capitalize(syncErr.Error()) + ".", + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: capitalize(ended(syncErr.Error())), } return next, result, syncErr } @@ -750,8 +1053,8 @@ func (run *configRun) sync( message += run.retainedSentence() message += run.heldSentence() if len(run.companyLeftOut) > 0 { - message += " The Company's own setup steps give values they let go without: " + - strings.Join(run.leftOutLines(run.companyLeftOut), "; ") + "." + message += " " + ended("The Company's own setup steps give values they let go without: "+ + strings.Join(run.leftOutLines(run.companyLeftOut), "; ")) } if len(run.companyEmpty) > 0 { message += " " + sentences(run.companyEmpty) @@ -807,32 +1110,97 @@ func diffWith(doc *render.Document, active *engine.ActiveConfig) ([]render.Path, return render.Diff(doc.YAML, active.Document) } +// keepInSync is enforce for an engine that runs the rendered document with +// every value it references, whose appliedHash is hash: it records the +// revision (inSync), makes the activation the secret sync's state owes +// (settle), and comes back at the resync, or sooner where that activation +// failed and its report says so, or a value is kept for a revision other than +// the fleet's (heldResync). reload and reloadAction are the sync's, as settle +// takes them. +func (run *configRun) keepInSync( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string, reload *engine.Reload, + reloadAction string, state secretsync.State, ref types.NamespacedName, +) (ctrl.Result, error) { + if err := run.inSync(ctx, eng, active, hash); err != nil { + return ctrl.Result{}, err + } + again, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) + if err != nil { + return ctrl.Result{}, err + } + next := run.resyncInterval() + if again > 0 { + next = min(again, next) + } + return run.heldResync(ctrl.Result{RequeueAfter: next}), nil +} + // inSync records an engine that runs the rendered document, with every // value it references, without a write: possibly as a revision the operator -// did not write itself, such as a reload of the same document, which it -// adopts. -func (run *configRun) inSync(active *engine.ActiveConfig) { - recorded := run.company.Status.Config +// has not recorded, such as a reload of the same document, which it adopts at +// the epoch the fleet's activation names for it, and from now: the engine is +// held to applying that epoch applyDeadline after the operator found it, not +// after the operator's last write, another revision's, which may be days old. +// It fails where the fleet cannot be read or answers without its activation, +// and with an *activationAhead, a lost race (engine.ErrRevisionAdvanced), +// where the activation names another revision: converge reads again from the +// start. hash is what it runs (appliedHash), which the record takes: the one +// recorded may be another, the operator's write of the same document whose +// record the reconcile that made it never saved being taken for in sync too. +// The overtaken writes go (status.config.overtaken): the operator records a +// revision of its own document, which none of them can say more about. +func (run *configRun) inSync(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, hash string) error { + recorded := run.recordedConfig() + config := recorded.DeepCopy() + config.AppliedHash, config.Overtaken = hash, nil if recorded.RevisionID != active.RevisionID { - config := recorded.DeepCopy() + // AT THE EPOCH IT ACTIVATED AS, which only the fleet's activation + // says: the one recorded is the revision it replaces, so the + // engine read as having applied it once it applied that one, + // before it had applied this, and the Applied event paired this + // revision with the other's epoch. + fleet, err := run.fleetOf(ctx, eng) + if err != nil { + return fmt.Errorf("the engine runs the rendered document as revision %s, which the operator has not recorded, "+ + "and its fleet, which says as what epoch it activated, could not be read: %w", active.RevisionID, err) + } + switch { + case fleet.Activation == nil: + // The read of the document named a revision, so one is active: + // the engine answered its fleet without reading the pointer. + return fmt.Errorf("the engine runs the rendered document as revision %s, which the operator has not recorded, "+ + "and its fleet answered without its activation, which says as what epoch it activated", active.RevisionID) + case fleet.Activation.RevisionID != active.RevisionID: + return &activationAhead{read: active.RevisionID, activation: *fleet.Activation} + } config.RevisionID = active.RevisionID + config.Epoch = new(fleet.Activation.Epoch) + found := metav1.NewTime(run.clock()) + config.AppliedAt = &found config.Objects = run.objectCounts() + } + if !equality.Semantic.DeepEqual(*config, recorded) { run.config = config } run.recordPlacement(*run.rendered, false) // The reason of the last write (Applied, DriftCorrected) stays until // the next one: it is what that write did. - if current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied); current != nil && - current.Status == metav1.ConditionTrue { + current := meta.FindStatusCondition(run.company.Status.Conditions, crewletaiv1alpha1.ConditionConfigApplied) + switch { + case run.confirmed != "" && run.configApplied != nil: + // That write is one this reconcile has just found running, which + // the engine could not confirm, and reported (confirmUnconfirmed). + case current != nil && current.Status == metav1.ConditionTrue: kept := *current run.configApplied = &kept - } else { + default: run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonApplied, Message: fmt.Sprintf("The engine runs the rendered document as revision %s.", active.RevisionID), } } run.applied = appliedVerdict{kind: appliedInRevision, revision: active.RevisionID} + return nil } // recordApply records a revision the operator wrote. @@ -855,6 +1223,8 @@ func (run *configRun) recordApply( } run.applied = appliedVerdict{kind: appliedInRevision, revision: applied.RevisionID} switch { + case run.unsealed != "": + run.reportReplaced(&condition, applied.RevisionID, run.unsealed, crewletaiv1alpha1.ReplacedUnsealed, "") case run.policy() == crewletaiv1alpha1.ManagementPolicySeed: condition.Reason = crewletaiv1alpha1.ReasonSeeded condition.Message = fmt.Sprintf("Seeded the engine with the rendered document as revision %s; "+ @@ -863,12 +1233,8 @@ func (run *configRun) recordApply( "Seeded into revision %s; under the Company's Seed policy the engine owns the document since, and changes to this object are not applied.", applied.RevisionID)} case drifted: - condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected - condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ - "revision %s replaced it.", active.RevisionID, placesOf(diffs), applied.RevisionID) - run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", - "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ - "under the Enforce policy revision %s replaced it.", active.RevisionID, placesOf(diffs), applied.RevisionID) + run.reportReplaced(&condition, applied.RevisionID, active.RevisionID, crewletaiv1alpha1.ReplacedDrifted, + placesOf(diffs)) } run.configApplied = &condition // Activated, not applied: the engine applies it asynchronously, and @@ -881,6 +1247,39 @@ func (run *configRun) recordApply( } } +// reportReplaced says on condition, ConfigApplied for revision written, and in +// a Warning event what written replaced where that was more than the +// operator's own document: base, which was replaced (an Unsealed revision the +// engine stores without a seal, or a Drifted revision the operator did not +// write, which differed from the rendered document at driftedAt). recordApply +// reports a write the engine confirmed with it, and confirmUnconfirmed one it +// could not confirm, once a read shows it runs, so both say the same. +func (run *configRun) reportReplaced( + condition *metav1.Condition, written, base string, replaced crewletaiv1alpha1.ReplacedRevision, driftedAt string, +) { + switch replaced { + case crewletaiv1alpha1.ReplacedUnsealed: + // Not drift, which says what differed: what the revision held is + // what nobody could read. Told as a Warning all the same, under the + // reason of the condition it blocks a document with under the other + // policies, since a revision went that nobody saw. + condition.Message = fmt.Sprintf("Applied the rendered document as revision %s, replacing revision %s, which the "+ + "engine stores without a seal and does not open: a whole document replaces such a revision, reading nothing "+ + "from it.", written, base) + run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonRevisionUnreadable, "ApplyConfig", + "The engine's active revision %s was one it stores without a seal and does not open, so nobody could read "+ + "what it held and its nodes ran the revision before it; under the Enforce policy revision %s, the "+ + "rendered document, replaced it, reading nothing from it.", base, written) + case crewletaiv1alpha1.ReplacedDrifted: + condition.Reason = crewletaiv1alpha1.ReasonDriftCorrected + condition.Message = fmt.Sprintf("Revision %s, which the operator did not write, differed from the rendered document at %s; "+ + "revision %s replaced it.", base, driftedAt, written) + run.event(corev1.EventTypeWarning, crewletaiv1alpha1.ReasonDrifted, "DetectDrift", + "The engine ran revision %s, which the operator did not write and which differed from the rendered document at %s; "+ + "under the Enforce policy revision %s replaced it.", base, driftedAt, written) + } +} + // observe compares the rendered document with the engine's and reports the // difference, writing nothing: no revision, and no Secret value. func (run *configRun) observe(ctx context.Context, eng *engine.Client, active *engine.ActiveConfig) (ctrl.Result, error) { @@ -961,7 +1360,23 @@ func (run *configRun) seeded(active *engine.ActiveConfig) error { } run.recordPlacement(running, false) run.runningRevision = active.RevisionID - recorded := run.company.Status.Config.RevisionID + prior := run.recordedConfig() + recorded := prior.RevisionID + switch seed := run.overtakenSeed(); { + case recorded == "" && seed != nil: + // Seeded by a write the engine could not confirm, and replaced + // since: recorded as the seed, as a confirmed seed is. + config := prior.DeepCopy() + config.RevisionID, config.AppliedHash, config.AppliedAt = seed.RevisionID, seed.AppliedHash, seed.WrittenAt.DeepCopy() + config.Objects, config.Overtaken = seed.Objects, nil + run.config, recorded = config, seed.RevisionID + case recorded != "" && len(prior.Overtaken) > 0: + // The engine owns the document since the seed, and the operator + // writes nothing more that one of them could say anything about. + config := prior.DeepCopy() + config.Overtaken = nil + run.config = config + } message := fmt.Sprintf("The engine had a company document (revision %s) before the operator seeded one; "+ "under the Company's Seed policy the operator leaves it to the engine.", active.RevisionID) if recorded != "" { @@ -1082,8 +1497,8 @@ func (run *configRun) announce(p *part) { } if p.executor != nil { if c := turned(crewletaiv1alpha1.ConditionReady, p.executor.Reason); c != nil { - run.Recorder.Eventf(p.obj, nil, corev1.EventTypeWarning, c.Reason, "CheckExecutor", "%s: %s.", - p.executor.Field, p.executor.Message) + run.Recorder.Eventf(p.obj, nil, corev1.EventTypeWarning, c.Reason, "CheckExecutor", "%s", + ended(p.executor.Field+": "+p.executor.Message)) } } if len(p.unresolved) == 0 { @@ -1233,13 +1648,150 @@ func (run *configRun) block(reason, message string) { why, run.company.Name)} } +// engineFailed records err, the engine's, or a request's to it, which ended a +// reconcile before it could decide anything, as what it is: a refusal of the +// operator's token, a seat the document takes away while somebody holds it, a +// revision the engine cannot read, or an engine that could not be reached or +// answer now. It reads the fleet for the Seats' placement, and returns when to +// look at the engine again (engineRetry). +func (run *configRun) engineFailed(ctx context.Context, eng *engine.Client, err error) ctrl.Result { + if !run.writeRefused(err) && !run.notGranted(err) && !run.seatHeld(err) && !run.revisionUnreadable(err) { + run.unavailable(err) + } + _, _ = run.fleetOf(ctx, eng) + return ctrl.Result{RequeueAfter: run.engineRetry(err)} +} + +// engineRetry is how soon the reconcile looks at a Company's engine again +// after err. A revision the engine cannot open is looked at again at the +// resync interval, as an engine with nothing to do is: the engine sends no +// Retry-After for it, no wait opens it, and nothing the operator sends does +// while it blocks the document. What does is given to the data pods (a key +// put back in the keyring Secret, which they read only when they start, or +// the build that sealed the revision), and the data pods restarting changes +// RuntimeReady, which runs the reconcile at once; or, for a revision stored +// without a seal, the Company's policy set to Enforce, which runs it at once +// too, or a whole document an admin sends. Under Enforce, though, the +// operator's own next look replaces a revision stored without a seal (one +// that landed after the reconcile's read, which is the only way one reaches +// here), so that look comes engineRetryInterval on rather than a resync +// later. Anything else is asked again as retryAfter says. +func (run *configRun) engineRetry(err error) time.Duration { + if _, ok := engine.Unsealed(err); ok && run.policy() == crewletaiv1alpha1.ManagementPolicyEnforce { + return engineRetryInterval + } + if errors.Is(err, engine.ErrRevisionUnreadable) { + return run.resyncInterval() + } + return retryAfter(err) +} + +// revisionUnreadable records an engine that cannot open the fleet's revision +// of the company document (engine.ErrRevisionUnreadable), and reports false +// for any other error. The engine shows that document to nobody and takes no +// write built from it, so the operator can neither see the document nor +// change it, but for the one write the engine takes over a revision stored +// without a seal: a whole document, which reads nothing from it and replaces +// it, and which converge sends under the Enforce policy. It is reported as +// what blocks the document, in the engine's words, with what restores the +// revision or replaces it: a key the revision is sealed under is one of the +// keyring Secret's, which the operator renders into every pod and never +// rewrites, so a key gone from it is put back there, from the backup kept with +// the data volumes, and a key the fleet sealed under that the data pod lacks +// is one it takes when it restarts with the Secret; a seal the engine's build +// does not read is most likely a newer build's, which the data pods run +// again, since no write replaces a document somebody sealed; and a revision +// without a seal no key opens, and a whole document replaces it: the rendered +// one under Enforce (unsealedRemedy). +func (run *configRun) revisionUnreadable(err error) bool { + var apiErr *engine.APIError + if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) { + return false + } + revision := "the revision" + if apiErr.RevisionID != "" { + revision = "revision " + apiErr.RevisionID + } + said := fmt.Sprintf("The engine's keyring does not open %s of the company document, so the engine neither shows "+ + "the document nor takes another, and no wait changes that", revision) + var remedy string + switch apiErr.Code { + case "unsealed_revision", "unsealed_fleet_revision": + said = fmt.Sprintf("The engine does not open %s of the company document, which it stores without a seal, so "+ + "the engine shows the document to nobody and takes no write but a whole document that replaces it, reading "+ + "nothing from it, and no wait changes that", revision) + remedy = "A revision without a seal was stored by an engine older than the keyring every engine now " + + "requires, and no keyring opens it. " + if apiErr.Code == "unsealed_fleet_revision" { + remedy = "Every engine seals what it activates, so a revision on the fleet's activation pointer without a " + + "seal was put there from outside the fleet, and no keyring opens it. " + } + remedy += run.unsealedRemedy() + case "unknown_seal_revision": + said = fmt.Sprintf("The engine does not read the seal of %s of the company document, so the engine neither "+ + "shows the document nor takes another, and no wait changes that", revision) + class := "the Company's EngineClass" + if run.class != nil { + class = "EngineClass " + run.class.Name + } + remedy = fmt.Sprintf("A build of Crewlet newer than the one the data pod runs most likely sealed it, so the "+ + "keyring Secret %s is not at fault, and no write replaces it, since that would discard it unread. Run the "+ + "data pods on the build that sealed it: the engine image of %s, with the Company's spec.runtime.version as "+ + "its tag where the class allows that. A data pod still rolling to it takes it when it restarts.", + keyringSecretName(run.company), class) + case "unreadable_fleet_revision": + remedy = fmt.Sprintf("Every pod of the company has its keyring from Secret %s: a data pod that has not "+ + "restarted since the Secret gained the key the fleet sealed %s under takes it when it restarts, and a key "+ + "gone from the Secret is put back in it from the backup kept with the data volumes.", + keyringSecretName(run.company), revision) + default: + remedy = fmt.Sprintf("The keyring is Secret %s, which the operator never rewrites: put the key %s is sealed "+ + "under back in it from the backup kept with the data volumes, and the data pods restart with it.", + keyringSecretName(run.company), revision) + } + run.blockAndTell(crewletaiv1alpha1.ReasonRevisionUnreadable, "ReadConfig", + ended(fmt.Sprintf("%s: %s", said, err))+" "+remedy) + return true +} + +// unsealedRemedy is what replaces a revision the engine stores without a seal, +// under the Company's policy: a whole document, which the operator sends +// itself only under Enforce. Seed writes only into an engine with no document +// and Observe writes nothing, so under those an admin of the company sends +// one, as the engine's hint says, or the policy is set to Enforce. Under +// Enforce converge replaces it as it reads it, so it blocks a document only +// where it lands after that read, until the next look. +func (run *configRun) unsealedRemedy() string { + switch policy := run.policy(); policy { + case crewletaiv1alpha1.ManagementPolicyEnforce: + return "Under the Company's Enforce policy the operator replaces it with the rendered document at its next look." + case crewletaiv1alpha1.ManagementPolicySeed: + return "Under the Company's Seed policy the operator writes only into an engine with no document, so it does " + + "not replace it: an admin of the company replaces it with a whole document, as the engine's hint says, or " + + "the Company's policy is set to Enforce, under which the operator replaces it with the rendered document." + default: + return fmt.Sprintf("Under the Company's %s policy the operator writes nothing, so it does not replace it: an "+ + "admin of the company replaces it with a whole document, as the engine's hint says, or the Company's "+ + "policy is set to Enforce, under which the operator replaces it with the rendered document.", policy) + } +} + // unavailable records an engine that could not be reached, or that failed a // request for a reason of its own. func (run *configRun) unavailable(err error) { + message := fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s", + retryAfter(err), err) + if engine.Waitless(err) { + // The engine says waiting won't change what it met, so a promise + // of a retry in ten seconds would contradict the words it quotes: + // the operator asks again only because nothing else tells it that + // somebody has cleared it. + message = fmt.Sprintf("The engine answered that waiting won't clear what it met, so somebody has to clear it "+ + "in the engine; the operator asks again every %s until then: %s", engineRetryInterval, err) + } run.configApplied = &metav1.Condition{ Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, - Reason: crewletaiv1alpha1.ReasonEngineUnavailable, - Message: fmt.Sprintf("The operator's request to the company's engine failed, and it tries again in %s: %s.", retryAfter(err), err), + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, Message: ended(message), } } @@ -1298,7 +1850,10 @@ func (run *configRun) notGranted(err error) bool { // seatHeld records an engine that refuses the document because it takes a // human seat out of the company, or makes it an agent's, while something in // the engine's identity directory holds the seat (engine.SeatHeldError), and -// reports false for any other error. The document is valid and stays +// reports false for any other error. A document replacing a revision stored +// without a seal (run.unsealed) is refused for every seat it does not keep as +// a human seat that something holds, since nobody can read which seats that +// revision had, and is told so. The document is valid and stays // unapplied until an admin of the company frees each seat, which the // operator neither can nor should do: a seat's holder is a person, or what a // person set up. The message names each seat and what kind of thing holds @@ -1346,10 +1901,18 @@ func (run *configRun) seatHeld(err error) bool { if len(remedies) > 0 { remedy = ": " + strings.Join(remedies, "; ") } - message := fmt.Sprintf("The engine refuses the rendered document: it takes human seats out of the company, or makes "+ - "them agents', while its identity directory holds them: %s. An admin of the company frees each seat first, in "+ - "the engine's dashboard or with `crewlet iam` (`crewlet iam seats` lists what holds each seat)%s. Or keep "+ - "each seat as a Human Seat.", strings.Join(seats, "; "), remedy) + what := "it takes human seats out of the company, or makes them agents', while its identity directory holds them" + if run.unsealed != "" { + // Replacing a revision nothing was read from, the engine asked about + // every seat the document does not keep as a human seat, so it + // cannot say the revision it replaces had them, and neither can the + // operator. + what = fmt.Sprintf("it replaces revision %s, which the engine does not open, and does not keep as human seats "+ + "these seats its identity directory holds", run.unsealed) + } + message := fmt.Sprintf("The engine refuses the rendered document: %s: %s. An admin of the company frees each seat "+ + "first, in the engine's dashboard or with `crewlet iam` (`crewlet iam seats` lists what holds each seat)%s. Or "+ + "keep each seat as a Human Seat.", what, strings.Join(seats, "; "), remedy) run.blockAndTell(crewletaiv1alpha1.ReasonSeatHeld, "DryRunConfig", message) return true } @@ -1434,9 +1997,11 @@ func placesOf(diffs []render.Path) string { } // summary is the X-Summary of a write: "crewlet-operator: " and the objects -// whose part of the document changed, or the Secrets whose rotated values -// the write makes the engine read. -func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, put []string) string { +// whose part of the document changed, or the Secrets whose values, written +// since a revision last activated (written, secretsync.State.Written), the +// write makes the engine read; for a write that replaces a revision stored +// without a seal (run.unsealed), that revision, and every object. +func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, written []string) string { var objects []string add := func(s string) { if !slices.Contains(objects, s) { @@ -1459,7 +2024,7 @@ func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, } } for _, k := range run.doc.Refs { - if slices.Contains(put, secretsync.StoreName(k)) && active != nil && len(diffs) == 0 { + if slices.Contains(written, secretsync.StoreName(k)) && active != nil && len(diffs) == 0 { add("Secret " + k.Namespace + "/" + k.Name) } } @@ -1473,6 +2038,11 @@ func (run *configRun) summary(active *engine.ActiveConfig, diffs []render.Path, if len(objects) > shown { text += fmt.Sprintf(" and %d more", len(objects)-shown) } + if run.unsealed != "" { + // What it replaced is the one record of it the engine's history + // keeps: the revision itself shows nothing. + return fmt.Sprintf("crewlet-operator: replaced revision %s, stored without a seal, with %s", run.unsealed, text) + } return "crewlet-operator: " + text } @@ -1538,7 +2108,7 @@ func engineHealthy(alarms *engine.Alarms, err error) *metav1.Condition { switch { case err != nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonAlarmsNotReported - c.Message = "The engine's health could not be read: " + err.Error() + c.Message = ended("The engine's health could not be read: " + err.Error()) case alarms == nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonAlarmsNotReported c.Message = "The engine has not evaluated its alarms yet." @@ -1620,7 +2190,7 @@ func identityClaimed(health engine.Health, err error, humanSeats []string, acces switch { case err != nil: c.Status, c.Reason = metav1.ConditionUnknown, crewletaiv1alpha1.ReasonIdentityNotReported - c.Message = "The engine's health could not be read: " + err.Error() + c.Message = ended("The engine's health could not be read: " + err.Error()) case health.Identity == engine.IdentityReady: c.Status, c.Reason = metav1.ConditionTrue, crewletaiv1alpha1.ReasonPersonEnrolled c.Message = "A person has joined the company: the engine's identity directory holds someone on a Human seat, " + diff --git a/internal/controller/config_serving.go b/internal/controller/config_serving.go index 8abdcb51..487d2f0c 100644 --- a/internal/controller/config_serving.go +++ b/internal/controller/config_serving.go @@ -396,7 +396,7 @@ func (run *configRun) heldSentence() string { if len(run.serving.unread) > 0 { why = append(why, "a revision the operator does not know of "+strings.Join(run.serving.unread, ", ")) } - return fmt.Sprintf(" It keeps %d %s the revision the engine is to run no longer needs, for %s, until no pod runs or "+ - "boots on a revision that does: %s.", len(held), plural(len(held), "value", "values"), strings.Join(why, "; and "), - strings.Join(held, ", ")) + return " " + ended(fmt.Sprintf("It keeps %d %s the revision the engine is to run no longer needs, for %s, until no pod "+ + "runs or boots on a revision that does: %s", len(held), plural(len(held), "value", "values"), strings.Join(why, "; and "), + strings.Join(held, ", "))) } diff --git a/internal/controller/config_status.go b/internal/controller/config_status.go index 52b0eb5d..f0b68bde 100644 --- a/internal/controller/config_status.go +++ b/internal/controller/config_status.go @@ -169,7 +169,7 @@ func (run *configRun) decideApplied(p *part) { switch { case len(p.drifted) > 0: setCondition(conditions, gen, crewletaiv1alpha1.ConditionApplied, false, crewletaiv1alpha1.ReasonDrifted, - "The engine's document differs from this object's part of the rendered one at: "+strings.Join(p.drifted, ", ")+".") + ended("The engine's document differs from this object's part of the rendered one at: "+strings.Join(p.drifted, ", "))) default: setCondition(conditions, gen, crewletaiv1alpha1.ConditionApplied, true, crewletaiv1alpha1.ReasonInRevision, fmt.Sprintf("In revision %s, which the engine runs.", verdict.revision)) diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go index 4888ab82..ac57db9c 100644 --- a/internal/controller/config_test.go +++ b/internal/controller/config_test.go @@ -1097,10 +1097,17 @@ var _ = Describe("Company config", Ordered, func() { applied(c, crewletaiv1alpha1.ReasonApplied) By("a person activating the running document again") - _, err := engine.New(e.URL, e.AccessToken).ReloadConfig(ctx, "reloaded by a person") + reloaded, err := engine.New(e.URL, e.AccessToken).ReloadConfig(ctx, "reloaded by a person") Expect(err).NotTo(HaveOccurred()) Eventually(count).WithTimeout(settle).Should(Equal(3)) Consistently(count).WithTimeout(3 * quietPeriod).Should(Equal(3)) + Eventually(func(g Gomega) crewletaiv1alpha1.ConfigStatus { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(c), c)).To(Succeed()) + return c.Status.Config + }).WithTimeout(settle).Should(SatisfyAll( + HaveField("RevisionID", reloaded.RevisionID), + HaveField("Epoch", HaveValue(Equal(reloaded.Epoch))), + ), "the reload is adopted at its own epoch, not at the epoch of the revision it replaced") By("deleting the key's Secret") Expect(k8sClient.Delete(ctx, &corev1.Secret{Namespace: namespace, Name: name + "-npm"})).To(Succeed()) @@ -1109,7 +1116,10 @@ var _ = Describe("Company config", Ordered, func() { Consistently(count).WithTimeout(6*time.Second).Should(Equal(4), "a revision was activated with the deleted Secret's value in the store, and activated again") Eventually(startedWith).WithTimeout(settle).ShouldNot(HaveKey(optionalRegistry)) - Expect(log.of(name)).NotTo(ContainElement(HavePrefix("Normal Reactivated "))) + // The engine runs the rendered document, which read every value the + // store still holds, so the one activation owed is the reload of it + // without the deleted value, not a write of the same document again. + Expect(log.of(name)).To(ContainElement(HavePrefix("Normal Reactivated "))) Expect(log.of(name)).NotTo(ContainElement(HavePrefix("Normal SecretKept "))) applied(c, crewletaiv1alpha1.ReasonApplied) runsRendered(e, c) diff --git a/internal/controller/config_unconfirmed.go b/internal/controller/config_unconfirmed.go new file mode 100644 index 00000000..bbb3d452 --- /dev/null +++ b/internal/controller/config_unconfirmed.go @@ -0,0 +1,374 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// An activation the engine could not confirm. +// +// A write of the company document stores a revision, then asks the fleet to +// activate it by a compare-and-set on its activation pointer, conditioned on +// the revision the write was built on; so does a reload, which activates the +// running revision again, conditioned on that one. When the answer to the +// compare-and-set is lost, the engine answers 503 naming the revision it +// stored (engine.UnconfirmedActivation): the pointer may name it or may not. +// Nothing the engine says afterwards says which write a revision was, so +// without a record of it a write that did land read on the next reconcile as +// a revision somebody else activated: under Enforce, where its document is +// the rendered one, a revision the operator adopts as one it did not write, +// and where the render had moved on since, an edit somebody else made, which +// the next write reported as drift it corrected; under Seed, a document the +// engine had before the operator seeded one. +// +// So each such write is recorded (status.config.unconfirmed) with what a +// confirmed write records, and settled by the next read of the revision the +// engine runs, before anything is written. The engine runs one of them: that +// write landed, at the epoch the fleet's activation names, and it is recorded +// as the operator's, as a confirmed write is. The engine runs the revision +// one was built on: that one has not landed yet, and still may, so it stays +// and the write is sent again on the same condition, which the engine refuses +// once one of them lands (and the reconcile then reads it, and records it). +// Every other one can no longer land, since it was conditioned on a revision +// the fleet has moved past, and goes; but for one written into an engine that +// ran no document, which under Seed is the seed whatever revision runs now, +// since the engine stored it while it had none, so either it landed and was +// replaced since, or another writer's create reached the engine first, which +// nothing the engine says tells apart (status.config.overtaken, overtakenSeed). +// It stays recorded until a reconcile decides with it, since one that fails +// before it gets that far must not lose it. +// +// What the write's activation read of the secret store is settled two ways. +// The changes it read were marked with the fleet's activation epoch before it +// was sent (markChanges), and the next read of a revision the engine runs +// above that epoch settles them, whoever activated it (settleRead). A change +// no read of the pointer followed, which it cannot have where the pointer was +// unreadable, and that is when the engine is likeliest to lose an answer, +// awaits the revision the engine stored instead (secretsync.State.Awaiting, +// saved as the answer comes back), and finding that revision running settles +// it (secretsync.State.Found). What a write needs beside that is how its +// document reads every name, which the state records under the write's +// appliedHash before it is sent (secretsync.State.Pending), since nothing +// else says it once the render has moved on; the revision found running a +// write of the operator's records that reading (settlePending). A reconcile +// whose document is held settles the state the same way, and leaves +// status.config, and the report of the write, to the reconcile that applies +// one: its conditions say why it holds. + +// maxUnconfirmed bounds status.config.unconfirmed and status.config.overtaken +// (their MaxItems): writes whose answers are all lost come ten seconds apart +// while the engine cannot reach its coordination store, and an activation +// that has neither landed nor been refused sixteen writes later is not one a +// later read will see land. +const maxUnconfirmed = 16 + +// recordedConfig is status.config as this reconcile leaves it: what it decided +// to write over the Company's record, where it decided anything. +func (run *configRun) recordedConfig() crewletaiv1alpha1.ConfigStatus { + if run.config != nil { + return *run.config + } + return run.company.Status.Config +} + +// boundedWrites is writes without the oldest past maxUnconfirmed. +func boundedWrites(writes []crewletaiv1alpha1.UnconfirmedRevision) []crewletaiv1alpha1.UnconfirmedRevision { + if extra := len(writes) - maxUnconfirmed; extra > 0 { + return writes[extra:] + } + return writes +} + +// unconfirmedIndex is the index in writes of the write of revision, -1 for +// none. +func unconfirmedIndex(writes []crewletaiv1alpha1.UnconfirmedRevision, revision string) int { + if revision == "" { + return -1 + } + return slices.IndexFunc(writes, func(write crewletaiv1alpha1.UnconfirmedRevision) bool { + return write.RevisionID == revision + }) +} + +// ownWrite is the record that makes active, the revision the engine runs as +// this reconcile read it, a write of the operator's, as +// status.config.unconfirmed holds one: status.config where it records that +// revision, or a write the engine could not confirm activating that is it +// and runs its document (runsItsDocument). ok is false for a revision the +// operator did not write, for one it recorded without the hash of what it +// runs, and for none. +func (run *configRun) ownWrite(active *engine.ActiveConfig) (write crewletaiv1alpha1.UnconfirmedRevision, ok bool) { + if active == nil { + return crewletaiv1alpha1.UnconfirmedRevision{}, false + } + recorded := run.recordedConfig() + if recorded.RevisionID == active.RevisionID && recorded.AppliedHash != "" { + return crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: active.RevisionID, AppliedHash: recorded.AppliedHash, Objects: recorded.Objects, + }, true + } + if i := unconfirmedIndex(recorded.Unconfirmed, active.RevisionID); i >= 0 && + runsItsDocument(&recorded.Unconfirmed[i], active) { + return recorded.Unconfirmed[i], true + } + return crewletaiv1alpha1.UnconfirmedRevision{}, false +} + +// runsItsDocument reports whether active, the revision write records, runs +// the document write was recorded with: a reload's (documentSHA256), since +// the engine activates again whichever revision is active when the request +// lands, so the revision it stored runs somebody else's document where they +// activated one in between. A write of the rendered document runs what the +// engine stored for it, and its record names no document. +func runsItsDocument(write *crewletaiv1alpha1.UnconfirmedRevision, active *engine.ActiveConfig) bool { + return write.DocumentSHA256 == "" || active != nil && write.DocumentSHA256 == documentDigest(active.Document) +} + +// documentDigest is documentSHA256 of document: its SHA-256, in hex. +func documentDigest(document []byte) string { + sum := sha256.Sum256(document) + return hex.EncodeToString(sum[:]) +} + +// unconfirmed records write, whose activation the engine could not confirm, +// as revision storedID, reports it on ConfigApplied, and returns when the next +// reconcile settles it (settleUnconfirmed): as soon as the engine's +// Retry-After says, the scale of one batch on its coordination store, and ten +// seconds on where it sends none (engineRetry). +func (run *configRun) unconfirmed(storedID string, write writeAttempt, err error) time.Duration { + retry := run.engineRetry(err) + recorded := run.recordedConfig() + config := recorded.DeepCopy() + now := metav1.NewTime(run.clock()) + config.Unconfirmed = boundedWrites(append(config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: storedID, BaseRevisionID: write.base, AppliedHash: write.hash, WrittenAt: &now, + Objects: run.objectCounts(), Replaced: write.replaced, DriftedAt: write.driftedAt, + })) + run.config = config + stored := fmt.Sprintf("revision %s", storedID) + if write.replaced == crewletaiv1alpha1.ReplacedUnsealed { + // Said now, since the next report may be a while: the engine + // runs a revision nobody can read, which the write would replace. + stored += fmt.Sprintf(", which would replace revision %s, stored without a seal,", write.base) + } + run.configApplied = &metav1.Condition{ + Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionFalse, + Reason: crewletaiv1alpha1.ReasonEngineUnavailable, + Message: ended(fmt.Sprintf("The engine stored the rendered document as %s and could not confirm that it "+ + "activated it, so it may run it or may not. The operator reads which revision the engine runs before it writes "+ + "again, in %s: %s", stored, retry, err)), + } + return retry +} + +// reloadUnconfirmed records stored, the revision a request to activate active +// again stored and could not confirm activating, as a write of the operator's +// of active's document, where active is the operator's (ownWrite): it was +// conditioned on active, and may land yet, and once a read finds it running it +// is recorded and reported as that write would be (settleUnconfirmed), rather +// than as a revision somebody else activated. A revision the operator did not +// write stays somebody else's when activated again. +// +// It is recorded with the digest of active's document (documentSHA256), since +// the request names no revision: where somebody activated another revision +// before it landed, the engine stored that one's document, which a read then +// finds running under stored, and that revision is not the operator's +// (runsItsDocument). And it replaced nothing of its own: its base is the +// operator's write, and what that write replaced, where it is recorded here +// too, is reported once, whichever of the two is confirmed +// (confirmUnconfirmed); copied onto this record, it named the operator's own +// write as one somebody else made, or as one stored without a seal. +func (run *configRun) reloadUnconfirmed(active *engine.ActiveConfig, stored string) { + own, ok := run.ownWrite(active) + recorded := run.recordedConfig() + if !ok || unconfirmedIndex(recorded.Unconfirmed, stored) >= 0 { + return + } + config := recorded.DeepCopy() + now := metav1.NewTime(run.clock()) + own.RevisionID, own.BaseRevisionID, own.WrittenAt = stored, active.RevisionID, &now + own.Replaced, own.DriftedAt, own.DocumentSHA256 = "", "", documentDigest(active.Document) + config.Unconfirmed = boundedWrites(append(config.Unconfirmed, own)) + run.config = config +} + +// overtakenBy splits writes, of status.config.unconfirmed, by runs, a revision +// none of them is that the engine runs: still are the ones built on runs, +// which may land yet, and overtaken the others written into an engine that ran +// no document, which Seed takes for the seed (overtakenSeed). Every other one +// can no longer land, and nothing reads it again. +func overtakenBy( + writes []crewletaiv1alpha1.UnconfirmedRevision, runs string, +) (still, overtaken []crewletaiv1alpha1.UnconfirmedRevision) { + for i := range writes { + switch writes[i].BaseRevisionID { + case runs: + still = append(still, writes[i]) + case "": + overtaken = append(overtaken, writes[i]) + } + } + return still, overtaken +} + +// settleUnconfirmed settles the writes status.config.unconfirmed records +// against runs, the revision the engine runs as this reconcile read it, empty +// for none: active, whose document it read, or one it could not open whose +// write the reconcile replaces (configRun.unsealed), on which a write the +// engine could not confirm may have been built as well. One of them runs, with +// the document it was recorded with (runsItsDocument): it is recorded as the +// operator's write (confirmUnconfirmed). Otherwise the ones built on runs +// stay, and every other one goes, to status.config.overtaken where it was +// written into an engine that ran no document; a reload found running +// another document was built on a revision somebody else activated, and goes +// too. Under Observe nothing is settled: the operator writes nothing there, +// and Observe records the revision it finds in sync. +func (run *configRun) settleUnconfirmed( + ctx context.Context, eng *engine.Client, runs string, active *engine.ActiveConfig, +) error { + recorded := run.recordedConfig() + if len(recorded.Unconfirmed) == 0 || run.policy() == crewletaiv1alpha1.ManagementPolicyObserve { + return nil + } + if i := unconfirmedIndex(recorded.Unconfirmed, runs); i >= 0 && runsItsDocument(&recorded.Unconfirmed[i], active) { + return run.confirmUnconfirmed(ctx, eng, &recorded, i) + } + still, overtaken := overtakenBy(recorded.Unconfirmed, runs) + if len(still) < len(recorded.Unconfirmed) { + config := recorded.DeepCopy() + config.Unconfirmed = still + config.Overtaken = boundedWrites(append(config.Overtaken, overtaken...)) + run.config = config + } + return nil +} + +// confirmUnconfirmed records recorded.Unconfirmed[i], a revision the engine +// runs, as the operator's write, and reports it, as recordApply records and +// reports one the engine confirmed: at the epoch the fleet's activation names, +// which only a read of it gives, since the engine answered none; from now, +// since nothing says when the activation landed, and the engine is held to +// applying its epoch applyDeadline after the operator found it running, as an +// adopted revision is; and saying what it replaced (reportReplaced), which the +// record kept, or where it is a reload of a write recorded here too, what +// that write replaced, which only it records (reloadUnconfirmed), and which +// then goes with it: the revision it activated again landed, since the reload +// runs. A write built on it stays unconfirmed, since it may land yet: +// a reconcile that holds its document reads the engine without recording what +// it finds (status.config is left to the one that applies a document), so it +// may find this one running and activate it again for a value it withdrew, and +// that reload's answer can be lost too (reloadUnconfirmed). Every other one is +// overtaken by it, and so is every write status.config.overtaken keeps: none +// can land now, nor be the seed, since a write of the operator's runs. +// +// It fails where the fleet cannot be read, or answers without its activation, +// and with an *activationAhead, a lost race, where the activation names +// another revision: converge reads again from the start, and waits for the +// node it reads where the second read finds the same. +func (run *configRun) confirmUnconfirmed( + ctx context.Context, eng *engine.Client, recorded *crewletaiv1alpha1.ConfigStatus, i int, +) error { + write := recorded.Unconfirmed[i] + fleet, err := run.fleetOf(ctx, eng) + if err != nil { + return fmt.Errorf("the engine runs revision %s, which the operator wrote and the engine could not confirm "+ + "activating, and its fleet, which says as what epoch, could not be read: %w", write.RevisionID, err) + } + switch { + case fleet.Activation == nil: + // The read of the document named the revision, so it is active: + // the engine answered its fleet without reading the pointer. + return fmt.Errorf("the engine runs revision %s, which the operator wrote and the engine could not confirm "+ + "activating, and its fleet answered without its activation, which says as what epoch", write.RevisionID) + case fleet.Activation.RevisionID != write.RevisionID: + return &activationAhead{read: write.RevisionID, activation: *fleet.Activation, unconfirmed: true} + } + epoch := fleet.Activation.Epoch + // The write a reload activated again, where that is recorded here too, + // and so on down: the first of them says what the rendered document + // replaced. + first, landed := write, []string{write.RevisionID} + for { + j := unconfirmedIndex(recorded.Unconfirmed, first.BaseRevisionID) + if j < 0 || slices.Contains(landed, recorded.Unconfirmed[j].RevisionID) { + break + } + first = recorded.Unconfirmed[j] + landed = append(landed, first.RevisionID) + } + still, _ := overtakenBy(slices.DeleteFunc(slices.Clone(recorded.Unconfirmed), + func(w crewletaiv1alpha1.UnconfirmedRevision) bool { return slices.Contains(landed, w.RevisionID) }), + write.RevisionID) + found := metav1.NewTime(run.clock()) + run.confirmed = write.RevisionID + run.config = &crewletaiv1alpha1.ConfigStatus{ + RevisionID: write.RevisionID, Epoch: new(epoch), AppliedHash: write.AppliedHash, AppliedAt: &found, + Objects: write.Objects, Unconfirmed: still, + } + condition := metav1.Condition{ + Type: crewletaiv1alpha1.ConditionConfigApplied, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonApplied, + Message: fmt.Sprintf("Applied the rendered document as revision %s.", write.RevisionID), + } + run.reportReplaced(&condition, write.RevisionID, first.BaseRevisionID, first.Replaced, first.DriftedAt) + run.configApplied = &condition + run.applied = appliedVerdict{kind: appliedInRevision, revision: write.RevisionID} + run.event(corev1.EventTypeNormal, "Activated", "ApplyConfig", + "The engine runs revision %s as epoch %d: the operator activated it, and the engine could not confirm the "+ + "activation when it was made.", write.RevisionID, epoch) + return nil +} + +// settlePending is state recording the reading of active, the revision the +// engine runs as this reconcile read it, where that is a write of the +// operator's whose reading state does not record (ownWrite): the write's +// activation went unconfirmed, or the save of its record after it was lost, +// and the reading state recorded under the write's appliedHash before it was +// sent says how its document reads every name (secretsync.State.Confirmed). +// The readings of the writes status.config.unconfirmed no longer names go: no +// revision the operator has not recorded can be one of them. A revision state +// already records keeps its reading, and so does one the operator did not +// write, or none, which the caller adopts where its document is the rendered +// one. +func (run *configRun) settlePending(state secretsync.State, active *engine.ActiveConfig) secretsync.State { + if active != nil && !state.Records(active.RevisionID) { + if own, ok := run.ownWrite(active); ok { + if confirmed, ok := state.Confirmed(active.RevisionID, own.AppliedHash); ok { + state = confirmed + } + } + } + recorded := run.recordedConfig() + keep := make([]string, len(recorded.Unconfirmed)) + for i := range recorded.Unconfirmed { + keep[i] = recorded.Unconfirmed[i].AppliedHash + } + return state.KeepPending(keep) +} + +// overtakenSeed is the write into an engine with no document, of +// status.config.overtaken, that seeded the engine under Seed, nil for none: the +// engine had no document when it was written, and runs one now, so that write +// landed, and the revision that runs replaced it, or another write reached +// the engine first in the moment between the reconcile's read and that write, +// which nothing the engine says tells apart, and which the operator takes for +// the first, whatever revision runs now. +func (run *configRun) overtakenSeed() *crewletaiv1alpha1.UnconfirmedRevision { + recorded := run.recordedConfig() + for i := range recorded.Overtaken { + if write := &recorded.Overtaken[i]; write.BaseRevisionID == "" { + return write + } + } + return nil +} diff --git a/internal/controller/config_unconfirmed_test.go b/internal/controller/config_unconfirmed_test.go new file mode 100644 index 00000000..4f524ee4 --- /dev/null +++ b/internal/controller/config_unconfirmed_test.go @@ -0,0 +1,1460 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/yaml" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretref" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// These tests hold what the operator does with a write whose activation the +// engine sent to its fleet and could not confirm: it never sends the write +// again on top of it, and a write that did land is recorded as its own, under +// Enforce and under Seed alike. The engine gives that answer only when its +// coordination store loses the answer to a compare-and-set, which a real +// engine cannot be made to do at will, so the storeEngine stands in for it. + +// outcome is how the activation of the storeEngine's next write goes. +type outcome int + +const ( + // confirmed: it activates, and the engine says so. + confirmed outcome = iota + // landed: it activates, and the engine's answer is lost. + landed + // pending: the revision is stored and not activated, and the engine's + // answer is lost. + pending +) + +// land activates the inert revision id, as a compare-and-set whose answer was +// lost lands after all. The caller holds e's lock. +func (e *storeEngine) land(id string) { + i := slices.IndexFunc(e.inert, func(r storeRevision) bool { return r.id == id }) + e.revisions = append(e.revisions, e.inert[i]) + e.inert = slices.Delete(e.inert, i, i+1) +} + +// r1Hash is the appliedHash a test records r1 under where r1 is the +// operator's own write. +const r1Hash = "sha256:r1" + +// answerLost is the engine's answer to a write whose activation it sent and +// whose answer its coordination store did not give (crewlet's +// internal/api/configapi, RefuseActivationUnknown): 503 unavailable with the +// unknown outcome, the revision it stored, the node whose history alone holds +// it, the revision the write was built on, and a hint saying how to look +// first, ended as httpjson.Sentence ends it, under the Retry-After of an +// undecided answer (authz.RetryUndecidedSeconds). +func answerLost(w http.ResponseWriter, stored, base string) { + hint := fmt.Sprintf("Revision %[1]s was stored on node %[2]s, and the coordination store did not answer its "+ + "activation, so whether the fleet runs it is unknown. GET /config names the active revision once the store "+ + "answers again. If it is not %[1]s, the revision is in node %[2]s's history alone: send GET "+ + "/config/revisions/%[1]s/diff, and POST /config/revisions/%[1]s/revert to make it the company, to that "+ + "node \u2014 by its own address where a shared one fronts several, since any other node holds no copy and "+ + "answers 404. Sending the write again makes another revision.", stored, storeNode) + body := map[string]string{ + "error": "unavailable", "message": "Can't answer right now. Try again in a moment, or ask another node.", + "outcome": "unknown", "stored_revision_id": stored, "node": storeNode, "hint": hint, + } + if base != "" { + body["your_base"] = base + } + w.Header().Set("Retry-After", "2") + answerJSON(w, http.StatusServiceUnavailable, body) +} + +// unconfirmedWrite is the withdrawal's reconcile under Enforce after the +// operator's write of the rendered document, built on r1, came back +// unconfirmed as revision r2, which how lands or keeps inert; and the Company +// as that reconcile's status write leaves it. A reconcile that fails is +// reported as the reconciler reports it, and retried. +func unconfirmedWrite(t *testing.T, how outcome) (*withdrawal, *crewletaiv1alpha1.Company) { + t.Helper() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + // r1 is the rendered document and lets the registry go, so the + // withdrawal alone is settled by activating r1 again; a rotated value + // is what has the operator write the document, where the engine's own + // re-activation at the sync's write fails. + w.rotate(t) + w.engine.activation = how + operatorActivates(w) + reconcileOnce(t, w) + w.engine.activation, w.engine.reloadAnswer = confirmed, nil + company := w.statusWritten() + return w, company +} + +// operatorActivates makes the engine's own re-activation at a sync's write or +// removal fail, as an engine whose coordination store does not answer it +// does, so that the activation is the operator's: the write or the reload +// whose answer these tests lose. +func operatorActivates(w *withdrawal) { + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") +} + +// reconcileOnce runs w's reconcile up to its status write, as the reconciler +// does: an error it fails with is reported on ConfigApplied. +func reconcileOnce(t *testing.T, w *withdrawal) { + t.Helper() + if _, err := w.run.converge(context.Background(), w.client); err != nil { + t.Logf("the reconcile failed, and is retried: %v", err) + w.run.unavailable(err) + } +} + +// TestAnActivationTheEngineCouldNotConfirmIsRecorded: the write the engine +// could not confirm is recorded with what a confirmed one records, and the +// reconcile asks again soon, reporting it. +func TestAnActivationTheEngineCouldNotConfirmIsRecorded(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || got[0].AppliedHash == "" || + got[0].WrittenAt.IsZero() { + t.Fatalf("status.config.unconfirmed is %+v, want r2, built on r1, with its hash and when", got) + } + c := w.run.configApplied + if c == nil || c.Status != metav1.ConditionFalse || c.Reason != crewletaiv1alpha1.ReasonEngineUnavailable || + !strings.Contains(c.Message, "revision r2 and could not confirm") { + t.Errorf("ConfigApplied is %+v, want False, EngineUnavailable, saying r2 could not be confirmed", c) + } +} + +// TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice: the write did +// land, so the next reconcile finds the engine running it, records it as the +// operator's at the epoch the fleet's activation names, and writes nothing; +// and its activation is recorded as a confirmed write's is, as the one that +// read the store without the withdrawal's registry, which the sync before it +// deleted, so nothing activates r2 again for it either. The reading recorded +// for it goes, nothing unconfirmed naming it any more. +func TestAnActivationTheEngineCouldNotConfirmIsNotWrittenTwice(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) { + t.Fatalf("the write's sync withdrew %v, want the registry", state.Withdrawn()) + } + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone: the engine ran it", written) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 activated without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !state.Records("r2") || + (len(claims) == 1 && state.Pends(claims[0].AppliedHash)) { + t.Errorf("the state owes %v and records %q, want nothing owed, r2's reading, and no reading left pending, "+ + "since nothing unconfirmed names it", state.Withdrawn(), state.Recorded()) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.AppliedHash != claims[0].AppliedHash || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r2's write recorded, with its hash, and nothing unconfirmed", config) + } + recorded := drained(w.events) + if i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated ") }); i < 0 || + !strings.Contains(recorded[i], "revision r2 as epoch 2") { + t.Errorf("events %q, want an Activated event saying the engine runs r2 as epoch 2", recorded) + } +} + +// TestAnActivationThatLandsLateIsTheOperatorsOwn: the write had not landed by +// the reconcile that wrote it, and lands before the next one reads, which +// records it as the operator's write and says so, rather than adopting it as +// a revision of the rendered document somebody else activated. +func TestAnActivationThatLandsLateIsTheOperatorsOwn(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + written, _ := w.engine.writes() + if config := w.statusWritten().Status.Config; !slices.Equal(written, []string{"r2"}) || len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed, want r2 alone, recorded", written, config.Unconfirmed) + } + if recorded := drained(w.events); !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) { + t.Errorf("events %q, want an Activated event saying the engine runs r2, the operator's write, as epoch 2", recorded) + } +} + +// TestAnActivationThatDidNotLandIsSentAgainOnItsCondition: the engine still +// runs r1, so the write is sent again on r1, and lands; and where the first +// one lands just ahead of it, the engine refuses it, and the reconcile finds +// the first one running and records it rather than writing a third. +func TestAnActivationThatDidNotLandIsSentAgainOnItsCondition(t *testing.T) { + t.Run("it never lands", func(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.again(t, company) + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) || len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed; want r3 written again on r1, and nothing "+ + "unconfirmed", written, config.Unconfirmed) + } + }) + t.Run("it lands ahead of the second", func(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.beforeWrite = func(e *storeEngine) { e.land("r2") } + w.again(t, company) + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + if written, refused := w.engine.writes(); !slices.Equal(written, []string{"r2"}) || refused != 1 || + len(config.Unconfirmed) != 0 { + t.Errorf("the operator wrote %v, %d refused, and keeps %+v unconfirmed; want the second write refused "+ + "and r2 recorded", written, refused, config.Unconfirmed) + } + if recorded := drained(w.events); !told(recorded, "Activated") { + t.Errorf("events %q, want an Activated event for r2", recorded) + } + }) +} + +// TestAnUnconfirmedActivationIsNeverWrittenOverUnread: the engine runs the +// unconfirmed revision and its fleet cannot be read, so the operator cannot +// tell at what epoch: it writes nothing, keeps the record, and says so. +func TestAnUnconfirmedActivationIsNeverWrittenOverUnread(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.fleetDown = true + w.again(t, company) + if _, err := w.run.converge(context.Background(), w.client); err == nil { + t.Fatal("the reconcile succeeded with no read of the fleet's activation") + } + kept := w.statusWritten().Status.Config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) || len(kept) != 1 { + t.Errorf("the operator wrote %v and keeps %+v, want r2 alone, still unconfirmed", written, kept) + } +} + +// TestAnUnconfirmedActivationOtherwiseOvertakenIsNotSentAgain: somebody +// activated r3 since, so the write, conditioned on r1, can never land: it +// goes rather than being sent again, and not to status.config.overtaken, +// which keeps only the writes into an engine that ran no document; and the +// next write, r4 on r3, whose answer is lost too, is recorded unconfirmed. +func TestAnUnconfirmedActivationOtherwiseOvertakenIsNotSentAgain(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + w.engine.mu.Lock() + w.engine.activateOther(w.requiring) + w.engine.mu.Unlock() + w.again(t, company) + w.engine.activation = pending + reconcileOnce(t, w) + config := w.statusWritten().Status.Config + got := config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r4"}) || len(got) != 1 || + got[0].RevisionID != "r4" || got[0].BaseRevisionID != "r3" { + t.Errorf("the operator wrote %v and keeps %+v unconfirmed, want r2 not sent again and r4, built on r3, kept", + written, got) + } + if over := config.Overtaken; len(over) != 0 { + t.Errorf("status.config.overtaken is %+v, want nothing: r2 was built on r1, and no Seed reads it", over) + } +} + +// TestALandedWriteActivatedAgainStaysInSyncThroughAReconcileThatFails: the +// write landed and was activated again (r3) before the next reconcile read it, +// and that reconcile failed before it decided anything with it: its save of +// the secret sync's state did not go through. Nothing records the write any +// more, r3 having overtaken it, and nothing needs to: the withdrawal its sync +// made and the values it wrote were marked with the fleet's epoch before it +// was sent, and r3 activated above it, so the next reconcile finds r3 in sync, +// records it at its own epoch, and owes nothing for it. Kept owed, the +// withdrawal had the operator activate r3 again, and the values had it write +// the same document again as r4. +func TestALandedWriteActivatedAgainStaysInSyncThroughAReconcileThatFails(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by the engine"); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 0 || len(config.Overtaken) != 0 { + t.Fatalf("status.config after a reconcile that failed is %+v, want r2 gone: r3 overtook it", config) + } + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry: the reconcile that failed saved nothing", state.Withdrawn()) + } + + w.failWrite.Store(0) + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r1 to r3: the operator wrote, or activated, what already ran", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || config.Epoch == nil || + *config.Epoch != 3 { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash, the values it was written with", config) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || len(state.Written()) != 0 { + t.Errorf("the state owes withdrawals %v and writes %v, want nothing: r3 activated after both", state.Withdrawn(), + state.Written()) + } +} + +// TestAHeldDocumentSettlesAnUnconfirmedWriteThatRuns: the first reconcile +// after the write holds its document, and the engine runs the write, which +// landed, or a reload of it. Either read the store without the withdrawal's +// registry, which the sync before the write deleted, so the held reconcile +// settles it and activates nothing again for it. Settled only by a reconcile +// that applies a document, it was owed there, and the held reconcile activated +// the running revision again: an epoch every node applied for nothing. The +// record of the write, and the report, are left to the reconcile that applies +// a document. +func TestAHeldDocumentSettlesAnUnconfirmedWriteThatRuns(t *testing.T) { + for _, reloaded := range []bool{false, true} { + t.Run(map[bool]string{false: "the write runs", true: "a reload of the write runs"}[reloaded], func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + want := []string{"r1", "r2"} + if reloaded { + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + want = append(want, "r3") + } + w.again(t, company) + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + if ids := w.engine.ids(); !slices.Equal(ids, want) { + t.Errorf("the engine's revisions are %v, want %v: what runs read the store without the registry", ids, want) + } + if recorded := drained(w.events); told(recorded, "Reactivated") { + t.Errorf("events %q, want no Reactivated: nothing was owed", recorded) + } + state := w.state(t) + if len(state.Withdrawn()) != 0 || (!reloaded && !state.Records("r2")) { + t.Errorf("the state owes %v and records %q, want nothing owed, and r2's reading where r2 runs", + state.Withdrawn(), state.Recorded()) + } + if w.run.config != nil || w.run.configApplied != nil { + t.Errorf("the held reconcile recorded %+v and reported %+v, want both left to the one that applies a "+ + "document", w.run.config, w.run.configApplied) + } + }) + } +} + +// TestAConfirmationThatLosesItsRaceIsReadAgain: the engine runs the write as +// this reconcile reads it, and by the read of its fleet, which says as what +// epoch, a reload has activated after it. That is a lost race, read again +// from the start, which finds the reload running the write's document with its +// values: in sync, and recorded at its own epoch, with nothing written. A fleet +// answering without its activation is no race, and the reconcile fails over +// it without claiming one, and records nothing. +func TestAConfirmationThatLosesItsRaceIsReadAgain(t *testing.T) { + t.Run("another revision activated", func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { e.activateOther(e.active().document) } + w.engine.mu.Unlock() + w.again(t, company) + if _, err := w.run.converge(context.Background(), w.client); err != nil { + t.Fatalf("converge() = %v, want the reconcile to read the engine again and find it in sync", err) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r1 to r3: nothing written", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.Epoch == nil || *config.Epoch != 3 || + config.AppliedHash != claims[0].AppliedHash { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash", config) + } + }) + t.Run("the fleet answers without its activation", func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.mu.Lock() + w.engine.noActivation = true + w.engine.mu.Unlock() + w.again(t, company) + _, err := w.run.converge(context.Background(), w.client) + if err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || strings.Contains(err.Error(), "instead") { + t.Errorf("converge() = %v, want an error that claims no race", err) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone", written) + } + if got := w.statusWritten().Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r2" { + t.Errorf("status.config.unconfirmed is %+v, want r2 kept", got) + } + }) +} + +// TestAConfirmedWriteIsReportedAppliedOnceTheEngineAppliesIt: the write's own +// reconcile reported it as one the engine could not confirm, and the next +// finds it running and its epoch applied: an Applied event names it, as one +// does for a write the engine confirmed, although the stored ConfigApplied is +// neither Applying nor ApplyFailed. +func TestAConfirmedWriteIsReportedAppliedOnceTheEngineAppliesIt(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + meta.SetStatusCondition(&company.Status.Conditions, *w.run.configApplied) + w.again(t, company) + reconcileOnce(t, w) + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + var applied []string + for _, e := range drained(w.events) { + if strings.HasPrefix(e, "Normal "+crewletaiv1alpha1.ReasonApplied+" ") { + applied = append(applied, e) + } + } + if len(applied) != 1 || !strings.Contains(applied[0], "revision r2 (epoch 2)") { + t.Errorf("Applied events %q, want one naming revision r2 at epoch 2", applied) + } +} + +// TestAValueWrittenBackIsSettledByTheWriteThatCarriedIt: the store lost the +// operator's value, so the sync wrote it back and the document was written +// again, as r3, whose activation the engine could not confirm. The engine runs +// r3, or a reload of it, either of which read the store with the value, so the +// next reconcile owes nothing for it and writes nothing. Kept owed, the value +// had the operator write the document once more. +func TestAValueWrittenBackIsSettledByTheWriteThatCarriedIt(t *testing.T) { + for _, reloaded := range []bool{false, true} { + t.Run(map[bool]string{false: "the write runs", true: "a reload of the write runs"}[reloaded], func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(w.state(t).Written(), []string{tavily}) { + t.Fatalf("the operator wrote %v and the state owes %v; want r3 written for the value written back, "+ + "and the value owed", written, w.state(t).Written()) + } + want := []string{"r1", "r2", "r3"} + if reloaded { + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + want = append(want, "r4") + } + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, want) { + t.Errorf("the engine's revisions are %v, want %v: what runs read the value written back", ids, want) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing", got) + } + }) + } +} + +// TestASeedTheEngineCouldNotConfirmIsReportedAsTheSeed: under Seed, a write +// into an engine with no document that landed unconfirmed is the seed, not a +// document the engine had before the operator seeded one. +func TestASeedTheEngineCouldNotConfirmIsReportedAsTheSeed(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = crewletaiv1alpha1.ManagementPolicySeed + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the seed", written) + } + seed := written[0] + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != seed || got[0].BaseRevisionID != "" { + t.Errorf("status.config.unconfirmed is %+v, want the seed %s, built on nothing", got, seed) + } + w.engine.activation = confirmed + w.again(t, company) + reconcileOnce(t, w) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonSeeded || + !strings.Contains(c.Message, "seeded with the rendered document as revision "+seed) { + t.Errorf("ConfigApplied is %+v, want Seeded, naming %s as the seed", c, seed) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{seed}) { + t.Errorf("the operator wrote %v, want the seed alone", written) + } +} + +// TestALandedWriteActivatedAgainIsNotWrittenAgain: the write landed, and was +// activated again before the next reconcile read it, by the engine's own +// reload after it minted a credential or by a person's crewlet config reload, +// so the engine runs the rendered document as r3, with the values r2 was +// written with. Dropped as a write that can no longer land, it left +// status.config's hash the one before it, and the operator wrote the same +// document a third time. It is in sync instead: r3 is recorded at its own +// epoch with r2's hash, and the withdrawal r2's sync made, which r3 activated +// after, is settled, so nothing activates it again either. +func TestALandedWriteActivatedAgainIsNotWrittenAgain(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + claims := company.Status.Config.Unconfirmed + if _, err := w.client.ReloadConfig(context.Background(), "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.again(t, company) + reconcileOnce(t, w) + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Fatalf("the engine's revisions are %v, want r1 to r3: the operator wrote, or activated, what already ran", ids) + } + config := w.statusWritten().Status.Config + if len(claims) != 1 || config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || + config.Epoch == nil || *config.Epoch != 3 || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 at epoch 3 with r2's hash, and nothing unconfirmed", config) + } + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue { + t.Errorf("ConfigApplied is %+v, want True: the engine runs the rendered document", c) + } + if state := w.state(t); len(state.Withdrawn()) != 0 { + t.Errorf("the state owes %v, want nothing: r3 activated after the registry was deleted", state.Withdrawn()) + } +} + +// TestASeedActivatedAgainIsReportedAsTheSeed: under Seed, the write into an +// engine with no document landed unconfirmed and was activated again before +// the next reconcile read it, so the engine runs another revision. It is +// still the seed, and the engine runs the revision that replaced it, not a +// document it had before the operator seeded one. +func TestASeedActivatedAgainIsReportedAsTheSeed(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = crewletaiv1alpha1.ManagementPolicySeed + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the seed", written) + } + seed := written[0] + reloaded, err := w.client.ReloadConfig(context.Background(), "reloaded by the engine") + if err != nil { + t.Fatal(err) + } + company = w.statusWritten() + w.again(t, company) + reconcileOnce(t, w) + want := fmt.Sprintf("seeded with the rendered document as revision %s and owns the document since; it runs revision %s", + seed, reloaded.RevisionID) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonSeeded || !strings.Contains(c.Message, want) { + t.Errorf("ConfigApplied is %+v, want Seeded, saying %q", c, want) + } + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != seed || len(config.Unconfirmed) != 0 || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want the seed %s recorded, and nothing unconfirmed or overtaken", config, seed) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{seed}) { + t.Errorf("the operator wrote %v, want the seed alone", written) + } + + // The engine owns the document since the seed: a write still recorded + // as overtaken, as a status written before the seed was recorded may + // hold, says nothing more, and goes. + company.Status.Config.Overtaken = []crewletaiv1alpha1.UnconfirmedRevision{{ + RevisionID: "r9", AppliedHash: "sha256:other", WrittenAt: &metav1.Time{}, + }} + w.again(t, company) + reconcileOnce(t, w) + if config := w.statusWritten().Status.Config; config.RevisionID != seed || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want the seed %s recorded, and nothing overtaken", config, seed) + } +} + +// TestAWriteAReactivationOvertookIsNotSentAgain: a held reconcile activates +// r1 again (r3), for the withdrawal of the registry, while a write built on it +// (r2) is unconfirmed. r2 can no longer land, and r3 activated after the +// changes r2's sync marked, so it read the values r2 was written with: the +// reload is recorded as the operator's, r2 is left for the next read to drop, +// and the next reconcile finds r3 in sync with the document and writes +// nothing. +func TestAWriteAReactivationOvertookIsNotSentAgain(t *testing.T) { + w, company := unconfirmedWrite(t, pending) + claims := company.Status.Config.Unconfirmed + w.again(t, company) + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r3"}) { + t.Fatalf("the engine's revisions are %v, want r1 activated again as r3", ids) + } + if config := company.Status.Config; len(claims) != 1 || config.RevisionID != "r3" || len(config.Unconfirmed) != 1 { + t.Fatalf("status.config is %+v, want r3 recorded and r2 left for the next read", config) + } + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: r3 read the values r2 was written with", written) + } + config := w.statusWritten().Status.Config + if config.RevisionID != "r3" || config.AppliedHash != claims[0].AppliedHash || len(config.Unconfirmed) != 0 || + len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want r3 with r2's hash, and r2 gone", config) + } +} + +// TestAConfirmationTheStateMissedIsSettledLater: the reconcile that found the +// write running recorded it in status.config and could not save the secret +// sync's state, so the next one reads what the state records of the write as +// the operator's, and settles the withdrawal its activation read the store +// without, rather than activating r2 again for it: whether that reconcile +// applies a document or holds it. +func TestAConfirmationTheStateMissedIsSettledLater(t *testing.T) { + for _, held := range []bool{false, true} { + t.Run(map[bool]string{false: "the next document is applied", true: "the next document is held"}[held], func(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + company = w.statusWritten() + if company.Status.Config.RevisionID != "r2" || len(company.Status.Config.Unconfirmed) != 0 { + t.Fatalf("status.config is %+v, want r2 recorded as the operator's", company.Status.Config) + } + if state := w.state(t); state.Records("r2") { + t.Fatal("the state records r2's reading, which the failed write was to save") + } + w.failWrite.Store(0) + w.again(t, company) + if held { + if _, err := w.run.withdrawHeld(context.Background(), w.resolve(t)); err != nil { + t.Fatal(err) + } + } else { + reconcileOnce(t, w) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 activated without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !state.Records("r2") { + t.Errorf("the state owes %v and records %q, want nothing owed and r2's reading", state.Withdrawn(), state.Recorded()) + } + }) + } +} + +// TestAConfirmedWriteLeavesTheSeatsToTheRevisionThatRuns: the write a +// reconcile finds running is one an earlier reconcile made, of what it +// rendered then, so the human seats IdentityClaimed names are those of the +// revision the engine runs, here one with a seat ana, and not those of this +// reconcile's render, which the engine refuses. Taken for this reconcile's own +// write, a confirmation named seats the engine does not have. +func TestAConfirmedWriteLeavesTheSeatsToTheRevisionThatRuns(t *testing.T) { + w, company := unconfirmedWrite(t, landed) + w.engine.mu.Lock() + w.engine.revisions[len(w.engine.revisions)-1].document = []byte("name: Acme\nroles:\n - name: Ana\n handle: ana\n kind: human\n") + w.engine.refusal = invalidDocument + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + if w.run.confirmed != "r2" { + t.Fatalf("the reconcile confirmed %q, want r2", w.run.confirmed) + } + if got := w.run.humanSeats(context.Background()); !slices.Equal(got, []string{seatAna}) { + t.Errorf("humanSeats() = %v, want [%s]: the engine runs r2", got, seatAna) + } +} + +// TestUnconfirmedActivationsAreBounded: a write whose answer is lost every +// time is recorded no more than maxUnconfirmed times, the newest kept, since +// the API server refuses a status that holds more: the generated CRD's +// maxItems, which this holds the constant to. +func TestUnconfirmedActivationsAreBounded(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "config", "crd", "bases", "crewlet.ai_companies.yaml")) + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatal(err) + } + for _, field := range []string{"unconfirmed", "overtaken"} { + var bound *int64 + for _, v := range crd.Spec.Versions { + if v.Name == "v1alpha1" { + config := v.Schema.OpenAPIV3Schema.Properties["status"].Properties["config"] + bound = config.Properties[field].MaxItems + } + } + if bound == nil { + t.Fatalf("the Company CRD does not bound status.config.%s; the operator does, at %d", field, maxUnconfirmed) + } + if *bound != maxUnconfirmed { + t.Fatalf("the Company CRD bounds status.config.%s at %d items, the operator at %d", field, *bound, maxUnconfirmed) + } + } + w, company := unconfirmedWrite(t, pending) + for range maxUnconfirmed { + w.again(t, company) + w.engine.activation = pending + reconcileOnce(t, w) + company = w.statusWritten() + } + written, _ := w.engine.writes() + got := make([]string, 0, len(company.Status.Config.Unconfirmed)) + for _, write := range company.Status.Config.Unconfirmed { + got = append(got, write.RevisionID) + } + if len(written) != maxUnconfirmed+1 || !slices.Equal(got, written[1:]) { + t.Errorf("after %d writes %v, status.config.unconfirmed holds %v; want the newest %d", len(written), written, got, + maxUnconfirmed) + } + + // Writes into an engine that ran no document move to overtaken once the + // engine runs a revision none of them is, beside the ones already there, + // and the oldest go: sixteen older ones here, which a list cut at its + // other end would keep instead. The reconcile fails before it writes, as + // the record is kept through one that does. + company = w.run.company.DeepCopy() + company.Status.Config.Unconfirmed, company.Status.Config.Overtaken = nil, nil + want := make([]string, 0, maxUnconfirmed) + for i := range maxUnconfirmed { + company.Status.Config.Overtaken = append(company.Status.Config.Overtaken, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: fmt.Sprintf("old%d", i), AppliedHash: "sha256:old", WrittenAt: &metav1.Time{}, + }) + seed := fmt.Sprintf("seed%d", i) + company.Status.Config.Unconfirmed = append(company.Status.Config.Unconfirmed, crewletaiv1alpha1.UnconfirmedRevision{ + RevisionID: seed, AppliedHash: "sha256:seed", WrittenAt: &metav1.Time{}, + }) + want = append(want, seed) + } + w.again(t, company) + w.stateWrites.Store(0) + w.failWrite.Store(1) + reconcileOnce(t, w) + w.failWrite.Store(0) + got = got[:0] + for _, write := range w.statusWritten().Status.Config.Overtaken { + got = append(got, write.RevisionID) + } + if config := w.statusWritten().Status.Config; !slices.Equal(got, want) || len(config.Unconfirmed) != 0 { + t.Errorf("status.config.overtaken holds %v and unconfirmed %d writes; want the newest %d, %v, and none", + got, len(config.Unconfirmed), maxUnconfirmed, want) + } +} + +// TestADriftCorrectionTheEngineCouldNotConfirmIsReportedOnceItRuns holds a +// write over a revision somebody else activated with other content, whose +// activation the engine could not confirm, to the report a confirmed one gets +// once a read shows the engine runs it: ConfigApplied DriftCorrected naming +// the revision it replaced and where it differed, and a Warning Drifted +// event, kept through the rest of that reconcile. Confirmed from the record +// as it was, it read Applied, then "The engine runs the rendered document", +// and nothing ever said a revision somebody else wrote had gone. +func TestADriftCorrectionTheEngineCouldNotConfirmIsReportedOnceItRuns(t *testing.T) { + w := newWithdrawal(t, runningOther) + operatorActivates(w) + // The operator's last write was r0: r1 is somebody else's. + company := w.run.company.DeepCopy() + company.Status.Config.RevisionID = "r0" + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company = w.statusWritten() + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].Replaced != crewletaiv1alpha1.ReplacedDrifted || + got[0].DriftedAt == "" { + t.Fatalf("status.config.unconfirmed is %+v, want r2, recorded as replacing a Drifted r1, with where", got) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonDrifted) { + t.Errorf("events %q, want no Drifted before the write is seen running: it may never land", recorded) + } + + w.again(t, company) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone", written) + } + want := fmt.Sprintf("Revision r1, which the operator did not write, differed from the rendered document at %s; "+ + "revision r2 replaced it.", got[0].DriftedAt) + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue || + c.Reason != crewletaiv1alpha1.ReasonDriftCorrected || c.Message != want { + t.Errorf("ConfigApplied is %+v, want True DriftCorrected, %q", c, want) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonDrifted+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "ran revision r1, which the operator did not write") || + !strings.Contains(recorded[i], "revision r2 replaced it") { + t.Errorf("events %q, want a Warning Drifted saying r2 replaced r1", recorded) + } +} + +// TestAHeldReconcileReleasesWhatARunningUnconfirmedWriteLetsGo: r1 requires +// the registry, and the operator's write r2, which lets it go, landed with its +// answer lost. Then the registry's Secret is deleted while the document is +// held by a render that no longer reads it at all, so the held reconcile reads +// nothing missing and owes nothing for r1. It reads the engine all the same +// while a write of the operator's may run: r2 runs, and the reading the state +// recorded before r2 was sent says it lets the registry go, so the value is +// taken out of the store and r2 activated again (r3) for it. Returning before +// the read, it left the deleted Secret's value for r2's tool server to start +// with until the document was applied. The record of r2 moves on to r3, which +// the next reconcile confirms as the operator's, rather than taking r3 for a +// revision somebody else activated and reporting the write over it as drift. +func TestAHeldReconcileReleasesWhatARunningUnconfirmedWriteLetsGo(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + claims := company.Status.Config.Unconfirmed + if len(claims) != 1 || claims[0].RevisionID != "r2" || !w.engine.holds(w.registry) || len(w.state(t).Retained()) != 0 { + t.Fatalf("status.config.unconfirmed is %+v, the store holds the registry: %t, and the state retains %v; want "+ + "r2 recorded, the registry pushed, and nothing retained", claims, w.engine.holds(w.registry), + w.state(t).Retained()) + } + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + if w.engine.holds(w.registry) { + t.Error("the engine's store still holds the registry, whose Secret is gone and which r2, which runs, lets go") + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) { + t.Errorf("the engine's revisions are %v, want r2 activated again as r3 without the registry", ids) + } + state := w.state(t) + if !state.Records("r3") || !slices.Equal(state.LetGo("r3"), []string{w.registry}) || len(state.Withdrawn()) != 0 { + t.Errorf("the state records %q letting %v go and owes %v; want r3 letting the registry go and nothing owed", + state.Recorded(), state.LetGo(state.Recorded()), state.Withdrawn()) + } + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != "r1" || len(config.Unconfirmed) != 1 || + config.Unconfirmed[0].RevisionID != "r3" || config.Unconfirmed[0].AppliedHash != claims[0].AppliedHash { + t.Fatalf("status.config is %+v, want r1 recorded and r2's record moved on to r3", config) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if warned(recorded, crewletaiv1alpha1.ReasonDrifted) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") + }) { + t.Errorf("events %q, want r3 confirmed as the operator's, at epoch 3, and no drift reported", recorded) + } +} + +// TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns: a held +// reconcile takes the registry, which r1, the operator's revision, lets go, +// out of the store and activates r1 again; the engine stores the reload as r2 +// and loses the answer to its activation. Where r2 runs by the read back, it +// activated after the withdrawal, which is settled, and it is recorded as the +// reactivation. Where it has not landed yet it may still: the withdrawal stays +// owed and reported, and r2 is recorded as a write of r1's document, so that +// once it lands the next reconcile records it as the operator's, at its epoch, +// and owes nothing for the withdrawal, rather than taking it for a revision +// somebody else activated. +func TestAReloadTheEngineCouldNotConfirmIsTheOperatorsOnceItRuns(t *testing.T) { + for _, how := range []outcome{landed, pending} { + t.Run(map[outcome]string{landed: "it landed", pending: "it lands later"}[how], func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = how + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.reloadActivation = confirmed + recorded := drained(w.events) + state := w.state(t) + company = w.statusWritten() + if how == landed { + if config := company.Status.Config; len(state.Withdrawn()) != 0 || config.RevisionID != "r2" || + len(config.Unconfirmed) != 0 || !told(recorded, "Reactivated") || warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes %v, status.config is %+v, events %q; want the withdrawal settled, r2 recorded "+ + "as the reactivation, and no failure reported", state.Withdrawn(), config, recorded) + } + return + } + config := company.Status.Config + if !slices.Equal(state.Withdrawn(), []string{w.registry}) || !warned(recorded, "SecretWithdrawalFailed") || + config.RevisionID != "r1" || len(config.Unconfirmed) != 1 || config.Unconfirmed[0].RevisionID != "r2" || + config.Unconfirmed[0].BaseRevisionID != "r1" || config.Unconfirmed[0].AppliedHash != r1Hash { + t.Fatalf("the state owes %v, status.config is %+v, events %q; want the registry owed, the failure reported, "+ + "and r2 recorded unconfirmed on r1 with r1's hash", state.Withdrawn(), config, recorded) + } + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.again(t, company) + reconcileOnce(t, w) + recorded = drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) { + t.Errorf("events %q, want r2 confirmed as the operator's, at epoch 2", recorded) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 read the store without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || !slices.Equal(state.LetGo("r2"), []string{w.registry}) { + t.Errorf("the state owes %v and lets %v go for r2; want nothing owed and the registry let go", + state.Withdrawn(), state.LetGo("r2")) + } + }) + } +} + +// TestAReloadBuiltOnAnUnconfirmedWriteStaysUnconfirmedOnceTheWriteIs: a held +// reconcile finds r2, a write of the operator's whose activation went +// unconfirmed, running, and activates it again for the registry it withdrew; +// the engine stores that reload as r3, conditioned on r2, and loses the +// answer, and r2 still runs by the read back, so r3 is recorded as a write +// built on r2 (reloadUnconfirmed). The next reconcile confirms r2 and keeps +// r3, which may land yet, and its own reload of r2 (r4) is lost the same way; +// once r3 lands it is confirmed in turn, as the operator's. Taken for a write +// r2 overtook, r3 was dropped with r2's confirmation, and its landing read as +// a revision somebody else activated. +func TestAReloadBuiltOnAnUnconfirmedWriteStaysUnconfirmedOnceTheWriteIs(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 2 || config.Unconfirmed[1].RevisionID != "r3" || + config.Unconfirmed[1].BaseRevisionID != "r2" { + t.Fatalf("status.config is %+v, want r2 unconfirmed and the reload r3 recorded on it", config) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + company = w.statusWritten() + if config := company.Status.Config; config.RevisionID != "r2" || !slices.ContainsFunc(config.Unconfirmed, + func(u crewletaiv1alpha1.UnconfirmedRevision) bool { + return u.RevisionID == "r3" && u.BaseRevisionID == "r2" + }) { + t.Fatalf("status.config is %+v, want r2 recorded as the operator's and r3, built on it, still unconfirmed", config) + } + w.engine.reloadActivation = confirmed + w.engine.mu.Lock() + w.engine.land("r3") + w.engine.mu.Unlock() + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 recorded as the operator's, and nothing unconfirmed", config) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonDrifted) || + !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3 as epoch 3") + }) { + t.Errorf("events %q, want r3 confirmed as the operator's, at epoch 3, and no drift reported", recorded) + } +} + +// TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound: the write's +// answer was lost three minutes before the reconcile that finds it running, +// and the engine is one epoch behind. Nothing says when the activation landed, +// so the engine is held to applying it from when the operator found it, as an +// adopted revision is: Applying, not an apply that failed. Dated by the write, +// the confirmation read as ApplyFailed, with a Warning, in the reconcile that +// found it. +func TestAWriteConfirmedLateIsHeldToTheDeadlineFromWhenItWasFound(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + written := metav1.NewTime(time.Date(2026, 10, 25, 8, 0, 0, 0, time.UTC)) + company.Status.Config.Unconfirmed[0].WrittenAt = &written + found := written.Add(3 * time.Minute) + w.again(t, company) + w.run.now = func() time.Time { return found } + w.engine.mu.Lock() + w.engine.appliedLag = 1 + w.engine.mu.Unlock() + reconcileOnce(t, w) + if config := w.run.config; config == nil || config.RevisionID != "r2" || config.AppliedAt == nil || + !config.AppliedAt.Time.Equal(found) { + t.Fatalf("status.config is %+v, want r2 confirmed, applied when the operator found it, %s", w.run.config, found) + } + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonApplying { + t.Errorf("ConfigApplied is %+v, want Applying: the operator found r2 a moment ago", c) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonApplyFailed) { + t.Errorf("events %q, want no ApplyFailed", recorded) + } +} + +// TestAWriteRecordsItsDocumentsReadingBeforeItIsSent: the state owes nothing, +// so no change of the store is marked before the write, and the reading of the +// document the write carries is saved before it is sent all the same, under +// the write's hash: its answer lost, nothing else says how the revision the +// engine stored reads the store, once the render has moved on. +func TestAWriteRecordsItsDocumentsReadingBeforeItIsSent(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + reconcileOnce(t, w) + company := w.statusWritten() + if state := w.state(t); state.Unobserved() || !state.Records("r2") || company.Status.Config.RevisionID != "r2" { + t.Fatalf("after the first write the state records %q, with changes unread: %t; want r2, and none", + state.Recorded(), state.Unobserved()) + } + w.engine.mu.Lock() + w.engine.activateOther(w.requiring) + w.engine.mu.Unlock() + w.again(t, company) + w.engine.activation = landed + reconcileOnce(t, w) + got := w.statusWritten().Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r4" { + t.Fatalf("status.config.unconfirmed is %+v, want r4, written over the r3 somebody activated", got) + } + if !w.state(t).Pends(got[0].AppliedHash) { + t.Errorf("the state records no reading under r4's hash, %s: saved after the write, its answer lost left none", + got[0].AppliedHash) + } +} + +// TestAWriteWhoseAnswerIsLostWhileTheFleetIsUnreadableSettlesWhatItRead: the +// fleet's activation pointer cannot be read after the changes the write is +// sent after, so nothing marks the rotated value and the withdrawn registry, +// and the engine loses the write's answer. The changes await the revision it +// stored instead, and the next reconcile, finding that revision running, +// settles them: no second write of the same document, and no reload of it. +// Left unread, no epoch the pointer named later could settle them, and the +// operator wrote the document again over its own landed write. The fleet +// answers the read that says what every pod runs, before the changes: one it +// does not answer keeps every value, so the registry would stay. +func TestAWriteWhoseAnswerIsLostWhileTheFleetIsUnreadableSettlesWhatItRead(t *testing.T) { + for _, down := range []bool{true, false} { + t.Run(map[bool]string{true: "the fleet answers 503", false: "the fleet answers without its activation"}[down], func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.rotate(t) + if down { + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + } + w.engine.noActivation = !down + w.engine.activation = landed + reconcileOnce(t, w) + if state := w.state(t); !state.Unobserved() { + t.Fatalf("the state owes %v and %v, all marked; want them unread, as the fleet could not be read", + state.Withdrawn(), state.Written()) + } + w.engine.fleetDown, w.engine.noActivation, w.engine.activation = false, false, confirmed + w.again(t, w.statusWritten()) + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: r2 ran, and read every change", written) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: nothing activated r2 again", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 || len(state.Written()) != 0 { + t.Errorf("the state owes %v and %v, want nothing", state.Withdrawn(), state.Written()) + } + }) + } +} + +// TestAFirstWriteWhoseAnswerIsLostIsNotWrittenAgain: under Enforce the engine +// runs no document, so the fleet names no activation to mark the values the +// sync pushed by, and the engine loses the answer to the first write. Once a +// read finds the revision it stored running, that revision read every value, +// and the operator writes nothing more. Marked by nothing, the values stayed +// owed, and the operator wrote the same document a second time. +func TestAFirstWriteWhoseAnswerIsLostIsNotWrittenAgain(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + w.engine.mu.Lock() + w.engine.revisions = nil + w.engine.mu.Unlock() + company := w.run.company.DeepCopy() + company.Status.Config = crewletaiv1alpha1.ConfigStatus{} + w.again(t, company) + w.rotate(t) + w.engine.activation = landed + reconcileOnce(t, w) + written, _ := w.engine.writes() + if len(written) != 1 { + t.Fatalf("the operator wrote %v, want the first write", written) + } + w.engine.activation = confirmed + w.again(t, w.statusWritten()) + reconcileOnce(t, w) + if again, _ := w.engine.writes(); !slices.Equal(again, written) { + t.Errorf("the operator wrote %v, want %v alone", again, written) + } + if state := w.state(t); len(state.Written()) != 0 { + t.Errorf("the state owes %v, want nothing: %s read every value", state.Written(), written[0]) + } +} + +// TestAReloadWhoseAnswerIsLostIsTheOperatorsWhenItsReadBackFails: a held +// reconcile activates r1, the operator's, again for the registry it withdrew, +// the engine stores the reload as r2 and loses the answer, and the read back +// fails: GET /config, or the fleet with r2 running. r2 is recorded as the +// operator's from the answer, so once it runs the next reconcile confirms it +// as the operator's, at its epoch, and settles the withdrawal, which awaits +// r2 where no read of the fleet marked it before the reload. Recorded only +// where the read back found r1 still running, it was recorded nowhere, and r2 +// was taken for a revision somebody else activated; and a withdrawal no read +// marked had r2 activated again for it. The fleet answers the read that says +// what every pod runs, before the withdrawal: one it does not answer keeps +// every value, and nothing is withdrawn to reload for. +func TestAReloadWhoseAnswerIsLostIsTheOperatorsWhenItsReadBackFails(t *testing.T) { + for _, c := range []struct { + name string + how outcome + fail func(e *storeEngine) + // fleetLost makes the fleet unreadable after the read that says + // what every pod runs, so no read marks the withdrawal before the + // reload is asked for. + fleetLost bool + }{ + {name: "GET /config fails, the reload landed", how: landed, fail: func(e *storeEngine) { e.configDown = true }}, + {name: "GET /config fails, the reload lands later", how: pending, fail: func(e *storeEngine) { e.configDown = true }}, + {name: "the fleet cannot be read, the reload landed", how: landed, fail: func(e *storeEngine) { e.fleetDown = true }}, + { + name: "the fleet cannot be read across the reload, which landed", how: landed, + fail: func(e *storeEngine) { e.configDown = true }, fleetLost: true, + }, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = c.how + w.engine.beforeReload = c.fail + if c.fleetLost { + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.configDown, w.engine.fleetDown, w.engine.reloadActivation = false, false, confirmed + if c.how == pending { + w.engine.land("r2") + } + r1 := w.engine.revisions[0].document + w.engine.mu.Unlock() + company = w.statusWritten() + got := company.Status.Config.Unconfirmed + if len(got) != 1 || got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || + got[0].DocumentSHA256 != documentDigest(r1) { + t.Fatalf("status.config.unconfirmed is %+v, want r2, a reload of r1, with r1's document's digest", got) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r2 as epoch 2") + }) || warned(recorded, crewletaiv1alpha1.ReasonDrifted) { + t.Errorf("events %q, want r2 confirmed as the operator's, at epoch 2, and no drift reported", recorded) + } + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2"}) { + t.Errorf("the engine's revisions are %v, want r1 and r2: r2 read the store without the registry", ids) + } + if state := w.state(t); len(state.Withdrawn()) != 0 { + t.Errorf("the state owes %v, want the withdrawal settled by r2", state.Withdrawn()) + } + }) + } +} + +// TestAReloadOfAnotherRevisionIsNotTheOperators: the operator asks to activate +// r1 again, and somebody activates r2, another document, before the request +// lands, so the engine stores r3 as a reload of r2, loses the answer, and the +// read back fails. r3 is recorded as a reload of r1, with r1's document's +// digest; found running r2's document, it is not the operator's: nothing +// confirms it as the operator's write, and the next write over it reports it +// as the drift it is. +func TestAReloadOfAnotherRevisionIsNotTheOperators(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.AppliedHash = r1Hash + w.again(t, company) + w.engine.reloadActivation = landed + w.engine.beforeReload = func(e *storeEngine) { + e.activateOther(w.requiring) + e.configDown = true + } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.configDown, w.engine.reloadActivation = false, confirmed + w.engine.mu.Unlock() + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r3" { + t.Fatalf("status.config.unconfirmed is %+v, want r3 recorded, as nothing said what it reloaded", got) + } + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + recorded := drained(w.events) + if slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Activated ") && strings.Contains(e, "revision r3") + }) { + t.Errorf("events %q, want r3 not confirmed as the operator's: it runs r2's document", recorded) + } + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonDriftCorrected || + !strings.HasPrefix(c.Message, "Revision r3, which the operator did not write") { + t.Errorf("ConfigApplied is %+v, want DriftCorrected over r3", c) + } + if config := w.statusWritten().Status.Config; len(config.Unconfirmed) != 0 || config.RevisionID != "r4" { + t.Errorf("status.config is %+v, want r4 recorded and nothing unconfirmed", config) + } +} + +// TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce: a held reconcile +// finds r2, a write of the operator's whose activation went unconfirmed, +// running, and its reload of r2 is stored as r3 and left unconfirmed too. In +// the next held reconcile r3 lands while that reconcile's own reload of r2 +// fails, so the read back finds r3 running r2's document and records it as +// the reactivation. r3 is recorded once: as two entries of one revision, the +// API server refused the status, and every condition the reconcile decided +// was lost. +func TestAReloadThatLandsWhileTheNextOneFailsIsRecordedOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningRequires) + operatorActivates(w) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company := w.statusWritten() + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + company = w.statusWritten() + if config := company.Status.Config; len(config.Unconfirmed) != 2 || config.Unconfirmed[1].RevisionID != "r3" { + t.Fatalf("status.config is %+v, want r2 unconfirmed and the reload r3 recorded on it", config) + } + w.again(t, company) + w.engine.reloadActivation = confirmed + w.engine.reloadDown = true + w.engine.beforeReload = func(e *storeEngine) { e.land("r3") } + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + config := w.statusWritten().Status.Config + ids := make([]string, len(config.Unconfirmed)) + for i := range config.Unconfirmed { + ids[i] = config.Unconfirmed[i].RevisionID + } + if slices.Contains(ids, "r2") || !slices.Equal(slices.Compact(slices.Sorted(slices.Values(ids))), slices.Sorted(slices.Values(ids))) || + !slices.Contains(ids, "r3") { + t.Errorf("status.config.unconfirmed names %v, want r3 once, as the reactivation of r2's record", ids) + } +} + +// TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced: the +// operator's write r2 replaced r1, somebody else's document or one the engine +// stores without a seal, and its answer was lost; a held reconcile then found +// r2 running and its reload of r2 was stored as r3, its answer lost too. Once +// r3 runs, its confirmation reports that the rendered document replaced r1, +// once, and never names r2, the operator's own write, as somebody else's or +// as stored without a seal: whether r2 was confirmed first or not at all. +func TestAReloadOfAnUnconfirmedReplacementReportsWhatTheWriteReplaced(t *testing.T) { + for _, c := range []struct { + name string + unsealed bool + // confirmFirst confirms r2 in a reconcile of its own before r3 lands. + confirmFirst bool + }{ + {name: "r1 somebody else's, r3 lands first"}, + {name: "r1 somebody else's, r2 confirmed first", confirmFirst: true}, + {name: "r1 stored without a seal, r3 lands first", unsealed: true}, + {name: "r1 stored without a seal, r2 confirmed first", unsealed: true, confirmFirst: true}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningOther) + operatorActivates(w) + company := w.run.company.DeepCopy() + company.Status.Config.RevisionID = "r0" + w.again(t, company) + if c.unsealed { + w.engine.cannotOpen(sealMissing) + } + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.engine.activation = landed + reconcileOnce(t, w) + w.engine.activation = confirmed + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 1 || got[0].RevisionID != "r2" || got[0].Replaced == "" { + t.Fatalf("status.config.unconfirmed is %+v, want r2 recorded as replacing r1", got) + } + if err := w.cluster.Delete(ctx, valueSecret(optionalSecret, nil)); err != nil { + t.Fatal(err) + } + w.again(t, company) + w.engine.reloadActivation = pending + if _, err := w.run.withdrawHeld(ctx, secretsync.Desired{}); err != nil { + t.Fatal(err) + } + w.engine.reloadActivation = confirmed + company = w.statusWritten() + if got := company.Status.Config.Unconfirmed; len(got) != 2 || got[1].RevisionID != "r3" || got[1].Replaced != "" || + got[1].DriftedAt != "" { + t.Fatalf("status.config.unconfirmed is %+v, want r3, built on r2, replacing nothing of its own", got) + } + var reports []string + if c.confirmFirst { + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + reports = append(reports, drained(w.events)...) + company = w.statusWritten() + } + w.engine.mu.Lock() + w.engine.land("r3") + w.engine.mu.Unlock() + drained(w.events) + w.again(t, company) + reconcileOnce(t, w) + reports = append(reports, drained(w.events)...) + reason := crewletaiv1alpha1.ReasonDrifted + if c.unsealed { + reason = crewletaiv1alpha1.ReasonRevisionUnreadable + } + var warnings []string + for _, e := range reports { + if strings.HasPrefix(e, "Warning "+reason+" ") { + warnings = append(warnings, e) + } + } + // What the event says of the revision replaced: "revision r1, + // which the operator did not write", or "active revision r1 was + // one it stores without a seal". + names := func(revision string) bool { + return strings.Contains(warnings[0], "revision "+revision+", which") || + strings.Contains(warnings[0], "revision "+revision+" was one") + } + if len(warnings) != 1 || !names("r1") || names("r2") { + t.Errorf("Warning %s events %q, want one, naming r1 and never r2", reason, warnings) + } + if config := w.statusWritten().Status.Config; config.RevisionID != "r3" || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r3 recorded as the operator's and nothing unconfirmed", config) + } + }) + } +} + +// TestAConfirmedWriteLeavesNothingOvertaken: a create the engine could not +// confirm was overtaken, and a later write of the operator's is confirmed +// running, by a reconcile that then ends before anything is in sync: a value +// rotated since needs a write (the engine's own re-activation at it failing), +// and the engine cannot decide the dry run. +// Nothing overtaken can land now, nor be the seed, so the record of it goes +// with the confirmation, as the API's own doc says it does once +// status.config records a write; kept, it stood beside that write until a +// later reconcile found the document in sync. +func TestAConfirmedWriteLeavesNothingOvertaken(t *testing.T) { + ctx := context.Background() + w, company := unconfirmedWrite(t, landed) + now := metav1.NewTime(time.Unix(1, 0)) + company.Status.Config.Overtaken = []crewletaiv1alpha1.UnconfirmedRevision{ + {RevisionID: "r0", AppliedHash: "sha256:r0", WrittenAt: &now}, + } + w.again(t, company) + rotated := &corev1.Secret{} + if err := w.cluster.Get(ctx, types.NamespacedName{Namespace: "ns", Name: requiredSecret}, rotated); err != nil { + t.Fatal(err) + } + rotated.Data[requiredKey] = []byte(rotatedValue + "-again") + if err := w.cluster.Update(ctx, rotated); err != nil { + t.Fatal(err) + } + operatorActivates(w) + w.engine.dryRunDown = true + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Fatalf("the operator wrote %v, want r2 alone: the dry run after the confirmation was not decided", written) + } + if config := w.statusWritten().Status.Config; config.RevisionID != "r2" || len(config.Overtaken) != 0 { + t.Errorf("status.config is %+v, want r2 recorded and nothing overtaken", config) + } +} diff --git a/internal/controller/config_unit_test.go b/internal/controller/config_unit_test.go index 919c26cd..5fe7e450 100644 --- a/internal/controller/config_unit_test.go +++ b/internal/controller/config_unit_test.go @@ -7,12 +7,19 @@ import ( "regexp" "slices" "strings" + "sync" "testing" "time" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/meta/testrestmapper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/yaml" @@ -24,8 +31,28 @@ import ( ) // decodeSamples decodes every object in config/samples, the Nimbus company, -// its parts and its class, the namespaced ones into the namespace given. +// its parts and its class, the namespaced ones into the namespace given. Each +// call has objects of its own, copied from one parse per test binary: the +// samples are read by dozens of tests, and parsing them again for each was +// most of what those tests cost. func decodeSamples(namespace string) ([]client.Object, error) { + parsed, err := samples() + if err != nil { + return nil, err + } + objects := make([]client.Object, 0, len(parsed)) + for _, obj := range parsed { + obj = obj.DeepCopyObject().(client.Object) + if _, cluster := obj.(*crewletaiv1alpha1.EngineClass); !cluster { + obj.SetNamespace(namespace) + } + objects = append(objects, obj) + } + return objects, nil +} + +// samples is every object in config/samples, decoded once, in no namespace. +var samples = sync.OnceValues(func() ([]client.Object, error) { files, err := filepath.Glob(filepath.Join("..", "..", "config", "samples", "v1alpha1_*.yaml")) if err != nil { return nil, err @@ -72,13 +99,31 @@ func decodeSamples(namespace string) ([]client.Object, error) { if err := yaml.UnmarshalStrict([]byte(manifest), obj); err != nil { return nil, fmt.Errorf("%s: %w", file, err) } - if typeMeta.Kind != render.KindEngineClass { - obj.SetNamespace(namespace) - } objects = append(objects, obj) } } return objects, nil +}) + +// testScheme holds the API types the fake clients of these tests serve, built +// once per test binary: a scheme is read-only once built, and building one, +// with the REST mapper a fake client otherwise derives from it, cost each +// test that made a client more than the test itself. +var testScheme = sync.OnceValue(func() *runtime.Scheme { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(crewletaiv1alpha1.AddToScheme(scheme)) + return scheme +}) + +// testRESTMapper maps every kind testScheme holds, built once with it. +var testRESTMapper = sync.OnceValue(func() meta.RESTMapper { + return testrestmapper.TestOnlyStaticRESTMapper(testScheme()) +}) + +// fakeClient is a fake client builder over testScheme and testRESTMapper. +func fakeClient() *fake.ClientBuilder { + return fake.NewClientBuilder().WithScheme(testScheme()).WithRESTMapper(testRESTMapper()) } // inputOf sorts objects into the input of a render. diff --git a/internal/controller/config_unreadable_test.go b/internal/controller/config_unreadable_test.go new file mode 100644 index 00000000..ddf803a2 --- /dev/null +++ b/internal/controller/config_unreadable_test.go @@ -0,0 +1,823 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretref" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// These tests hold what the operator does with an engine that cannot open the +// fleet's revision of the company document: the engine answers every read of +// the document 409 with no Retry-After, and every write built from it too, and +// no wait clears it. A revision sealed in a way the engine cannot open (under a +// key its keyring lacks, or in a form its build does not read) is somebody's +// document, which no write replaces: it is reported as what blocks the +// document, in the engine's words and with the remedy, and looked at again at +// the resync interval, not ten seconds on with a promise of a retry that +// changes nothing. A revision stored without a seal is the one a whole document +// replaces, reading nothing from it: under Enforce the operator replaces it +// with the rendered document, and under Seed and Observe, which write no such +// thing, it blocks the document as the others do. + +// keyringLost, sealMissing and unknownSeal are the engine's answers to a read +// of revision, sealed under a key its keyring no longer holds, stored without a +// seal, or sealed in a form its build does not read, as crewlet's configapi +// words them (RefuseUnreadable, UnreadableRevisionError) and its +// httpjson.Envelope sends them, the detail and the hint each ended as a +// sentence. +func keyringLost(revision string) string { + return fmt.Sprintf(`{"error":"unreadable_revision","revision_id":%q,`+ + `"message":"That revision is sealed under a key this deployment no longer holds, so it cannot be read. `+ + `Put the key back in the node's keyring first.",`+ + `"detail":"This node's keyring doesn't hold the key that revision is sealed under.",`+ + `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's `+ + `secrets.keys first."}`, revision) +} + +func sealMissing(revision string) string { + return fmt.Sprintf(`{"error":"unsealed_revision","revision_id":%q,`+ + `"message":"That revision is stored without a seal, which a build older than the mandatory keyring wrote, `+ + `and a node reads only sealed revisions because the seal is what authenticates one. The hint says what `+ + `brings its document back.",`+ + `"detail":"This node's copy of that revision is stored without a seal, so nothing vouches for who wrote it.",`+ + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown, edited nor `+ + `reverted to; to replace it, an admin sends the whole document from your own copy with PUT /config, or `+ + `imports it with `+"`crewlet config import`"+`, either of which stores it sealed."}`, revision) +} + +func unknownSeal(revision string) string { + return fmt.Sprintf(`{"error":"unknown_seal_revision","revision_id":%q,`+ + `"message":"That revision is sealed in a form this build of Crewlet doesn't read, most likely by a newer `+ + `build, so this node can't read or replace it. The hint says what fixes it.",`+ + `"detail":"This node's copy of that revision is sealed in a form this build of Crewlet doesn't read.",`+ + `"hint":"A newer build of Crewlet most likely sealed it here, and no write here replaces it, since that `+ + `would discard it unread; run that build on this node again."}`, revision) +} + +// fleetKeyMissing, fleetSealMissing and fleetUnknownSeal are the engine's +// answers to a read of revision met through the fleet's activation pointer, +// sealed under a key this node's keyring was never given, carrying no seal, or +// sealed in a form this node's build does not read, worded and ended as the +// three above. A seal the build does not read is one code for either origin. +func fleetKeyMissing(revision string) string { + return fmt.Sprintf(`{"error":"unreadable_fleet_revision","revision_id":%q,`+ + `"message":"This node doesn't hold the key the fleet sealed its revision under, so it can't read it. `+ + `The hint says what fixes it.",`+ + `"detail":"This node doesn't hold the key the fleet sealed this revision under.",`+ + `"hint":"Add the key the fleet seals under to this node's secrets.keys and restart it."}`, revision) +} + +func fleetSealMissing(revision string) string { + return fmt.Sprintf(`{"error":"unsealed_fleet_revision","revision_id":%q,`+ + `"message":"The fleet's revision arrived without a seal, so this node won't read it. The hint says `+ + `what to do.",`+ + `"detail":"The fleet's revision arrived without a seal, so this node won't read it.",`+ + `"hint":"Every node seals what it activates, so this body came from outside the fleet and is neither shown `+ + `nor edited; to replace it, an admin sends the whole document with PUT /config, or imports it with `+ + "`crewlet config import`"+`, either of which stores it sealed and activates it."}`, revision) +} + +func fleetUnknownSeal(revision string) string { + return fmt.Sprintf(`{"error":"unknown_seal_revision","revision_id":%q,`+ + `"message":"That revision is sealed in a form this build of Crewlet doesn't read, most likely by a newer `+ + `build, so this node can't read or replace it. The hint says what fixes it.",`+ + `"detail":"The fleet's revision is sealed in a form this build of Crewlet doesn't read.",`+ + `"hint":"A newer build of Crewlet most likely sealed it, and no write here replaces it, since that would `+ + `discard it unread; upgrade this node to the build the rest of the fleet runs."}`, revision) +} + +// cannotOpen makes the fleet's revision one the engine cannot open, answering +// every request that reads it with answer of its id. +func (e *storeEngine) cannotOpen(answer func(revision string) string) { + e.mu.Lock() + defer e.mu.Unlock() + e.revisions[len(e.revisions)-1].unopened = answer +} + +// readsActive reports whether r is a request the engine answers by reading +// active, which it cannot open, and so refuses as it refuses a read of it: a +// read of the document; a reload, which activates it again; and a write +// (a dry run included) over any but one stored without a seal, which a whole +// document replaces, reading nothing from it. +func readsActive(r *http.Request, active storeRevision) bool { + switch { + case r.URL.Path == configRoute && r.Method == http.MethodGet: + return true + case r.URL.Path == configRoute && r.Method == http.MethodPut: + return !replaceable(active) + } + return r.URL.Path == reloadRoute && r.Method == http.MethodPost +} + +// replaceable reports whether a write replaces revision, which the engine +// cannot open, as crewlet's configapi decides it (Service.prepare): a whole +// document goes on over a revision stored without a seal, and over no other. +func replaceable(revision storeRevision) bool { + var answer struct { + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(revision.unopened(revision.id)), &answer); err != nil { + panic(err) + } + return answer.Error == "unsealed_revision" || answer.Error == "unsealed_fleet_revision" +} + +// What the RevisionUnreadable message says, as more than one case expects +// it: what the keyring does not do and which Secret it is, what the engine +// does not do with a revision without a seal, and what replaces one under a +// policy that does not. +const ( + saysKeyring = "keyring does not open" + saysKeyringSecret = "Secret acme-keyring" + saysUnsealed = "which it stores without a seal" + saysEnforce = "set to Enforce" +) + +// withPolicy makes w's reconcile one of its Company under policy. +func withPolicy(t *testing.T, w *withdrawal, policy crewletaiv1alpha1.ManagementPolicy) { + t.Helper() + company := w.run.company.DeepCopy() + company.Spec.Config.ManagementPolicy = policy + w.again(t, company) +} + +// TestARevisionTheEngineCannotOpenBlocksTheDocument holds an engine that cannot +// open the fleet's revision, where no write of the operator's may replace it, +// to ConfigApplied=False RevisionUnreadable: the message names the revision, +// quotes the engine's account and its remedy, and says what restores it, or +// replaces it, in the Company's terms; the operator writes nothing, neither +// the document nor the store; and the next look is at the resync interval. +// Reported as an engine that is unavailable, it promised a retry in ten +// seconds that no retry could keep. A revision sealed in a way the engine +// cannot open blocks under every policy, Enforce included, and one stored +// without a seal under Seed and Observe, which write no document over the +// fleet's revision. +func TestARevisionTheEngineCannotOpenBlocksTheDocument(t *testing.T) { + for _, c := range []struct { + name string + answer func(revision string) string + policy crewletaiv1alpha1.ManagementPolicy + says []string + }{ + { + name: "a key the keyring lost", answer: keyringLost, policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{saysKeyring, "doesn't hold the key that revision is sealed under", + "restore that key to the node's secrets.keys first", saysKeyringSecret}, + }, + { + name: "the fleet's revision under a key this node lacks", answer: fleetKeyMissing, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{saysKeyring, "doesn't hold the key the fleet sealed this revision under", + "Add the key the fleet seals under", saysKeyringSecret, "takes it when it restarts"}, + }, + { + name: "a seal this node's build does not read", answer: unknownSeal, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{"does not read the seal of revision r1", "sealed in a form this build of Crewlet doesn't read", + "run that build on this node again", "newer than the one the data pod runs", + "keyring Secret acme-keyring is not at fault", "no write replaces it", + "the engine image of EngineClass standard", "takes it when it restarts"}, + }, + { + name: "the fleet's revision under a seal this node's build does not read", answer: fleetUnknownSeal, + policy: crewletaiv1alpha1.ManagementPolicyEnforce, + says: []string{"does not read the seal of revision r1", "The fleet's revision is sealed in a form", + "upgrade this node to the build the rest of the fleet runs", "keyring Secret acme-keyring is not at fault", + "the engine image of EngineClass standard"}, + }, + { + name: "a key the keyring lost, under Seed", answer: keyringLost, policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysKeyring, saysKeyringSecret}, + }, + { + name: "a revision stored without a seal, under Seed", answer: sealMissing, + policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysUnsealed, "stored without a seal, so nothing vouches", + "an admin sends the whole document from your own copy with PUT /config", + "stored by an engine older than the keyring", "Under the Company's Seed policy", + "writes only into an engine with no document", saysEnforce}, + }, + { + name: "a revision stored without a seal, under Observe", answer: sealMissing, + policy: crewletaiv1alpha1.ManagementPolicyObserve, + says: []string{saysUnsealed, "an admin sends the whole document from your own copy", + "Under the Company's Observe policy the operator writes nothing", saysEnforce}, + }, + { + name: "the fleet's revision without a seal, under Seed", answer: fleetSealMissing, + policy: crewletaiv1alpha1.ManagementPolicySeed, + says: []string{saysUnsealed, "arrived without a seal", "came from outside the fleet", + "put there from outside the fleet", "Under the Company's Seed policy", saysEnforce}, + }, + { + name: "the fleet's revision without a seal, under Observe", answer: fleetSealMissing, + policy: crewletaiv1alpha1.ManagementPolicyObserve, + says: []string{"arrived without a seal", "put there from outside the fleet", + "Under the Company's Observe policy the operator writes nothing", saysEnforce}, + }, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + withPolicy(t, w, c.policy) + w.engine.cannotOpen(c.answer) + _, err := w.run.converge(ctx, w.client) + if err == nil { + t.Fatal("converge() = nil over a revision the engine cannot open") + } + result := w.run.engineFailed(ctx, w.client, err) + + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable { + t.Fatalf("ConfigApplied = %+v, want False RevisionUnreadable", got) + } + // Every answer names the fleet's revision, r1. + for _, s := range append([]string{"revision r1"}, c.says...) { + if !strings.Contains(got.Message, s) { + t.Errorf("ConfigApplied does not say %q: %s", s, got.Message) + } + } + if strings.Contains(got.Message, "tries again") || strings.Contains(got.Message, ".).") || + !endsOnce(got.Message) { + t.Errorf("ConfigApplied promises a retry, or ends the engine's words or its own with a second "+ + "full stop: %s", got.Message) + } + if result.RequeueAfter != w.run.resyncInterval() { + t.Errorf("the next look is %s on, want the resync interval, %s", result.RequeueAfter, w.run.resyncInterval()) + } + if written, refused := w.engine.writes(); len(written) != 0 || refused != 0 { + t.Errorf("the operator wrote %v (and %d refused) over a revision the engine cannot open, want nothing", + written, refused) + } + if !w.engine.holds(w.registry) || !slices.Equal(w.engine.ids(), []string{"r1"}) { + t.Errorf("the store holds the registry: %t, and the revisions are %v; want both as they were", + w.engine.holds(w.registry), w.engine.ids()) + } + if recorded := drained(w.events); !warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want a Warning RevisionUnreadable", recorded) + } + + // The next reconcile meets the same answer with the condition + // already recorded: it is not told again. + meta.SetStatusCondition(&w.run.company.Status.Conditions, *got) + w.again(t, w.run.company) + if _, err := w.run.converge(ctx, w.client); err != nil { + w.run.engineFailed(ctx, w.client, err) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want the refusal ConfigApplied already reports told once", recorded) + } + }) + } +} + +// TestEnforceReplacesARevisionStoredWithoutASeal holds the operator, under +// Enforce, to the engine's remedy for a revision it stores without a seal, +// this node's copy or the fleet's: the rendered document replaces it, as one +// write conditioned on that revision (If-Match), which is still the base +// whatever nobody can read of it, and the Company ends Applied, the engine +// having applied the epoch, with what was replaced said in the condition, the +// new revision's summary and a Warning event. Nothing is read from the +// revision, so nothing the store holds is deleted before the write, the value +// of a deleted Secret included: the engine's nodes run, and boot again, the +// revision before it, whose reading nobody knows. The sync after the write +// removes it, the replacement letting it go, and the replacement is activated +// again at once for the engine to read the store without it. The next +// reconcile finds the engine running the rendered document and writes +// nothing. +func TestEnforceReplacesARevisionStoredWithoutASeal(t *testing.T) { + for _, c := range []struct { + name string + answer func(revision string) string + }{ + {"this node's copy without a seal", sealMissing}, + {"the fleet's revision without a seal", fleetSealMissing}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + // The state records no reading of r1, so nothing says which of + // the store's names r1 reads. + w := newWithdrawal(t, runningNoRecord) + w.engine.cannotOpen(c.answer) + var heldAtWrite bool + w.engine.beforeWrite = func(e *storeEngine) { _, heldAtWrite = e.store[w.registry] } + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the rendered document written over a revision without a seal", err) + } + written, refused := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r2"}) || refused != 0 || + !slices.Equal(conditions, []string{"r1"}) { + t.Fatalf("the operator wrote %v on the conditions %v (and %d were refused), want r2 alone, "+ + "conditioned on r1", written, conditions, refused) + } + if !heldAtWrite { + t.Error("the store lost the value of a deleted Secret before the write, while the nodes run a revision " + + "nobody can read") + } + if summary := w.engine.summaryOfRevision("r2"); !strings.HasPrefix(summary, + "crewlet-operator: replaced revision r1, stored without a seal, with Company/acme") { + t.Errorf("r2 is summarised %q, want it saying it replaced r1, stored without a seal", summary) + } + // The engine activates r2 again itself as the value r2 reads is + // removed after it. + if ids := w.engine.ids(); !slices.Equal(ids, []string{"r1", "r2", "r3"}) || + w.engine.summaryOf() != "reload: secret "+w.registry+" removed" { + t.Errorf("the engine's revisions are %v, the last summarised %q; want r1, the replacement r2, and r2 "+ + "activated again as r3 at the removal of the deleted Secret's value", ids, w.engine.summaryOf()) + } + if w.engine.holds(w.registry) { + t.Error("the store keeps the value of a deleted Secret the replacement lets go") + } + if withdrawn := w.state(t).Withdrawn(); len(withdrawn) != 0 { + t.Errorf("the state owes an activation without %v, want none: r3 read the store without it", withdrawn) + } + + health, healthErr := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, health, healthErr, result) + got := w.run.configApplied + if got == nil || got.Status != metav1.ConditionTrue || got.Reason != crewletaiv1alpha1.ReasonApplied || + !strings.Contains(got.Message, "as revision r2, replacing revision r1, which the engine stores without a seal") { + t.Errorf("ConfigApplied = %+v, want True Applied, saying r2 replaced r1, stored without a seal", got) + } + if w.run.applied.kind != appliedInRevision || w.run.applied.revision != "r3" { + t.Errorf("the parts are %+v, want in revision r3, r2 activated again", w.run.applied) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonRevisionUnreadable+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "active revision r1 was one it stores without a seal") || + !strings.Contains(recorded[i], "revision r2, the rendered document, replaced it") { + t.Errorf("events %q, want a Warning RevisionUnreadable saying r2 replaced r1", recorded) + } + if !told(recorded, "Reactivated") || warned(recorded, "SecretWithdrawalFailed") || told(recorded, "SecretKept") { + t.Errorf("events %q, want r2 activated again, and nothing kept or failed", recorded) + } + + w.again(t, w.statusWritten()) + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2"}) { + t.Errorf("the operator wrote %v, want r2 alone: the next reconcile finds it running", written) + } + }) + } +} + +// TestAReplacementOverARevisionWithoutASealKeepsWhatTheNodesRun holds the +// replacement of a revision the engine stores without a seal, under Enforce, +// to what the fleet says its nodes run: having refused that revision, each +// runs the one before it, which the operator reads (GET +// /config/revisions/{id}) and keeps the names of, beside every name the store +// lists, while the rendered document is written over the revision without a +// seal, conditioned on it, all the same. The value of a deleted Secret the +// revision before reads as required stays in the store after the write, for +// as long as a node runs that revision, and goes once the nodes run the +// replacement. Deciding by the fleet's revision alone, the operator would +// delete a value a running node reads at its next apply attempt. +func TestAReplacementOverARevisionWithoutASealKeepsWhatTheNodesRun(t *testing.T) { + ctx := context.Background() + // r1 requires the registry, as the state records, and the rendered + // document lets it go. r2, on the fleet's pointer, carries no seal, so + // the node refused it and runs r1. + w := newWithdrawal(t, runningRequires) + w.engine.mu.Lock() + w.engine.revisions = append(w.engine.revisions, storeRevision{id: w.engine.mint(), unopened: fleetSealMissing}) + w.engine.serving = map[string]string{storeNode: "r1"} + w.engine.mu.Unlock() + w.engine.takeRequests() + + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the rendered document written over a revision without a seal", err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r3 alone, conditioned on r2", written, conditions) + } + if requests := w.engine.takeRequests(); !slices.Contains(requests, "GET /config/revisions/r1") { + t.Errorf("the engine answered %q, want r1, which the node runs, read", requests) + } + if !w.engine.holds(w.registry) { + t.Error("the store lost the value of a deleted Secret r1 requires while a node runs r1") + } + + // The node applies the replacement. + w.engine.mu.Lock() + w.engine.serving = nil + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + if w.engine.holds(w.registry) { + t.Error("the store keeps the value of a deleted Secret once the nodes run the replacement, which lets it go") + } + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r3"}) { + t.Errorf("the operator wrote %v, want r3 alone: the next reconcile finds the nodes running it", written) + } +} + +// TestAReplacementRefusedOverARevisionWithoutASealKeepsTheStore holds a +// document the engine refuses over a revision it stores without a seal to the +// store as it was and to an activation owed and not asked for. Nothing the +// store holds is deleted: the nodes run, and boot again, the revision before +// the unsealed one, which may read a name the rendered document does not. And +// the engine neither shows nor activates again a revision without a seal, so +// asking for one is refused every time: asked, the reconcile came back every +// ten seconds over a refusal no retry clears, each time with a +// SecretWithdrawalFailed. The write that replaces it, once the document is +// valid, settles what is owed and removes what it lets go. +func TestAReplacementRefusedOverARevisionWithoutASealKeepsTheStore(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 let the registry go, and an earlier sync withdrew it, the + // engine's re-activation at the removal failing: an activation without + // it is owed. + operatorActivates(w) + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + if !slices.Equal(owed.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry", owed.Withdrawn()) + } + // A value the store holds that the rendered document does not + // reference: one the revision the nodes run may. + other := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: "earlier", Key: optionalKey}) + w.engine.mu.Lock() + w.engine.store[other] = "earlier" + w.engine.mu.Unlock() + w.engine.cannotOpen(sealMissing) + w.engine.refusal = invalidDocument + w.engine.takeRequests() + + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: a refused document waits for its objects", result.RequeueAfter) + } + requests := w.engine.takeRequests() + if i := slices.Index(requests, "PUT "+configRoute); i < 0 || + slices.ContainsFunc(requests[i+1:], func(r string) bool { return strings.HasSuffix(r, configRoute) || strings.HasSuffix(r, reloadRoute) }) { + t.Errorf("the engine answered %q, want the dry run, and no read or reload of the document after it", requests) + } + if !w.engine.holds(other) || len(w.engine.ids()) != 1 { + t.Errorf("the store holds %s: %t, and the revisions are %v; want it kept and r1 alone", other, + w.engine.holds(other), w.engine.ids()) + } + if withdrawn := w.state(t).Withdrawn(); !slices.Equal(withdrawn, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry still owed", withdrawn) + } + if recorded := drained(w.events); warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } + + w.again(t, w.statusWritten()) + w.engine.refusal = nil + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatal(err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r2"}) || !slices.Equal(conditions, []string{"r1"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r2, conditioned on r1", written, conditions) + } + if withdrawn := w.state(t).Withdrawn(); len(withdrawn) != 0 || w.engine.holds(other) { + t.Errorf("the state owes %v and the store holds %s: %t; want r2's activation to settle both", withdrawn, + other, w.engine.holds(other)) + } +} + +// TestAHeldDocumentOverARevisionWithoutASealKeepsTheStore holds a document +// held for a missing Secret, over a revision the engine stores without a seal, +// to what enforce does before the write that replaces it: the store keeps the +// value of a deleted Secret, told as kept for a revision nobody can read. +func TestAHeldDocumentOverARevisionWithoutASealKeepsTheStore(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.cannotOpen(fleetSealMissing) + w.engine.takeRequests() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: the hold's own reason brings the reconcile back", result.RequeueAfter) + } + if requests := w.engine.takeRequests(); slices.Contains(requests, "POST "+reloadRoute) || + slices.ContainsFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) { + t.Errorf("the engine answered %q, want no reload and no deletion", requests) + } + if !w.engine.holds(w.registry) { + t.Error("the store lost the value of a deleted Secret while the nodes run a revision nobody can read") + } + recorded := drained(w.events) + if warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal SecretKept ") }) + if i < 0 || !strings.Contains(recorded[i], "revision r1, the engine's active revision, is stored without a seal") || + strings.Contains(recorded[i], "which the engine runs") { + t.Errorf("events %q, want SecretKept calling r1, stored without a seal, the engine's active revision, which its "+ + "nodes do not run", recorded) + } + // And what SecretsSynced and a part's ResolvedRefs line say of it. + if got := w.run.retainedSentence(); !strings.Contains(got, ": the engine's active revision is stored without a seal") { + t.Errorf("SecretsSynced's sentence is %q, want it calling r1 the engine's active revision", got) + } + registry := secretref.Key{Namespace: "ns", Name: optionalSecret, Key: optionalKey} + if got := w.run.leftOutLines([]leftOutValue{{key: registry, line: registry.String()}}); len(got) != 1 || + !strings.Contains(got[0], "the value it held for its active revision, which is stored without a seal") { + t.Errorf("ResolvedRefs's line is %q, want it calling r1 the engine's active revision", got) + } +} + +// TestAHeldDocumentOverARevisionWithoutASealOwesItsActivation holds a +// document held for a missing Secret, over a revision the engine stores +// without a seal, with an activation owed for a value an earlier sync +// withdrew, to the withdrawal owed and not asked for: the engine neither shows +// nor activates again a revision without a seal, so asking for one is refused +// every time, and the reconcile came back every ten seconds over a refusal no +// retry clears, each time with a SecretWithdrawalFailed. Nor does the +// revision without a seal settle it, although the fleet's pointer names it +// at an epoch above the one the withdrawal was marked by: the nodes run the +// revision before it, which activated before the withdrawal. The write that +// replaces the revision settles it. +func TestAHeldDocumentOverARevisionWithoutASealOwesItsActivation(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 let the registry go, and an earlier sync withdrew it, the + // engine's re-activation at the removal failing: an activation without + // it is owed. + operatorActivates(w) + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + // Marked by r1's epoch, the pointer's when the withdrawal was made. + owed = owed.Observed(1) + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + if !slices.Equal(owed.Withdrawn(), []string{w.registry}) { + t.Fatalf("the state owes %v, want the registry", owed.Withdrawn()) + } + // Then somebody's revision r2, stored without a seal, takes the + // pointer, at epoch 2. + w.engine.mu.Lock() + w.engine.revisions = append(w.engine.revisions, storeRevision{id: w.engine.mint(), document: w.engine.active().document}) + w.engine.mu.Unlock() + w.engine.cannotOpen(fleetSealMissing) + w.engine.takeRequests() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != 0 { + t.Errorf("the next look is %s on, want none: the hold's own reason brings the reconcile back", result.RequeueAfter) + } + requests := w.engine.takeRequests() + if i := slices.Index(requests, "GET "+configRoute); i < 0 || slices.ContainsFunc(requests[i+1:], func(r string) bool { + return strings.HasSuffix(r, configRoute) || strings.HasSuffix(r, reloadRoute) + }) { + t.Errorf("the engine answered %q, want the one read of the document and no reload after it", requests) + } + if withdrawn := w.state(t).Withdrawn(); !slices.Equal(withdrawn, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry still owed", withdrawn) + } + if recorded := drained(w.events); warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("events %q, want no SecretWithdrawalFailed over a revision nothing activates again", recorded) + } +} + +// TestUnderEnforceAnUnsealedRevisionAfterTheWriteIsLookedAtSoon holds the one +// report of a revision without a seal under Enforce, one that lands on the +// fleet's pointer after the operator's read (from outside the fleet), to the +// operator's own remedy and its timing: ConfigApplied says the next look +// replaces it, and that look comes ten seconds on, not at the resync interval, +// since nothing but that look is waited for. +func TestUnderEnforceAnUnsealedRevisionAfterTheWriteIsLookedAtSoon(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 is the rendered document, so a rotated value, whose re-activation + // by the engine fails, is what has the operator write it. + w.rotate(t) + operatorActivates(w) + w.engine.afterApply = func(e *storeEngine) { + e.revisions = append(e.revisions, storeRevision{id: e.mint(), unopened: fleetSealMissing}) + } + _, err := w.run.converge(ctx, w.client) + if _, ok := engine.Unsealed(err); !ok { + t.Fatalf("converge() = %v, want the read after the write refused over a revision without a seal", err) + } + result := w.run.engineFailed(ctx, w.client, err) + if result.RequeueAfter != engineRetryInterval { + t.Errorf("the next look is %s on, want %s", result.RequeueAfter, engineRetryInterval) + } + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonRevisionUnreadable || + !strings.Contains(got.Message, "the operator replaces it with the rendered document at its next look") { + t.Errorf("ConfigApplied = %+v, want RevisionUnreadable with the Enforce remedy", got) + } +} + +// TestAReplacementTheEngineCouldNotConfirmIsKept holds a write over a +// revision stored without a seal whose activation the engine could not +// confirm to the record every such write keeps (config_unconfirmed.go): while +// the engine still runs that revision, the write may land, so it is kept, and +// the write sent again on the same condition is recorded beside it; once one +// lands, the reconcile records it as the operator's, reports what it replaced +// as a confirmed replacement is reported, and writes nothing more. Settled +// against no revision, the first would be dropped as one that can never land, +// and its landing read as somebody else's document; confirmed from a record +// that did not keep what it replaced, it was reported as a plain Applied, and +// nothing said a revision nobody could read had gone. +func TestAReplacementTheEngineCouldNotConfirmIsKept(t *testing.T) { + w := newWithdrawal(t, runningNoRecord) + w.engine.cannotOpen(sealMissing) + for range 2 { + w.engine.activation = pending + reconcileOnce(t, w) + if c := w.run.configApplied; c == nil || c.Reason != crewletaiv1alpha1.ReasonEngineUnavailable || + !strings.Contains(c.Message, "which would replace revision r1, stored without a seal") { + t.Errorf("ConfigApplied is %+v, want EngineUnavailable saying the write would replace r1", c) + } + w.again(t, w.statusWritten()) + } + got := w.run.company.Status.Config.Unconfirmed + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) || len(got) != 2 || + got[0].RevisionID != "r2" || got[0].BaseRevisionID != "r1" || got[1].RevisionID != "r3" || + got[1].BaseRevisionID != "r1" || got[0].Replaced != crewletaiv1alpha1.ReplacedUnsealed { + t.Fatalf("the operator wrote %v and keeps %+v unconfirmed; want r2 and r3, each built on r1, kept, "+ + "recorded as replacing a revision without a seal", written, got) + } + drained(w.events) + w.engine.mu.Lock() + w.engine.land("r2") + w.engine.mu.Unlock() + w.engine.activation = confirmed + reconcileOnce(t, w) + if written, _ := w.engine.writes(); !slices.Equal(written, []string{"r2", "r3"}) { + t.Errorf("the operator wrote %v, want nothing beyond r2 and r3: r2 landed", written) + } + if config := w.statusWritten().Status.Config; config.AppliedHash != got[0].AppliedHash || len(config.Unconfirmed) != 0 { + t.Errorf("status.config is %+v, want r2's write recorded, with its hash, and nothing unconfirmed", config) + } + if c := w.run.configApplied; c == nil || c.Status != metav1.ConditionTrue || c.Reason != crewletaiv1alpha1.ReasonApplied || + !strings.Contains(c.Message, "as revision r2, replacing revision r1, which the engine stores without a seal") { + t.Errorf("ConfigApplied is %+v, want True Applied, saying r2 replaced r1, stored without a seal", c) + } + recorded := drained(w.events) + if i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Normal Activated ") }); i < 0 || + !strings.Contains(recorded[i], "runs revision r2 as epoch") { + t.Errorf("events %q, want an Activated event saying the engine runs r2", recorded) + } + i := slices.IndexFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning "+crewletaiv1alpha1.ReasonRevisionUnreadable+" ") + }) + if i < 0 || !strings.Contains(recorded[i], "active revision r1 was one it stores without a seal") || + !strings.Contains(recorded[i], "revision r2, the rendered document, replaced it") { + t.Errorf("events %q, want a Warning RevisionUnreadable saying r2 replaced r1", recorded) + } +} + +// TestAWriteIsConditionedOnTheRevisionTheReconcileRead holds the operator's +// write to the revision its reconcile read, which its diff, its drift and the +// names it kept in the store were decided against. Another writer's revision +// that lands between that read and the dry run is not written over unseen, nor +// reported as the revision the reconcile read: the reconcile starts over from +// a fresh read, and writes over what it read then. Over a revision the engine +// stores without a seal, that is what keeps the operator from saying it +// replaced one when it replaced another writer's document. +func TestAWriteIsConditionedOnTheRevisionTheReconcileRead(t *testing.T) { + for _, c := range []struct { + name string + // unsealed makes r1 a revision the engine stores without a seal. + unsealed bool + }{ + {name: "the reconcile read the engine's document"}, + {name: "the reconcile read a revision stored without a seal", unsealed: true}, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if c.unsealed { + // The operator writes over a revision it cannot read. + w.engine.cannotOpen(sealMissing) + } else { + // r1 is the rendered document, so a rotated value, whose + // re-activation by the engine fails, is what has the + // operator write it. + w.rotate(t) + operatorActivates(w) + } + // Somebody activates another document, which the engine opens, + // after the operator read r1 and before its dry run. + w.engine.beforeCheck = func(e *storeEngine) { e.activateOther(w.requiring) } + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the reconcile started over and the rendered document written", err) + } + written, _ := w.engine.writes() + if conditions := w.engine.writtenOn(); !slices.Equal(written, []string{"r3"}) || + !slices.Equal(conditions, []string{"r2"}) { + t.Fatalf("the operator wrote %v on the conditions %v, want r3 alone, conditioned on r2, the revision "+ + "its second read found", written, conditions) + } + got := w.run.configApplied + if got == nil || got.Reason != crewletaiv1alpha1.ReasonDriftCorrected || + !strings.HasPrefix(got.Message, "Revision r2, which the operator did not write") { + t.Errorf("ConfigApplied = %+v, want DriftCorrected over r2, the revision the write replaced", got) + } + if summary := w.engine.summaryOfRevision("r3"); strings.Contains(summary, "without a seal") { + t.Errorf("r3 is summarised %q, which says it replaced a revision without a seal; it replaced r2", summary) + } + if recorded := drained(w.events); warned(recorded, crewletaiv1alpha1.ReasonRevisionUnreadable) { + t.Errorf("events %q, want none saying a revision without a seal was replaced", recorded) + } + }) + } +} + +// TestASeatHeldOverAReplacementSaysTheDocumentDoesNotKeepIt holds the SeatHeld +// report on a document replacing a revision stored without a seal to what the +// engine can say of it: the document does not keep those seats as human seats, +// since nobody can read whether the revision it replaces had them, and not that +// it takes them out of the company. +func TestASeatHeldOverAReplacementSaysTheDocumentDoesNotKeepIt(t *testing.T) { + held := asReturned(&engine.SeatHeldError{Held: map[string]engine.SeatHolder{ + seatFounder: {Kind: engine.HolderPerson, Stage: "active"}, + }}) + for _, c := range []struct { + name, unsealed, says, not string + }{ + {name: "a document replacing the engine's", says: "it takes human seats out of the company", + not: "does not keep"}, + {name: "a document replacing a revision without a seal", unsealed: "r1", + says: "it replaces revision r1, which the engine does not open, and does not keep as human seats these " + + "seats its identity directory holds: founder (held by a person, active)", + not: "takes human seats out"}, + } { + t.Run(c.name, func(t *testing.T) { + run, _, _, _ := refusalRun() + run.unsealed = c.unsealed + if !run.seatHeld(held) { + t.Fatal("seatHeld() = false for a *SeatHeldError") + } + if got := run.configApplied.Message; !strings.Contains(got, c.says) || strings.Contains(got, c.not) { + t.Errorf("ConfigApplied says %q; want it saying %q, and not %q", got, c.says, c.not) + } + }) + } +} + +// TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt holds the +// withdrawal of a deleted Secret's value to the same next look: the engine +// cannot take its revision again while its keyring does not open it either, +// so the operator says so and looks again at the resync interval. +func TestAWithdrawalOverARevisionTheEngineCannotOpenWaitsForIt(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.cannotOpen(keyringLost) + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != w.run.resyncInterval() { + t.Errorf("the next look is %s on, want the resync interval, %s", result.RequeueAfter, w.run.resyncInterval()) + } + recorded := drained(w.events) + if !warned(recorded, "SecretWithdrawalFailed") || + !strings.Contains(strings.Join(recorded, "\n"), "unreadable_revision") { + t.Errorf("events %q, want SecretWithdrawalFailed quoting the engine's refusal", recorded) + } + if !w.engine.holds(w.registry) || len(w.engine.ids()) != 1 { + t.Errorf("the store holds the registry: %t, and the revisions are %v; want both as they were", + w.engine.holds(w.registry), w.engine.ids()) + } +} diff --git a/internal/controller/config_withdraw.go b/internal/controller/config_withdraw.go index d5cb0e25..d665e3c8 100644 --- a/internal/controller/config_withdraw.go +++ b/internal/controller/config_withdraw.go @@ -8,6 +8,7 @@ import ( "maps" "slices" "strings" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -57,7 +58,7 @@ import ( // pushes in one request (engine.Client.PutSecrets), one activation for all of // them, and a rotation of a value the running document reads needs no apply // of the same document from the operator, who records the engine's -// re-activation as its own (reloadedBySync). +// re-activation as its own (syncActivation). // // The store serves every revision by that revision's own reading of a key, // though, and the revision the engine runs is the last good one, which keeps @@ -101,7 +102,11 @@ import ( // later reconcile, of this operator or one restarted; the activation an // apply makes is recorded before anything is deleted after it, so a deletion // whose own record a failed write lost is withdrawn again by a later -// reconcile, which knows the revision lets the key go. +// reconcile, which knows the revision lets the key go. And it is settled by +// any activation the engine is read running after the deletion, whoever made +// it, by the fleet's activation epoch the deletion was marked with +// (settleRead, markChanges): a reload that failed, or lost its race to +// another writer, is read back rather than taken for a failed withdrawal. // // The engine offers no conditional reload and no conditional delete: // whichever revision is active when the request lands is the one activated @@ -110,7 +115,7 @@ import ( // deletion of a key that one lets go may require the key. The operator // records an activation again, its own or the engine's, only once it has read // back that the revision activated is the document it read (reactivate, -// reloadedBySync), and otherwise reads the revision activated as one whose +// syncActivation), and otherwise reads the revision activated as one whose // reading it does not know, and says so. // withdrawHeld does, for a document this reconcile could not apply, what the @@ -122,12 +127,15 @@ import ( // again if it references one, and says what the store keeps for it. It does // nothing but under the Enforce policy, on an engine that runs, and nothing // where no such reference is absent, no activation is owed, and the store -// keeps nothing it has said it keeps. +// keeps nothing it has said it keeps. Over a revision the engine stores +// without a seal it removes nothing and activates nothing: nobody can read +// what that revision reads, and the write that replaces it once the document +// is no longer held settles what is owed (KeepListed, reactivate). // // An error is the API server's, for the reconcile to retry. One of the // engine's, or one reaching it, is reported (withdrawalFailed) and tried -// again engineRetryInterval later: the document's conditions already say why -// it is held. +// again when engineRetry says: the document's conditions already say why it +// is held. func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) (ctrl.Result, error) { c := run.company if run.policy() != crewletaiv1alpha1.ManagementPolicyEnforce || suspended(c) || @@ -152,21 +160,44 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err != nil { return ctrl.Result{}, err } - if len(read.Missing()) == 0 && len(released) == 0 && len(state.Withdrawn()) == 0 && len(state.Retained()) == 0 { + if len(read.Missing()) == 0 && len(released) == 0 && len(state.Withdrawn()) == 0 && len(state.Written()) == 0 && + len(state.Retained()) == 0 && len(run.recordedConfig().Unconfirmed) == 0 { // No value to take out of the store or to say it keeps, no - // activation owed, and nothing said kept that has come back. + // activation owed, nothing said kept that has come back, and no + // write of the operator's that may run, which may let go what the + // revision the state records does not (settlePending). return ctrl.Result{}, nil } eng, err := run.Engines.Engine(ctx, c) if err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + return run.withdrawalFailed(ctx, err), nil } active, err := activeConfig(ctx, eng) + if revision, ok := engine.Unsealed(err); ok { + // A revision stored without a seal, which the rendered document + // replaces once nothing holds it (converge): until then nobody can + // read which values it reads and the nodes run, and boot again, the + // revision before it, so the store keeps everything it lists, as + // enforce keeps it before that write, and no activation can be made + // over it (reactivate). + run.unsealed, err = revision, nil + read.KeepListed(revision) + } if err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + return run.withdrawalFailed(ctx, err), nil + } + // The revision that runs may be a write of the operator's the engine + // could not confirm activating, whose reading the state recorded before + // it was sent, and whose document, where the render has moved on, lets + // go of what the one the state records does not. + settled := run.settlePending(state, active) + if settled.Recorded() != state.Recorded() { + // What the revision the state now records goes without. + if released, err = secretsync.Absent(ctx, run.APIReader, settled.LetGoKeys(settled.Recorded())); err != nil { + return ctrl.Result{}, err + } } + state = settled if active != nil { if run.doc != nil { // Held for a Secret, not refused by the render: there is a @@ -187,6 +218,17 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) if err := run.keepServing(ctx, eng, &read); err != nil { return ctrl.Result{}, err } + // What the fleet keepServing read settles or marks is saved before + // anything is asked of the engine: the reload below may land and the save + // after it be lost, and a mark the read lowered that only this reconcile + // knows of would be loaded unread again, which no activation's epoch is + // above (markChanges). + state, moved := run.settleRead(ctx, eng, active, state) + if moved { + if err := secretsync.SaveState(ctx, run.Client, stateRef, c, state); err != nil { + return ctrl.Result{}, err + } + } next, res, withdrawErr := secretsync.Withdraw(ctx, eng, read, state) if len(res.Deleted) > 0 { run.event(corev1.EventTypeNormal, "SecretsSynced", "SyncSecrets", @@ -202,14 +244,16 @@ func (run *configRun) withdrawHeld(ctx context.Context, read secretsync.Desired) } var reactivateErr error if withdrawErr == nil { - next, reactivateErr = run.reactivate(ctx, eng, active, res.Reload, res.ReloadAction, next) + next, reactivateErr = run.reactivate(ctx, eng, active, res.Reload, res.ReloadAction, next, stateRef) } if err := secretsync.SaveState(ctx, run.Client, stateRef, c, next); err != nil { return ctrl.Result{}, err } - if err := errors.Join(withdrawErr, reactivateErr); err != nil { - run.withdrawalFailed(ctx, err) - return ctrl.Result{RequeueAfter: engineRetryInterval}, nil + if withdrawErr != nil { + return run.withdrawalFailed(ctx, withdrawErr), nil + } + if reactivateErr != nil { + return run.activationFailed(ctx, reactivateErr, next), nil } return run.heldResync(ctrl.Result{}), nil } @@ -241,7 +285,8 @@ func (run *configRun) keep(read secretsync.Desired, next secretsync.State, retai // The fleet's revision first, which is the one the conditions // have always named, then those pods run beside it. for _, revision := range slices.Sorted(maps.Keys(holders)) { - k := keptRevision{revision: revision, fleets: revision == read.Running(), unknown: !next.Records(revision)} + k := keptRevision{revision: revision, fleets: revision == read.Running(), unknown: !next.Records(revision), + unsealed: run.unsealed != "" && revision == run.unsealed} if run.serving != nil { k.pods = run.serving.revisions[revision] } @@ -266,12 +311,14 @@ func (run *configRun) keep(read secretsync.Desired, next secretsync.State, retai // keptRevision is a revision that keeps a value whose Secret is gone: the // fleet's, or one engine pods run or start on beside it (pods), which reads // it as required, or is one the operator does not know the reading of -// (unknown). +// (unknown), or the fleet's that the engine stores without a seal, which +// nobody can read and its nodes do not run (unsealed). type keptRevision struct { revision string fleets bool pods revisionPods unknown bool + unsealed bool } // tellKept records a Normal SecretKept event naming the names keep found the @@ -299,7 +346,9 @@ func (run *configRun) keptPlainly() bool { // revision the engine runs" otherwise, reads them as required, or is one the // operator does not know the reading of and takes to; and a pod whose // revisions the operator does not know may run or start on one that does. -// them names what they read. +// The fleet's revision, where the engine stores it without a seal, is "the +// engine's active revision" and never the one it runs: nobody can read it, +// and its nodes run the revision before it. them names what they read. func (run *configRun) keptBecause(byID bool, them string) string { var reasons []string for _, k := range run.keptFor { @@ -307,11 +356,20 @@ func (run *configRun) keptBecause(byID bool, them string) string { switch { case !k.fleets: subject = fmt.Sprintf("revision %s, which %s,", k.revision, k.pods.phrase()) + case k.unsealed && byID: + subject = fmt.Sprintf("revision %s, the engine's active revision,", k.revision) + case k.unsealed: + subject = "the engine's active revision" case byID: subject = fmt.Sprintf("revision %s, which the engine runs,", k.revision) } reading := "reads " + them + " as required" - if k.unknown { + switch { + case k.unsealed: + reading = "is stored without a seal and nobody can read it, while the engine's nodes run the revision they " + + "applied before it, which reads the store again whenever a node restarts, so the operator takes it to " + + "read " + them + " as required" + case k.unknown: reading = "is one the operator has no record of activating and does not render, so it takes it to read " + them + " as required" } @@ -360,80 +418,198 @@ func (run *configRun) retainedSentence() string { // sync may have withdrawn a name the revision just applied started with. // active is the engine's document as the reconcile last read it, before the // sync, nil to read it; reload is the engine's own re-activation at the -// sync's removals (secretsync.Result.Reload), which reactivate takes for the -// activation where it landed. Its error is the API server's; an activation -// that failed is reported (withdrawalFailed), and failed says so, for the -// reconcile to come back engineRetryInterval later. +// sync's last write or removal (secretsync.Result.Reload), which reactivate +// takes for the activation where it landed. Its error is the API server's; an +// activation that failed is reported (activationFailed), and retry is when +// that report says the operator tries again, zero where nothing failed. func (run *configRun) settle( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, -) (failed bool, err error) { - if len(state.Withdrawn()) == 0 { - return false, nil +) (retry time.Duration, err error) { + if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { + return 0, nil } - next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state) + next, reactivateErr := run.reactivate(ctx, eng, active, reload, reloadAction, state, ref) if err := secretsync.SaveState(ctx, run.Client, ref, run.company, next); err != nil { - return false, err + return 0, err } if reactivateErr != nil { - run.withdrawalFailed(ctx, reactivateErr) - return true, nil + return run.activationFailed(ctx, reactivateErr, next).RequeueAfter, nil } - return false, nil + return 0, nil } // settleRefused is settle for a document the engine refused, as the -// reconcile ends there. +// reconcile ends there: it comes back when the report of an activation that +// failed says it does, and otherwise when an object changes. func (run *configRun) settleRefused( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, state secretsync.State, ref types.NamespacedName, ) (ctrl.Result, error) { - failed, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) - if failed { - return ctrl.Result{RequeueAfter: engineRetryInterval}, err - } - return run.heldResync(ctrl.Result{}), err + retry, err := run.settle(ctx, eng, active, reload, reloadAction, state, ref) + return run.heldResync(ctrl.Result{RequeueAfter: retry}), err } // withdrawalFailed reports err, which kept the engine from reading its store // without a value whose Secret no longer exists: until the operator gets // through, the engine may still start a tool server or an executor run with it. -func (run *configRun) withdrawalFailed(ctx context.Context, err error) { +// It returns when to try again, as engineRetry says for err. +func (run *configRun) withdrawalFailed(ctx context.Context, err error) ctrl.Result { + retry := run.engineRetry(err) logf.FromContext(ctx).Error(err, "Failed to keep the value of a deleted Secret from the engine") - run.event(corev1.EventTypeWarning, "SecretWithdrawalFailed", "ReloadConfig", + again := fmt.Sprintf("the operator tries again in %s", retry) + if engine.Waitless(err) { + again = fmt.Sprintf("the engine answered that waiting won't clear what it met, so somebody has to clear it in "+ + "the engine, and the operator asks again every %s until then", retry) + } + run.event(corev1.EventTypeWarning, "SecretWithdrawalFailed", "ReloadConfig", "%s", ended(fmt.Sprintf( "The engine may still start a tool server or an executor run with the value of a Secret that no longer exists, "+ - "which the revision it runs lets go without; the operator tries again in %s: %s", engineRetryInterval, err) + "which the revision it runs lets go without; %s: %s", again, err))) + return ctrl.Result{RequeueAfter: retry} +} + +// activationFailed reports err, which kept the engine from reading the store +// changes state still owes an activation for: a withdrawal where it owes one, +// as withdrawalFailed reports it; otherwise only values written, which the +// revision the engine runs read before they were written, so it may go on +// starting tool servers and executor runs with the values the Secrets held +// before, a credential somebody rotated included (SecretRefreshFailed). It +// returns when to try again, as engineRetry says for err. +func (run *configRun) activationFailed(ctx context.Context, err error, state secretsync.State) ctrl.Result { + if len(state.Withdrawn()) > 0 || len(state.Written()) == 0 { + return run.withdrawalFailed(ctx, err) + } + retry := run.engineRetry(err) + logf.FromContext(ctx).Error(err, "Failed to have the engine read the values the operator wrote into its secret store") + run.event(corev1.EventTypeWarning, "SecretRefreshFailed", "ReloadConfig", "%s", ended(fmt.Sprintf( + "The engine may still start a tool server or an executor run with a value a Secret held before the operator "+ + "wrote its current one into the engine's secret store (%s), which the revision it runs read before; the "+ + "operator tries again in %s: %s", strings.Join(state.Written(), ", "), retry, err))) + return ctrl.Result{RequeueAfter: retry} +} + +// storeChanges are the changes of the engine's secret store that the revision +// the engine runs has not read and reads once it activates again: names +// withdrawn that it lets go without, and names written that it references. +// A withdrawn name it references as required, or one its reading the operator +// does not know, it keeps until the next revision runs, and a written name it +// does not reference changes nothing it runs. +type storeChanges struct { + withdrawn, written []string +} + +// owedBy is what an activation of active again would read of the store +// changes state records. +func owedBy(state secretsync.State, active *engine.ActiveConfig) storeChanges { + letGo := state.LetGo(active.RevisionID) + references := storeNamesIn(active.Document) + return storeChanges{ + withdrawn: slices.DeleteFunc(state.Withdrawn(), func(name string) bool { return !slices.Contains(letGo, name) }), + written: slices.DeleteFunc(state.Written(), func(name string) bool { return !slices.Contains(references, name) }), + } +} + +func (c storeChanges) empty() bool { return len(c.withdrawn) == 0 && len(c.written) == 0 } + +// owedIn reports whether state still owes an activation for any of c. +func (c storeChanges) owedIn(state secretsync.State) bool { + withdrawn, written := state.Withdrawn(), state.Written() + return slices.ContainsFunc(c.withdrawn, func(name string) bool { return slices.Contains(withdrawn, name) }) || + slices.ContainsFunc(c.written, func(name string) bool { return slices.Contains(written, name) }) +} + +// reads is how the engine reads its store once a revision activates after c: +// without what was withdrawn, with what was written. +func (c storeChanges) reads() string { + var parts []string + if len(c.withdrawn) > 0 { + parts = append(parts, "without "+strings.Join(c.withdrawn, ", ")) + } + if len(c.written) > 0 { + parts = append(parts, "with the values now written under "+strings.Join(c.written, ", ")) + } + return strings.Join(parts, " and ") +} + +// why is reads with the reason for each change, which revision, the one +// activated again, lets go without or references. +func (c storeChanges) why(revision string) string { + var parts []string + if len(c.withdrawn) > 0 { + parts = append(parts, fmt.Sprintf("without %s, whose Secret or key no longer exists and which %s lets go "+ + "without", strings.Join(c.withdrawn, ", "), revision)) + } + if len(c.written) > 0 { + parts = append(parts, fmt.Sprintf("with the values now written under %s, which %s references and read "+ + "before they were written", strings.Join(c.written, ", "), revision)) + } + return strings.Join(parts, ", and ") +} + +// made is what the operator did to the store. +func (c storeChanges) made() string { + switch { + case len(c.written) == 0: + return fmt.Sprintf("removed %s from the engine's secret store", strings.Join(c.withdrawn, ", ")) + case len(c.withdrawn) == 0: + return fmt.Sprintf("wrote %s into the engine's secret store", strings.Join(c.written, ", ")) + } + return fmt.Sprintf("removed %s from and wrote %s into the engine's secret store", strings.Join(c.withdrawn, ", "), + strings.Join(c.written, ", ")) } // reactivate activates the engine's running revision again when state records -// withdrawn names it lets go without (State.LetGo), and returns state with -// every withdrawal settled: the engine has read its store without them, runs -// nothing that references them, or runs a revision that reads them as -// required, or one whose reading state does not record, which keeps what it -// read until the next one runs. active is the engine's document as this -// reconcile read it before it removed anything, nil to read it here. +// a change of the store it reads once it does (owedBy): a withdrawn name it +// lets go without (State.LetGo), or a value written under a name it +// references, which it read before the write: a rotated credential, or a +// value the sync wrote back. It returns state with every withdrawal settled: +// the engine has read its store without them, runs nothing that references +// them, or runs a revision that reads them as required, or one whose reading +// state does not record, which keeps what it read until the next one runs. A +// value written under a name the running revision does not reference stays +// owed, for the next write of the document to read. active is the engine's +// document as this reconcile read it, nil to read it here, and nil with +// run.unsealed set where the engine runs a revision stored without a seal, +// over which nothing is activated and state is returned as it is. +// +// It is the activation for every store change no write of the document is +// coming to settle: a document the engine refused, or one held for a Secret. +// A value rotated while the document was refused went unread by the revision +// the engine runs for as long as it was refused, while every node that +// restarted read the new one. // // The engine activates its revision again itself when a write or a removal // changes a value that revision reads (Crewlet's ADR-0070), and reload is // what the last such request answered (secretsync.Result.Reload), made by -// the engine call reloadAction (secretsync.Result.ReloadAction): where it landed, -// the engine read its store without every name removed, and it is the +// the engine call reloadAction (secretsync.Result.ReloadAction): where it +// landed, it came after every change the sync made, and it is the // activation; asking for another would activate the same document twice. // Where it is nil or failed, the operator asks for one (POST /config/reload). // // Neither names a revision: the engine stores again whichever revision the // fleet's activation pointer names when the request lands, built on it, and // a write landing inside the request fails it as a lost race. So the -// revision activated again is read back, and recorded as letting go what the -// one read lets go only where it is that document; otherwise the state goes -// on recording the one read, which is no longer the fleet's, and a Warning -// says so. +// revision it activates again is read back, and recorded as letting go what +// the one read lets go only where it is that document; otherwise the state +// goes on recording the one read, which is no longer the fleet's, and a +// Warning says so. Before the operator's request the changes are marked by +// the fleet's activation epoch, and state saved in ref (markChanges): a +// request that fails, or loses its race to another writer, is read back, and a revision +// the fleet runs above that epoch came after every change, whoever activated +// it, and settles them (reloadFailed). func (run *configRun) reactivate( ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, reload *engine.Reload, reloadAction string, - state secretsync.State, + state secretsync.State, ref types.NamespacedName, ) (secretsync.State, error) { - withdrawn := state.Withdrawn() - if len(withdrawn) == 0 { + if len(state.Withdrawn()) == 0 && len(state.Written()) == 0 { + return state, nil + } + if active == nil && run.unsealed != "" { + // The engine runs a revision it stores without a seal, which it + // neither shows nor activates again: no activation can be made over + // it, and asking for one is refused every time. The withdrawals stay + // owed, and the write that replaces it settles them, its activation + // reading the store without them (State.Activated). return state, nil } if active == nil { @@ -445,9 +621,8 @@ func (run *configRun) reactivate( if active == nil { return state.Reactivated(), nil } - letGo := state.LetGo(active.RevisionID) - held := slices.DeleteFunc(withdrawn, func(name string) bool { return !slices.Contains(letGo, name) }) - if len(held) == 0 { + owed := owedBy(state, active) + if owed.empty() { return state.Reactivated(), nil } // WHO ACTIVATED IT: the engine, at the request of the sync that made @@ -456,91 +631,209 @@ func (run *configRun) reactivate( if reload.Landed() { by = reloadAction } - byEngine := by != "" reloaded := engine.ApplyResult{} - if byEngine { + if by != "" { reloaded.RevisionID, reloaded.Epoch = reload.RevisionID, reload.Epoch } else { + // How the revision the request may store reads every name, should + // the engine not confirm activating it and a later read find it + // running (config_unconfirmed.go): as the one it activates again + // does. + own, ok := run.ownWrite(active) + ahead := ok && !state.Pends(own.AppliedHash) + if ahead { + state = state.Pending(own.AppliedHash, state.LetGoKeys(active.RevisionID)) + } var err error - if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(held)); err != nil { - return state, fmt.Errorf("activating revision %s again, so that the engine reads its secret store without %s: %w", - active.RevisionID, strings.Join(held, ", "), err) + if state, err = run.markChanges(ctx, eng, state, ref, ahead); err != nil { + return state, err + } + if reloaded, err = eng.ReloadConfig(ctx, reactivationSummary(owed)); err != nil { + return run.reloadFailed(ctx, eng, active, state, owed, + fmt.Errorf("activating revision %s again, so that the engine reads its secret store %s: %w", + active.RevisionID, owed.reads(), err)) } } now, err := activeConfig(ctx, eng) switch { case err != nil: - run.reactivatedUnconfirmed(active, reloaded, by, held, "the operator could not read it back: "+err.Error()) + run.reactivatedUnconfirmed(active, reloaded, by, owed, "the operator could not read it back: "+err.Error()) case now == nil || now.RevisionID != reloaded.RevisionID: - run.reactivatedUnconfirmed(active, reloaded, by, held, "the engine runs another revision since") + run.reactivatedUnconfirmed(active, reloaded, by, owed, "the engine runs another revision since") case !bytes.Equal(now.Document, active.Document): - run.reactivatedUnconfirmed(active, reloaded, by, held, + run.reactivatedUnconfirmed(active, reloaded, by, owed, "its document is not that one: somebody activated another revision before the request landed") default: - run.recordReactivation(active, reloaded, by, held) + run.recordReactivation(active, reloaded, by, owed, nil) return state.Activated(reloaded.RevisionID, state.LetGoKeys(active.RevisionID)), nil } - return state.Reactivated(), nil + // Whatever its document, the revision the request activated read the + // store after every write and withdrawal the state records. + return state.Reloaded(), nil +} + +// reloadFailed is state once a request to activate active again, so that the +// engine reads the store changes owed, failed with failed: settled as far as +// the fleet's revision, read afresh, activated after the changes, since +// nothing read before the request saw what activated since, and failed where +// any of owed is still owed. A revision the fleet's activation names above +// the epoch the changes were marked by (markChanges) came after every one of +// them, whoever activated it: the request itself, where it landed and its +// answer did not, or another writer, who won the race with it. Where its +// document is active's, the fleet runs that document again reading the store +// as owed says, as a reactivation the operator confirmed does, and it is +// recorded as one (recordReactivation, whose event says the request failed); +// otherwise it is a revision whose reading the operator does not know, which +// a Warning says, and the state goes on recording the one read, as +// reactivatedUnconfirmed leaves it. A request that failed outright leaves the +// fleet on active, which settles nothing. +// +// One whose revision the engine stored and could not confirm activating may +// yet land. That is decided from the answer, before anything is read back, +// since a read that fails, or finds the revision running with the fleet's +// pointer unreadable, leaves nothing else to say it: where active is the +// operator's, the revision is recorded as its own write of the same document, +// for the next read to settle as it settles one (reloadUnconfirmed), and the +// changes it was asked for after await it, so that finding it running +// settles them (State.Awaiting) however the pointer reads. Recorded only +// where the read back still found active, it was taken, once it landed, for a +// revision somebody else activated. +func (run *configRun) reloadFailed( + ctx context.Context, eng *engine.Client, active *engine.ActiveConfig, state secretsync.State, owed storeChanges, + failed error, +) (secretsync.State, error) { + if stored, ok := engine.UnconfirmedActivation(failed); ok { + run.reloadUnconfirmed(active, stored) + state = state.Awaiting(stored) + } + now, err := activeConfig(ctx, eng) + if err != nil || now == nil || now.RevisionID == active.RevisionID { + return state, failed + } + fleet, err := eng.Fleet(ctx) + if err != nil || fleet.Activation == nil || fleet.Activation.RevisionID != now.RevisionID { + return state, failed + } + settled := state.Ran(fleet.Activation.Epoch) + if owed.owedIn(settled) { + return state, failed + } + activation := engine.ApplyResult{RevisionID: now.RevisionID, Epoch: fleet.Activation.Epoch} + if bytes.Equal(now.Document, active.Document) { + run.recordReactivation(active, activation, "", owed, failed) + return settled.Adopted(now.RevisionID, state.LetGoKeys(active.RevisionID)), nil + } + run.event(corev1.EventTypeWarning, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( + "Did not activate revision %s again: revision %s, another document, activated as epoch %d after the operator "+ + "%s, and read the store %s; the operator takes it to read every value as required, as it does any revision "+ + "it has no record of activating, and the request failed: %s", + active.RevisionID, now.RevisionID, fleet.Activation.Epoch, owed.made(), owed.reads(), failed))) + return settled, nil } // reactivationSummary is the summary of the revision that activates the -// running document again without held. -func reactivationSummary(held []string) string { +// running document again to read the store changes owed. +func reactivationSummary(owed storeChanges) string { const shown = 4 - text := strings.Join(held[:min(len(held), shown)], ", ") - if len(held) > shown { - text += fmt.Sprintf(" and %d more", len(held)-shown) + list := func(names []string) string { + text := strings.Join(names[:min(len(names), shown)], ", ") + if len(names) > shown { + text += fmt.Sprintf(" and %d more", len(names)-shown) + } + return text + } + var parts []string + if len(owed.withdrawn) > 0 { + parts = append(parts, "without "+list(owed.withdrawn)+", whose Secret or key no longer exists") + } + if len(owed.written) > 0 { + parts = append(parts, "to read the values now written under "+list(owed.written)) } - return "crewlet-operator: activated again without " + text + ", whose Secret or key no longer exists" + return "crewlet-operator: activated again " + strings.Join(parts, ", and ") } // recordReactivation records the running document activated again as -// reloaded, without held, by the engine at the request of the sync's engine -// call by (WriteSecret, PutSecrets or DeleteSecret: a write or a removal of -// a value the revision reads), or at the operator's request, by empty. A revision the operator recorded +// reloaded, reading the store changes owed, by the engine at the request of +// the sync's engine call by (WriteSecret, PutSecrets or DeleteSecret: a write +// or a removal of a value the revision reads), or at the operator's request, +// by empty. A revision the operator recorded // as its own is replaced by the one that activates it again, with the same // content, so that the next one it applies does not take it for an edit -// somebody else made; one it did not write stays somebody else's. -func (run *configRun) recordReactivation(active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string) { - who, action := reactivatedBy(by) - run.event(corev1.EventTypeNormal, "Reactivated", action, - "%s revision %s again, unchanged, as revision %s (epoch %d), so that the engine reads its secret store "+ - "without %s: the Secret or key each stands for no longer exists, and the revision lets it go without its value.", - who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", ")) - recorded := run.company.Status.Config - if run.config != nil { - recorded = *run.config - } - if recorded.RevisionID != active.RevisionID { - return +// somebody else made; so is a write of the operator's the engine could not +// confirm activating and runs, whose record (status.config.unconfirmed) moves +// to the revision that activates it again, for the next reconcile that +// applies a document to confirm and report (config_unconfirmed.go); one it +// did not write stays somebody else's. failed is the error the operator's +// request answered where the engine was read back running the document again +// anyway (reloadFailed), nil where it answered the activation. +// +// reloaded is recorded once: a record status.config.unconfirmed already holds +// of it, a reload the engine stored and could not confirm activating +// (reloadUnconfirmed), goes, as the read back settled it. Kept, it was a +// second entry of one revision, which the API server refuses, or a revision +// the next reconcile confirmed a second time. +func (run *configRun) recordReactivation( + active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, owed storeChanges, failed error, +) { + if failed == nil { + who, action := reactivatedBy(by) + run.event(corev1.EventTypeNormal, "Reactivated", action, + "%s revision %s again, unchanged, as revision %s (epoch %d), so that the engine reads its secret store "+ + "%s.", who, active.RevisionID, reloaded.RevisionID, reloaded.Epoch, owed.why("the revision")) + } else { + run.event(corev1.EventTypeNormal, "Reactivated", "ReloadConfig", "%s", ended(fmt.Sprintf( + "The engine runs revision %s again, unchanged, as revision %s (epoch %d), activated after the operator %s, "+ + "so it reads the store %s; the operator's request to activate it again failed: %s", + active.RevisionID, reloaded.RevisionID, reloaded.Epoch, owed.made(), owed.why("the revision"), failed))) } + recorded := run.recordedConfig() config := recorded.DeepCopy() - now := metav1.Now() - config.RevisionID, config.Epoch, config.AppliedAt = reloaded.RevisionID, new(reloaded.Epoch), &now - run.config = config - if run.applied.kind == appliedInRevision && run.applied.revision == active.RevisionID { - run.applied.revision = reloaded.RevisionID + if i := unconfirmedIndex(config.Unconfirmed, reloaded.RevisionID); i >= 0 { + config.Unconfirmed = slices.Delete(config.Unconfirmed, i, i+1) + run.config = config + } + now := metav1.NewTime(run.clock()) + switch i := unconfirmedIndex(config.Unconfirmed, active.RevisionID); { + case recorded.RevisionID == active.RevisionID: + config.RevisionID, config.Epoch, config.AppliedAt = reloaded.RevisionID, new(reloaded.Epoch), &now + if run.applied.kind == appliedInRevision && run.applied.revision == active.RevisionID { + run.applied.revision = reloaded.RevisionID + } + case i >= 0: + // Confirmed only in the secret sync's state (settlePending), by a + // reconcile that holds its document: the record goes on to the + // revision that runs it now, which is the write's as much as the + // one it activated again was, rather than staying with one the + // next read finds overtaken. + config.Unconfirmed[i].RevisionID, config.Unconfirmed[i].WrittenAt = reloaded.RevisionID, &now + default: + return } + run.config = config } -// reactivatedUnconfirmed reports an activation of active again, without -// held, by the engine at the sync's engine call by or at the operator's -// request (by empty), -// that activated reloaded, which the operator could not confirm is active's -// document, for why: it takes reloaded to read every name as required, and a -// revision somebody else activated may have needed a value it removed for -// active, which let it go. +// reactivatedUnconfirmed reports an activation of active again, reading the +// store changes owed, by the engine at the sync's engine call by or at the +// operator's request (by empty), that activated reloaded, which the operator +// could not confirm is active's document, for why: it takes reloaded to read every +// name as required, and, where owed withdrew a name, a revision somebody else +// activated may have needed a value it removed for active, which let it go. func (run *configRun) reactivatedUnconfirmed( - active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, held []string, why string, + active *engine.ActiveConfig, reloaded engine.ApplyResult, by string, owed storeChanges, why string, ) { who, action := reactivatedBy(by) - run.event(corev1.EventTypeWarning, "Reactivated", action, - "%s the engine's document again as revision %s (epoch %d), so that the engine reads its secret store without "+ - "%s, whose Secret or key no longer exists and which revision %s lets go without; but the operator cannot confirm "+ - "that revision %s is revision %s's document, as %s. It takes revision %s to read every value as required, and "+ - "one that reads a value it removed so goes without what it starts with it until the Secret exists again.", - who, reloaded.RevisionID, reloaded.Epoch, strings.Join(held, ", "), active.RevisionID, reloaded.RevisionID, - active.RevisionID, why, reloaded.RevisionID) + // why ends with what the read back met, which may be the engine's own + // sentence: the sentence that quotes it is ended by ended. + activated := ended(fmt.Sprintf("%s the engine's document again as revision %s (epoch %d), so that the engine "+ + "reads its secret store %s; but the operator cannot confirm that revision %s is revision %s's document, as %s", + who, reloaded.RevisionID, reloaded.Epoch, owed.why("revision "+active.RevisionID), reloaded.RevisionID, + active.RevisionID, why)) + takes := fmt.Sprintf("It takes revision %s to read every value as required", reloaded.RevisionID) + if len(owed.withdrawn) > 0 { + takes += ", and one that reads a value it removed so goes without what it starts with it until the Secret " + + "exists again" + } + run.event(corev1.EventTypeWarning, "Reactivated", action, "%s %s.", activated, takes) } // reactivatedBy is who an activation again is said to be by, and the action diff --git a/internal/controller/config_withdraw_test.go b/internal/controller/config_withdraw_test.go index ccb705b7..7ec4d913 100644 --- a/internal/controller/config_withdraw_test.go +++ b/internal/controller/config_withdraw_test.go @@ -1,6 +1,7 @@ package controller import ( + "bytes" "context" "encoding/json" "errors" @@ -10,20 +11,21 @@ import ( "net/http" "net/http/httptest" "slices" + "strconv" "strings" "sync" "sync/atomic" "testing" + "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/yaml" @@ -42,11 +44,22 @@ import ( // storeEngine answers the engine's /health, /config, /config/revisions, // /fleet and /secrets routes from a revision history, the last of which is -// the fleet's, and a store, as the operator's client reads them. A document -// it is to refuse is refused as the engine refuses an invalid one. -// beforeReload, afterReload and afterApply, when set, run once, under its -// lock, just before a reload is answered and just after a reload or an apply -// is: somebody else's write landing between two of the operator's. +// the fleet's (none, an engine that runs no document yet), and a store, as the +// operator's client reads them. A write of the document is refused on the +// condition it names, as the engine refuses it, and so is every document while +// it is to refuse them, as refusal says; written are the revisions the writes +// it took stored, conditions the If-Match each of those was sent with, and +// refused counts the ones it refused on their condition. beforeReload, +// afterReload, beforeCheck, beforeWrite and afterApply, when set, run once, +// under its lock, just before a reload, a dry run or a write is answered and +// just after a reload or an apply is: somebody else's write landing between +// two of the operator's. activation is how the next write's activation goes, +// and inert the revisions writes stored whose activation has not landed +// (config_unconfirmed_test.go). minted counts the revisions it has named, +// active or not, so that no two share an id, and requests are the requests it +// answered since the last takeRequests, each its method and path. Its fleet +// answers the activation pointer beside its nodes: the fleet's revision at its +// place in the history, which is its epoch. // // A write into its store (PUT /secrets, PUT /secrets/{name}), and a removal // of a value (DELETE /secrets/{name}), re-activates the fleet's revision when its @@ -61,8 +74,14 @@ import ( // refused the fleet's, "" for one that has recorded no attempt and so not // said); runsNone names nodes beside those that refused the fleet's revision // and run no company, as a seat pod that started on nothing and refused it -// does; fleetDown makes GET /fleet unanswerable, and missing names the -// revisions GET /config/revisions does not find. +// does; fleetDown makes GET /fleet unanswerable, as an engine whose +// coordination store does not answer the read of its lease table +// (fleetUnreadable), and missing names the revisions GET /config/revisions +// does not find. configDown, secretsDown and reloadDown make it answer a read +// of the fleet's document, a write to the store, and a reload of the fleet's +// revision, as an engine that cannot decide who is asking does; a revision it +// cannot open answers what reads it as the engine does (unopened, +// config_unreadable_test.go). // // Each node's row says what it starts on as the engine's does: a data node // (dataNode) the revision it runs, and a seat node, whose store is scratch, @@ -70,12 +89,24 @@ import ( // in place of that, as a data node whose record of it the engine has not // moved yet to the revision it runs, "" for none. type storeEngine struct { - mu sync.Mutex - revisions []storeRevision - store map[string]string - refuse bool + mu sync.Mutex + revisions []storeRevision + minted int + written, conditions []string + refused int + store map[string]string + // refusal, when set, is how the engine refuses every PUT of the document, + // the dry run included: invalidDocument, a document it finds invalid, or + // for what only its own users can clear, heldSeat, a human seat somebody + // holds that the document takes away, or managedDocument, a token the + // engine does not let change the document. A reload it answers as ever: + // none of them refuses one. + refusal *engineRefusal + configDown, secretsDown bool + reloadDown bool beforeReload, afterReload func(e *storeEngine) - afterApply func(e *storeEngine) + beforeCheck func(e *storeEngine) + beforeWrite, afterApply func(e *storeEngine) reloadAnswer *map[string]any afterSecretsReload func(e *storeEngine) serving map[string]string @@ -83,6 +114,27 @@ type storeEngine struct { runsNone map[string]bool fleetDown bool missing map[string]bool + activation outcome + // reloadActivation is how the activation of the next reload goes, as + // activation is the next write's. + reloadActivation outcome + inert []storeRevision + requests []string + // beforeConfig and beforeFleet run once, before the engine answers the + // next read of the fleet's document or of its fleet: an activation + // somebody else makes in that moment. + beforeConfig, beforeFleet func(e *storeEngine) + // noActivation makes /fleet answer without its activation pointer, as + // an engine that could not read it for the answer does; appliedLag is + // how many epochs /health reports the node behind the latest. + noActivation bool + appliedLag int + // dryRunDown makes the engine answer a dry run as one that cannot + // decide it, a failure no wait of the reconcile's own clears. + dryRunDown bool + // afterPut, when set, runs once, under its lock, just after a value is + // written into the store. + afterPut func(e *storeEngine) } // dataNode reports whether node is a pod of the Company's data StatefulSet, @@ -95,38 +147,124 @@ func dataNode(node string) bool { // otherwise. const storeNode = "acme-data-0" +// mint names the engine's next revision: one past every revision it has +// named, active or not, and past every one a test put in its history by +// hand, so that no two share an id. The caller holds e's lock. +func (e *storeEngine) mint() string { + e.minted = max(e.minted, len(e.revisions)+len(e.inert)) + 1 + return fmt.Sprintf("r%d", e.minted) +} + +// down reports whether r is a request configDown, secretsDown, reloadDown or +// dryRunDown makes the engine refuse: a read of the fleet's document, a write +// to the store, one name's or a batch's, a reload of the fleet's revision, or +// a dry run. +func (e *storeEngine) down(r *http.Request) bool { + if r.Method == http.MethodGet { + return r.URL.Path == configRoute && e.configDown + } + if r.URL.Path == reloadRoute { + return e.reloadDown + } + if r.URL.Path == configRoute && dryRun(r) { + return e.dryRunDown + } + return (r.URL.Path == secretsRoute || strings.HasPrefix(r.URL.Path, secretsRoute+"/")) && e.secretsDown +} + +// cannotDecide is the engine's answer to a request it cannot decide who is +// asking: its request guard's, which cannot read who presented the credential +// (crewlet's internal/api/auth, Caller). The authority table's own unknown +// never meets the operator's tokens on /config or /secrets, since they hold +// admin and admin is decided before the chart. +func cannotDecide(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"identity_unavailable",` + + `"message":"Can't check who you are right now. Try again in a moment."}`)) +} + +// fleetUnreadable is the engine's answer to GET /fleet when its coordination +// store does not answer the read of the lease table, as crewlet#142 words it: +// the unavailable code's own sentence, the condition's as the detail, and the +// five seconds of the health tick that clears it. +func fleetUnreadable(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"unavailable",` + + `"message":"Can't answer right now. Try again in a moment, or ask another node.",` + + `"detail":"This node couldn't reach the rest of the fleet. Try again in a moment."}`)) +} + type storeRevision struct { id, summary string document []byte + // unopened, when set, is the engine's answer to a request that reads the + // revision, which it cannot open (config_unreadable_test.go): nil for + // one it opens. + unopened func(revision string) string } // newStoreEngine runs the engine with document as its active revision r1. func newStoreEngine(t *testing.T, document []byte) (*storeEngine, *engine.Client) { t.Helper() - e := &storeEngine{revisions: []storeRevision{{id: "r1", document: document}}, store: map[string]string{}} + e := &storeEngine{revisions: []storeRevision{{id: "r1", document: document}}, minted: 1, store: map[string]string{}} server := httptest.NewServer(http.HandlerFunc(e.serve)) t.Cleanup(server.Close) return e, engine.New(server.URL, "token") } +// engineRefusal is an answer the engine refuses a document with, as it +// writes it. +type engineRefusal struct { + status int + body string +} + +func (refusal *engineRefusal) answer(w http.ResponseWriter) { + answerRaw(w, refusal.status, refusal.body) +} + +var ( + // invalidDocument is the engine's refusal of a document it finds + // invalid. + invalidDocument = &engineRefusal{status: http.StatusBadRequest, body: `{"error":"validation_error",` + + `"detail":"refused","problems":[{"path":"roles","kind":"invalid","message":"refused"}]}`} + // heldSeat is the engine's refusal of a document that takes away a + // human seat a person holds, as crewlet#142 words it. + heldSeat = &engineRefusal{status: http.StatusConflict, body: `{"error":"seat_held",` + + `"detail":"This change removes human seats that somebody still holds: founder (held by p-1).",` + + `"held":{"founder":{"person":"p-1","kind":"person","stage":"active"}},` + + `"hint":"Move each person to another human seat (PATCH /iam/people/{person} with seat) or remove them, ` + + `unbind a service account (PATCH /iam/people/{person} with no seat), cancel an open invitation ` + + `(DELETE /iam/invitations/{id}), each an admin's gesture, then send the write again.",` + + `"message":"This change removes a seat someone holds or is invited to. Move or remove them, or cancel ` + + `the invitation, first. The detail says who."}`} + // managedDocument is the engine's refusal of a write by a token that + // api.auth.company_writers does not name. + managedDocument = &engineRefusal{status: http.StatusForbidden, body: `{"error":"config_managed",` + + `"managed_by":["ci"],"detail":"the company document is managed by ci"}`} +) + func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { e.mu.Lock() defer e.mu.Unlock() + e.requests = append(e.requests, r.Method+" "+r.URL.Path) body, _ := io.ReadAll(r.Body) answer := func(status int, v any) { answerJSON(w, status, v) } - if r.Method == http.MethodPost && r.URL.Path == "/config/reload" && e.beforeReload != nil { - hook := e.beforeReload - e.beforeReload = nil - hook(e) - } - active := e.revisions[len(e.revisions)-1] + e.before(r) + active := e.active() activate := func(document []byte) storeRevision { - revision := storeRevision{id: fmt.Sprintf("r%d", len(e.revisions)+1), document: document, summary: r.Header.Get("X-Summary")} + revision := storeRevision{id: e.mint(), document: document, summary: r.Header.Get("X-Summary")} e.revisions = append(e.revisions, revision) return revision } - activated := func(revision storeRevision) { - answer(http.StatusCreated, map[string]any{revisionIDField: revision.id, "epoch": len(e.revisions)}) + activated := func(revision storeRevision) { e.activated(answer, revision) } + if e.down(r) { + cannotDecide(w) + return } if r.Method == http.MethodGet && e.serveServing(w, r.URL.Path, active) { return @@ -135,33 +273,24 @@ func (e *storeEngine) serve(w http.ResponseWriter, r *http.Request) { return } switch { + case active.unopened != nil && readsActive(r, active): + answerRaw(w, http.StatusConflict, active.unopened(active.id)) case r.Method == http.MethodGet && r.URL.Path == "/health": - answer(http.StatusOK, map[string]any{"status": "ok", "configured": true, "posture": "serving", "applied_epoch": len(e.revisions)}) + posture := engine.PostureServe + if e.appliedLag > 0 { + posture = engine.PostureWait + } + answer(http.StatusOK, map[string]any{"status": "ok", "configured": active.id != "", "posture": posture, + "applied_epoch": len(e.revisions) - e.appliedLag}) + case r.Method == http.MethodGet && r.URL.Path == configRoute && active.id == "": + answerRaw(w, http.StatusNotFound, `{"error":"no_active_revision"}`) case r.Method == http.MethodGet && r.URL.Path == configRoute: - w.Header().Set("ETag", `"`+active.id+`"`) - _, _ = w.Write(active.document) - case r.Method == http.MethodPut && r.URL.Path == configRoute && e.refuse: - answer(http.StatusUnprocessableEntity, map[string]any{"error": "validation_error", "detail": "refused", - "problems": []map[string]string{{"path": "roles", "kind": "invalid", "message": "refused"}}}) - case r.Method == http.MethodPut && r.URL.Path == configRoute && r.URL.Query().Get("dry_run") == "true": - answer(http.StatusOK, map[string]any{"base_revision_id": active.id}) + e.readActive(w, active) case r.Method == http.MethodPut && r.URL.Path == configRoute: - applied := activate(body) - if e.afterApply != nil { - hook := e.afterApply - e.afterApply = nil - hook(e) - } - activated(applied) - case r.Method == http.MethodPost && r.URL.Path == "/config/reload": - reloaded := activate(active.document) - if e.afterReload != nil { - hook := e.afterReload - e.afterReload = nil - hook(e) - } - activated(reloaded) - case r.Method == http.MethodGet && r.URL.Path == "/secrets": + e.put(w, r, body, answer) + case r.Method == http.MethodPost && r.URL.Path == reloadRoute: + e.reload(w, active, r.Header.Get("X-Summary"), activate, activated) + case r.Method == http.MethodGet && r.URL.Path == secretsRoute: listing := make([]map[string]string, 0, len(e.store)) for _, name := range slices.Sorted(maps.Keys(e.store)) { listing = append(listing, map[string]string{"name": name, "updated_by": operatorLogin, "operator_id": operatorLogin}) @@ -181,8 +310,16 @@ func (e *storeEngine) serveSecretWrite( ) bool { reads := func(name string) bool { return strings.Contains(string(active.document), "${"+name+"}") } name, isSecret := strings.CutPrefix(r.URL.Path, "/secrets/") + if active.unopened != nil && (r.URL.Path == secretsRoute || isSecret) { + // The engine asks which paths of the fleet's revision read each + // name before it writes or removes one (crewlet's secretsapi, + // config.Reads), which opens the revision: over one it cannot + // open, the request fails whole, as an internal error. + answerJSON(w, http.StatusInternalServerError, refusal("internal_error", "Something went wrong on our side.")) + return true + } switch { - case r.Method == http.MethodPut && r.URL.Path == "/secrets": + case r.Method == http.MethodPut && r.URL.Path == secretsRoute: var batch struct { Values map[string]string `json:"values"` } @@ -201,6 +338,11 @@ func (e *storeEngine) serveSecretWrite( read = append(read, name) } } + if e.afterPut != nil { + hook := e.afterPut + e.afterPut = nil + hook(e) + } var reload any if len(read) > 0 { reload = e.reactivate(activate, active, "reload: secrets "+strings.Join(read, ", ")+" written") @@ -208,6 +350,11 @@ func (e *storeEngine) serveSecretWrite( answerJSON(w, http.StatusOK, map[string]any{"names": names, readByField: readBy, reloadField: reload}) case r.Method == http.MethodPut && isSecret: e.store[name] = string(body) + if e.afterPut != nil { + hook := e.afterPut + e.afterPut = nil + hook(e) + } readBy, reload := []string{}, any(nil) if reads(name) { readBy = []string{readPath} @@ -270,8 +417,7 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st revisionID, isRevision := strings.CutPrefix(path, "/config/revisions/") switch { case path == fleetRoute && e.fleetDown: - w.Header().Set("Retry-After", "1") - answerJSON(w, http.StatusServiceUnavailable, refusal("unavailable", "coordination is unreachable")) + fleetUnreadable(w) case path == fleetRoute: serving := e.serving if serving == nil { @@ -311,13 +457,20 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st for _, node := range slices.Sorted(maps.Keys(e.runsNone)) { nodes = append(nodes, row(node, len(e.revisions), active.id, engine.ApplyError, "")) } - answerJSON(w, http.StatusOK, map[string]any{"this_node": storeNode, "nodes": nodes, "target_epoch": len(e.revisions)}) + answerJSON(w, http.StatusOK, map[string]any{"this_node": storeNode, "nodes": nodes, "target_epoch": len(e.revisions), + "activation": e.pointer()}) case isRevision: i := slices.IndexFunc(e.revisions, func(rev storeRevision) bool { return rev.id == revisionID }) if i < 0 || e.missing[revisionID] { answerJSON(w, http.StatusNotFound, refusal("not_found", "no revision "+revisionID)) return true } + if unopened := e.revisions[i].unopened; unopened != nil { + // The engine opens the revision to answer it, and refuses one + // it cannot open as it refuses a read of the document. + answerRaw(w, http.StatusConflict, unopened(e.revisions[i].id)) + return true + } payload, err := yaml.YAMLToJSON(e.revisions[i].document) if err != nil { answerJSON(w, http.StatusInternalServerError, refusal("internal", err.Error())) @@ -331,6 +484,75 @@ func (e *storeEngine) serveServing(w http.ResponseWriter, path string, active st return true } +// reload answers a request, summarised as summary, to activate active again as +// reloadActivation says it goes: stored and never activated (pending), +// activated with the answer lost (landed), or activated. +func (e *storeEngine) reload( + w http.ResponseWriter, active storeRevision, summary string, activate func([]byte) storeRevision, + activated func(storeRevision), +) { + if e.reloadActivation == pending { + stored := storeRevision{id: e.mint(), document: active.document, summary: summary} + e.inert = append(e.inert, stored) + answerLost(w, stored.id, active.id) + return + } + reloaded := activate(active.document) + if e.afterReload != nil { + hook := e.afterReload + e.afterReload = nil + hook(e) + } + if e.reloadActivation == landed { + answerLost(w, reloaded.id, active.id) + return + } + activated(reloaded) +} + +// before runs the hook, if one is set, that r's route takes before the engine +// answers it, once: beforeReload, beforeConfig or beforeFleet. +func (e *storeEngine) before(r *http.Request) { + var hook *func(e *storeEngine) + switch { + case r.Method == http.MethodPost && r.URL.Path == reloadRoute: + hook = &e.beforeReload + case r.Method == http.MethodGet && r.URL.Path == configRoute: + hook = &e.beforeConfig + case r.Method == http.MethodGet && r.URL.Path == fleetRoute: + hook = &e.beforeFleet + default: + return + } + if run := *hook; run != nil { + *hook = nil + run(e) + } +} + +// activationAnswer is the activation pointer as GET /fleet answers it. +type activationAnswer struct { + RevisionID string `json:"revision_id"` + Epoch int `json:"epoch"` +} + +// pointer is the activation pointer the engine answers beside its nodes: the +// fleet's revision at the epoch it activated as, which is its place in the +// history; nil for an engine with no revision, and where noActivation says the +// engine could not read it. The caller holds e's lock. +func (e *storeEngine) pointer() *activationAnswer { + running := e.active().id + if running == "" || e.noActivation { + return nil + } + return &activationAnswer{RevisionID: running, Epoch: len(e.revisions)} +} + +// dryRun reports whether r asks only for a check of the document. +func dryRun(r *http.Request) bool { + return r.URL.Query().Get("dry_run") == "true" +} + // answerJSON writes v as the JSON answer, with status. func answerJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json") @@ -343,6 +565,123 @@ func refusal(code, detail string) map[string]any { return map[string]any{"error": code, "detail": detail} } +// readActive answers a read of the fleet's document, active: the document and +// its ETag. +func (e *storeEngine) readActive(w http.ResponseWriter, active storeRevision) { + w.Header().Set("ETag", `"`+active.id+`"`) + _, _ = w.Write(active.document) +} + +// active is the revision the engine runs, the zero revision for none. The +// caller holds e's lock. +func (e *storeEngine) active() storeRevision { + if n := len(e.revisions); n > 0 { + return e.revisions[n-1] + } + return storeRevision{} +} + +// answerRaw answers with status and body, the engine's JSON as it words it. +func answerRaw(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) +} + +// put answers PUT /config, a write of the document whose body is body or its +// dry run, as the engine does: a document it is to refuse is refused as refusal +// says; a write is refused when its condition (If-Match, If-None-Match) no +// longer holds, and otherwise stored and activated, the engine's answer to the +// activation lost as activation says. The caller holds e's lock. +func (e *storeEngine) put(w http.ResponseWriter, r *http.Request, body []byte, answer func(int, any)) { + if e.refusal != nil { + e.refusal.answer(w) + return + } + if dryRun(r) { + if hook := e.beforeCheck; hook != nil { + e.beforeCheck = nil + hook(e) + } + answer(http.StatusOK, map[string]any{"base_revision_id": e.active().id}) + return + } + if hook := e.beforeWrite; hook != nil { + e.beforeWrite = nil + hook(e) + } + running := e.active().id + if base := strings.Trim(r.Header.Get("If-Match"), `"`); base != "" && base != running { + // A failed precondition: the write named in If-Match a revision + // that no longer runs, which the engine answers 412 whether it + // found it so before the write or lost the race to another after, + // naming the revision that runs and the base as the header gave it. + e.refused++ + answerJSON(w, http.StatusPreconditionFailed, struct { + Code string `json:"error"` + Current string `json:"current_revision_id"` + Base string `json:"your_base"` + }{"revision_advanced", running, r.Header.Get("If-Match")}) + return + } + if r.Header.Get("If-None-Match") == "*" && running != "" { + e.refused++ + answerRaw(w, http.StatusPreconditionFailed, fmt.Sprintf(`{"error":"already_configured","current_revision_id":%q}`, running)) + return + } + revision := storeRevision{id: e.mint(), document: body, summary: r.Header.Get("X-Summary")} + e.written = append(e.written, revision.id) + e.conditions = append(e.conditions, strings.Trim(r.Header.Get("If-Match"), `"`)) + switch e.activation { + case confirmed: + e.revisions = append(e.revisions, revision) + if e.afterApply != nil { + hook := e.afterApply + e.afterApply = nil + hook(e) + } + e.activated(answer, revision) + return + case landed: + e.revisions = append(e.revisions, revision) + case pending: + e.inert = append(e.inert, revision) + } + answerLost(w, revision.id, strings.Trim(r.Header.Get("If-Match"), `"`)) +} + +// activated answers a write or a reload that activated revision, the +// newest, at an epoch per activation. The caller holds e's lock. +func (e *storeEngine) activated(answer func(int, any), revision storeRevision) { + answer(http.StatusCreated, map[string]any{"revision_id": revision.id, "epoch": len(e.revisions)}) +} + +// writes are the revisions the engine's writes of the document stored, and +// how many it refused on their condition. +func (e *storeEngine) writes() ([]string, int) { + e.mu.Lock() + defer e.mu.Unlock() + return slices.Clone(e.written), e.refused +} + +// takeRequests returns the requests the engine answered since the last +// takeRequests, each its method and path. +func (e *storeEngine) takeRequests() []string { + e.mu.Lock() + defer e.mu.Unlock() + requests := e.requests + e.requests = nil + return requests +} + +// writtenOn is the If-Match each write the engine stored was sent with, as +// writes names them, "" for none. +func (e *storeEngine) writtenOn() []string { + e.mu.Lock() + defer e.mu.Unlock() + return slices.Clone(e.conditions) +} + // ids are the engine's revisions, oldest first. func (e *storeEngine) ids() []string { e.mu.Lock() @@ -361,6 +700,18 @@ func (e *storeEngine) summaryOf() string { return e.revisions[len(e.revisions)-1].summary } +// summaryOfRevision is the summary of revision id, "" for none. +func (e *storeEngine) summaryOfRevision(id string) string { + e.mu.Lock() + defer e.mu.Unlock() + for _, r := range e.revisions { + if r.id == id { + return r.summary + } + } + return "" +} + func (e *storeEngine) holds(name string) bool { e.mu.Lock() defer e.mu.Unlock() @@ -372,18 +723,21 @@ func (e *storeEngine) holds(name string) bool { // edit through the engine's own API: the caller holds e's lock. func (e *storeEngine) activateOther(document []byte) { e.revisions = append(e.revisions, storeRevision{ - id: fmt.Sprintf("r%d", len(e.revisions)+1), document: document, summary: "somebody else's", + id: e.mint(), document: document, summary: "somebody else's", }) } -// configRoute is the engine's route of the company document, fleetRoute that -// of its fleet, and dataRole and seatsRole the roles of a data pod's node and -// a seat pod's there. +// configRoute is the engine's route of the company document, reloadRoute the +// one that activates it again, fleetRoute that of its fleet, secretsRoute +// that of its secret store, and dataRole and seatsRole the roles of a data +// pod's node and a seat pod's there. const ( - configRoute = "/config" - fleetRoute = "/fleet" - dataRole = "data" - seatsRole = "seats" + configRoute = "/config" + reloadRoute = "/config/reload" + fleetRoute = "/fleet" + secretsRoute = "/secrets" + dataRole = "data" + seatsRole = "seats" ) // running is the revision the engine runs when a withdrawal's reconcile @@ -482,13 +836,6 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { } fe, eng := newStoreEngine(t, served) - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - if err := crewletaiv1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } company := run1.company.DeepCopy() company.Spec.Mission = "" company.UID = "6f1c1a5e-0000-4000-8000-000000000002" @@ -497,7 +844,7 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { Type: crewletaiv1alpha1.ConditionRuntimeReady, Status: metav1.ConditionTrue, Reason: crewletaiv1alpha1.ReasonValid, }) cluster := interceptor.NewClient( - fake.NewClientBuilder().WithScheme(scheme).WithObjects(company, tavily, npm).WithStatusSubresource(company).Build(), + fakeClient().WithObjects(company, tavily, npm).WithStatusSubresource(company).Build(), interceptor.Funcs{Apply: func(ctx context.Context, c client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { if n := w.stateWrites.Add(1); n == w.failWrite.Load() { return errStateWrite @@ -512,18 +859,19 @@ func newWithdrawal(t *testing.T, r running) *withdrawal { if err != nil { t.Fatal(err) } - var state secretsync.State - if r.recorded != "" { - state = state.Activated(r.recorded, runningDoc.Optional) - } // Pushed before r1 activated, so that r1 read them: nothing is // re-activated for them. fe.reloadAnswer = new(map[string]any(nil)) - state, _, err = secretsync.Sync(ctx, eng, present, state, operatorLogin) + state, _, err := secretsync.Sync(ctx, eng, present, secretsync.State{}, operatorLogin, discardState) if err != nil { t.Fatal(err) } fe.reloadAnswer = nil + // r1 activated after both values were pushed, so it read them. + state = state.Reloaded() + if r.recorded != "" { + state = state.Activated(r.recorded, runningDoc.Optional) + } stateRef := types.NamespacedName{Namespace: company.Namespace, Name: secretSyncStateName(company)} if err := secretsync.SaveState(ctx, cluster, stateRef, company, state); err != nil { t.Fatal(err) @@ -570,6 +918,15 @@ func (w *withdrawal) statusWritten() *crewletaiv1alpha1.Company { return company } +// ref is where the operator keeps the secret sync's state of w's Company. +func (w *withdrawal) ref() types.NamespacedName { + return types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} +} + +// discardState is a save that takes every State secretsync.Sync hands it +// ahead of a write, and keeps none: the test saves what Sync returns itself. +func discardState(secretsync.State) error { return nil } + // state is the secret sync's state as the reconcile saved it. func (w *withdrawal) state(t *testing.T) secretsync.State { t.Helper() @@ -614,6 +971,26 @@ func (w *withdrawal) enforced(t *testing.T) (ctrl.Result, error) { return w.run.enforce(ctx, w.client, *w.run.desired, active) } +// rotatedValue is the value rotate gives the required Secret. +const rotatedValue = "tvly-rotated" + +// rotate gives the Secret behind the required value a new one, as somebody +// rotating a credential does: the next sync writes it into the engine's +// store, and what the engine runs reads it only once a revision activates +// after it. +func (w *withdrawal) rotate(t *testing.T) { + t.Helper() + ctx := context.Background() + rotated := &corev1.Secret{} + if err := w.cluster.Get(ctx, types.NamespacedName{Namespace: "ns", Name: requiredSecret}, rotated); err != nil { + t.Fatal(err) + } + rotated.Data[requiredKey] = []byte(rotatedValue) + if err := w.cluster.Update(ctx, rotated); err != nil { + t.Fatal(err) + } +} + // told reports whether the reconcile recorded a Normal event of reason, and // warned whether a Warning one. func told(recorded []string, reason string) bool { @@ -807,7 +1184,7 @@ func TestARevisionThatLetsAValueGoGoesWithoutItWhateverTheNextDocumentReads(t *t t.Fatal(err) } w.run.doc = doc - w.engine.refuse = true + w.engine.refusal = invalidDocument if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -845,8 +1222,10 @@ func TestARevisionThatLetsAValueGoGoesWithoutItWhateverTheNextDocumentReads(t *t // the running revision lets it go too, the operator's own or the document it // renders, it goes before the apply, and the engine activates the running // revision again at its removal, which reads it: that is the document the -// operator renders, so it is the one activation it takes, recorded as the -// operator's apply, and nothing is dry-run that the engine could refuse. +// operator renders, which the engine runs with every value, so it is the one +// activation it takes, the withdrawal it owes (Reactivated), recorded in place +// of the revision before it, and nothing is dry-run that the engine could +// refuse. func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { for _, tc := range []struct { running @@ -856,8 +1235,8 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { revisions []string kept bool // byEngine is the engine's re-activation at the removal, the - // document being the one the engine runs, taken for the apply - // before anything is dry-run. + // document being the one the engine runs, taken for the withdrawal + // it owes before anything is dry-run. byEngine bool }{ {running: runningRequires, revisions: []string{"r1", "r2", "r3"}}, @@ -874,7 +1253,9 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { } t.Run(name, func(t *testing.T) { w := newWithdrawal(t, tc.running) - w.engine.refuse = tc.refused + if tc.refused { + w.engine.refusal = invalidDocument + } if err := w.enforce(t); err != nil { t.Fatal(err) } @@ -906,11 +1287,13 @@ func TestTheNextDocumentLetsGoWhatTheRunningRevisionRequired(t *testing.T) { t.Errorf("the state lets %v go for %s, owes %v and retains %v, want the registry, nothing and nothing", state.LetGo(last), last, state.Withdrawn(), state.Retained()) } - reactivated := len(tc.revisions) == 3 || tc.refused && !tc.byEngine - if told(recorded, "Reactivated") != reactivated { - t.Errorf("events %q: Reactivated told %t, want %t", recorded, !reactivated, reactivated) + // Every revision that lets the registry go is activated again + // without it: by the operator after the next document, or by the + // engine at the removal where the running one lets it go. + if !told(recorded, "Reactivated") { + t.Errorf("events %q, want Reactivated", recorded) } - if reactivated && !strings.Contains(w.engine.summaryOf(), w.registry) { + if !strings.Contains(w.engine.summaryOf(), w.registry) { t.Errorf("revision %s is summarised %q, which does not name the registry", last, w.engine.summaryOf()) } if (!tc.refused || tc.byEngine) && @@ -936,16 +1319,18 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { for _, tc := range []struct { name string // fails is the write of the state in the first reconcile that - // fails: 1 before the apply, 2 its record, 3 after the deletion. + // fails: 1 the sync's before the apply, 2 the reading of the + // document saved before the write, 3 the apply's record, 4 after + // the deletion. fails int32 // held is whether the next reconcile's document is another one, // held for a Secret, so that only the record says how the revision // the engine runs reads the value. held bool }{ - {name: "the record of the apply is lost", fails: 2}, - {name: "the record of the deletion is lost", fails: 3}, - {name: "the record of the deletion is lost, and the next document is held", fails: 3, held: true}, + {name: "the record of the apply is lost", fails: 3}, + {name: "the record of the deletion is lost", fails: 4}, + {name: "the record of the deletion is lost, and the next document is held", fails: 4, held: true}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() @@ -958,13 +1343,13 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { // r2 reads, is removed: a record of the deletion lost is one of // that activation lost too. first := []string{"r1", "r2"} - if tc.fails == 3 { + if tc.fails == 4 { first = append(first, "r3") } if got, deleted := w.engine.ids(), !w.engine.holds(w.registry); !slices.Equal(got, first) || - deleted != (tc.fails == 3) { + deleted != (tc.fails == 4) { t.Fatalf("the revisions are %v and the registry is deleted: %t, want %v, and deleted: %t", - got, deleted, first, tc.fails == 3) + got, deleted, first, tc.fails == 4) } w.failWrite.Store(0) @@ -996,7 +1381,7 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { // is one whose reading the operator does not know, which needs no // activation for a value it already went without. want := []string{"r1", "r2", "r3"} - if tc.fails == 3 && !tc.held { + if tc.fails == 4 && !tc.held { want = append(want, "r4") } last := want[len(want)-1] @@ -1008,7 +1393,7 @@ func TestAnApplyIsRecordedBeforeAnythingIsDeletedAfterIt(t *testing.T) { if len(state.Withdrawn()) != 0 { t.Errorf("the state owes %v, want nothing", state.Withdrawn()) } - if letsGo := tc.fails != 3 || !tc.held; letsGo != slices.Equal(state.LetGo(last), []string{w.registry}) { + if letsGo := tc.fails != 4 || !tc.held; letsGo != slices.Equal(state.LetGo(last), []string{w.registry}) { t.Errorf("the state lets %v go for %s, want the registry: %t", state.LetGo(last), last, letsGo) } }) @@ -1060,10 +1445,14 @@ func TestAReloadIsRecordedOnlyForTheDocumentItRead(t *testing.T) { if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3"}) { t.Fatalf("the revisions are %v, want r1, r2 and r3, the reload %s", got, reload) } + // The reload, whatever its document, activated after every + // value the operator wrote too, the one the fixture pushed. state := w.state(t) - if state.Records(reload) || len(state.LetGo(reload)) != 0 || len(state.Withdrawn()) != 0 { - t.Errorf("the state records %s: %t, letting %v go, and owes %v; want %s unknown and nothing owed", - reload, state.Records(reload), state.LetGo(reload), state.Withdrawn(), reload) + if state.Records(reload) || len(state.LetGo(reload)) != 0 || len(state.Withdrawn()) != 0 || + len(state.Written()) != 0 { + t.Errorf("the state records %s: %t, letting %v go, and owes %v withdrawn and %v written; "+ + "want %s unknown and nothing owed", + reload, state.Records(reload), state.LetGo(reload), state.Withdrawn(), state.Written(), reload) } if !warned(recorded, "Reactivated") || told(recorded, "Reactivated") { t.Errorf("events %q, want a Warning Reactivated", recorded) @@ -1131,14 +1520,14 @@ func TestReactivateRepeatsOnlyARevisionThatLetsTheValueGo(t *testing.T) { t.Fatal(err) } // Removed by a sync that kept nothing for the running revision. - state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) if err != nil || !slices.Equal(state.Withdrawn(), []string{w.registry}) { t.Fatalf("the sync withdrew %v (%v), want the registry", state.Withdrawn(), err) } if res.Reload.Landed() != byEngine { t.Fatalf("the sync reports the engine's re-activation %+v", res.Reload) } - next, err := w.run.reactivate(ctx, w.client, active, res.Reload, res.ReloadAction, state) + next, err := w.run.reactivate(ctx, w.client, active, res.Reload, res.ReloadAction, state, w.ref()) if err != nil { t.Fatal(err) } @@ -1190,13 +1579,13 @@ func TestAReactivationByTheEngineNamesTheRequestThatMadeIt(t *testing.T) { if err != nil { t.Fatal(err) } - state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) if err != nil || !res.Reload.Landed() { t.Fatalf("the sync = (%+v, %v), want the engine's re-activation", res.Reload, err) } recorder := &actionRecorder{} w.run.Recorder = recorder - if _, err := w.run.reactivate(ctx, w.client, active, res.Reload, action, state); err != nil { + if _, err := w.run.reactivate(ctx, w.client, active, res.Reload, action, state, w.ref()); err != nil { t.Fatal(err) } if !slices.Contains(recorder.actions, "Reactivated "+action) { @@ -1205,3 +1594,835 @@ func TestAReactivationByTheEngineNamesTheRequestThatMadeIt(t *testing.T) { }) } } + +// TestARefusedDocumentComesBackWhenItsWithdrawalReportSays holds a document +// the engine refused, after which the activation owed could not be made, to +// the retry the SecretWithdrawalFailed event promises: here the engine's own +// two seconds. It came back ten seconds on whatever the event said, and a +// revision the engine cannot open, which the event says is looked at again +// at the resync interval, ten seconds on as well. +func TestARefusedDocumentComesBackWhenItsWithdrawalReportSays(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // r1 lets the registry go, and an earlier sync withdrew it: an + // activation without it is owed. + read := w.resolve(t) + read.Keep("r1", w.registry) + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + owed, _, err := secretsync.Sync(ctx, w.client, read, w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, ref, w.run.company, owed); err != nil { + t.Fatal(err) + } + // The dry run is refused, and the activation owed then meets an engine + // that cannot say who is asking, for two seconds. + w.engine.refusal = invalidDocument + w.engine.reloadDown = true + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Warning SecretWithdrawalFailed ") }) + if i < 0 || !strings.Contains(recorded[i], "tries again in 2s") || result.RequeueAfter != 2*time.Second { + t.Errorf("the next look is %s on and events %q; want SecretWithdrawalFailed promising 2s, and 2s", + result.RequeueAfter, recorded) + } +} + +// TestADocumentTheEngineHoldsStillSettlesTheStore pins that a document the +// engine refuses for what only its own users can clear, a human seat somebody +// holds or a token it does not let change the document, settles what the +// store owes as a document it refuses as invalid does: the running revision is +// activated again without a value whose Secret was deleted, and the refusal is +// reported and looked at again ten seconds on, or sooner where an activation +// that failed says so. It went back as an error, which nothing settled, so for +// as long as the seat was held the engine went on starting tool servers and +// executor runs with the deleted Secret's value. +func TestADocumentTheEngineHoldsStillSettlesTheStore(t *testing.T) { + for _, tc := range []struct { + refusal *engineRefusal + reason string + reloadDown bool + }{ + {refusal: heldSeat, reason: crewletaiv1alpha1.ReasonSeatHeld}, + {refusal: managedDocument, reason: crewletaiv1alpha1.ReasonWriteRefused}, + {refusal: heldSeat, reason: crewletaiv1alpha1.ReasonSeatHeld, reloadDown: true}, + } { + name := tc.reason + if tc.reloadDown { + name += ", and the activation owed fails" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // A rotated value is the change the engine is asked to apply: + // its own re-activation at the write fails, so the operator + // writes the document. + w.rotate(t) + w.engine.mu.Lock() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.refusal, w.engine.reloadDown = tc.refusal, tc.reloadDown + w.engine.mu.Unlock() + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + if got := w.run.configApplied; got == nil || got.Reason != tc.reason { + t.Errorf("ConfigApplied = %+v, want %s", got, tc.reason) + } + if tc.reloadDown { + recorded := drained(w.events) + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretWithdrawalFailed ") && strings.Contains(e, "tries again in 2s") + }) || result.RequeueAfter != 2*time.Second { + t.Errorf("the next look is %s on and events %q; want SecretWithdrawalFailed promising 2s, and 2s", + result.RequeueAfter, recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Errorf("the revisions are %v, want r1 activated again as r2, without the registry", got) + } + if w.engine.holds(w.registry) || len(w.state(t).Withdrawn()) != 0 { + t.Errorf("the store holds the registry: %t, and the state owes %v; want it deleted and settled by r2", + w.engine.holds(w.registry), w.state(t).Withdrawn()) + } + if result.RequeueAfter != engineRetryInterval { + t.Errorf("the next look is %s on, want %s: nothing in the cluster changes when the engine's users "+ + "clear it", result.RequeueAfter, engineRetryInterval) + } + }) + } +} + +// TestARotatedValueReachesTheRevisionTheEngineRunsWhileTheDocumentIsHeld pins +// that a value the sync writes into the store while the engine refuses the +// document, a credential somebody rotated, is read by the revision the engine +// runs: the operator activates that revision again for it, as it does for the +// value of a deleted Secret, since no write of the document is coming to do +// it. Only a withdrawal got that, so the running revision kept the old value, +// a revoked key say, for as long as the document was refused, while every +// node that restarted read the new one. A revision that does not reference the +// value is not activated again for it, and an activation that fails says so +// and comes back when it promises. +func TestARotatedValueReachesTheRevisionTheEngineRunsWhileTheDocumentIsHeld(t *testing.T) { + for _, tc := range []struct { + name string + refusal *engineRefusal + unreferenced, fails bool + }{ + {name: "the engine finds the document invalid", refusal: invalidDocument}, + {name: "a seat somebody holds", refusal: heldSeat}, + {name: "the running revision does not reference the value", refusal: invalidDocument, unreferenced: true}, + {name: "the activation fails", refusal: invalidDocument, fails: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // Nothing is withdrawn: the registry's Secret is back, and the + // engine runs a revision that read every value. + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.settled(t) + if owed := w.state(t).Written(); len(owed) != 0 { + t.Fatalf("the state owes %v before the rotation, want nothing", owed) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.rotate(t) + w.engine.mu.Lock() + // The engine's own re-activation at the write fails: the + // activation is the operator's. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.refusal, w.engine.reloadDown = tc.refusal, tc.fails + if tc.unreferenced { + // Somebody activated a document that names no such value. + doc := w.engine.revisions[len(w.engine.revisions)-1].document + w.engine.activateOther(bytes.ReplaceAll(doc, []byte("${"+tavily+"}"), []byte("tvly-literal"))) + } + w.engine.mu.Unlock() + before := w.engine.ids() + result, err := w.run.converge(ctx, w.client) + if err != nil { + t.Fatalf("converge() = %v, want the refusal reported", err) + } + w.engine.mu.Lock() + stored := w.engine.store[tavily] + w.engine.mu.Unlock() + if stored != rotatedValue { + t.Fatalf("the store holds %q under %s, want the rotated value", stored, tavily) + } + recorded := drained(w.events) + ids, written := w.engine.ids(), w.state(t).Written() + switch { + case tc.unreferenced: + if !slices.Equal(ids, before) || !slices.Equal(written, []string{tavily}) { + t.Errorf("the revisions are %v and the state owes %v; want %v, nothing activated again for a value "+ + "the running revision does not name, and the value owed to the next write", ids, written, before) + } + case tc.fails: + if !slices.Equal(written, []string{tavily}) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretRefreshFailed ") && strings.Contains(e, "tries again in 2s") + }) || result.RequeueAfter != 2*time.Second { + t.Errorf("the state owes %v, the next look is %s on and events %q; want the value owed, "+ + "SecretRefreshFailed promising 2s, and 2s", written, result.RequeueAfter, recorded) + } + default: + if want := append(slices.Clone(before), fmt.Sprintf("r%d", len(before)+1)); !slices.Equal(ids, want) || + len(written) != 0 { + t.Errorf("the revisions are %v and the state owes %v; want %v, the running revision activated "+ + "again for the rotated value, and nothing owed", ids, written, want) + } + if !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Normal Reactivated ") && strings.Contains(e, tavily) + }) { + t.Errorf("events %q, want Reactivated naming %s", recorded, tavily) + } + } + }) + } +} + +// TestAHeldDocumentHasTheEngineReadAValueWrittenBeforeIt pins that a reconcile +// whose document is held makes the activation a value written earlier is +// owed, where the revision the engine runs references it: a reconcile wrote a +// rotated value and went no further, and the next one finds the document +// held. It returned at once when nothing was withdrawn, missing or kept, so +// the running revision went on with the old value for as long as the +// document was held. An activation that fails is reported as one owed for a +// value written, not as a withdrawal that failed. +func TestAHeldDocumentHasTheEngineReadAValueWrittenBeforeIt(t *testing.T) { + for _, fails := range []bool{false, true} { + t.Run(map[bool]string{false: "the activation lands", true: "the activation fails"}[fails], func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.cluster.Create(ctx, valueSecret(optionalSecret, map[string]string{optionalKey: npmRegistry})); err != nil { + t.Fatal(err) + } + w.rotate(t) + // A reconcile wrote the rotated value, whose re-activation by the + // engine failed, and went no further. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + owed, _, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + if err := secretsync.SaveState(ctx, w.cluster, w.ref(), w.run.company, owed); err != nil { + t.Fatal(err) + } + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + if !slices.Equal(owed.Written(), []string{tavily}) { + t.Fatalf("the state owes %v, want the rotated value", owed.Written()) + } + w.engine.mu.Lock() + w.engine.reloadDown = fails + w.engine.mu.Unlock() + result, err := w.run.withdrawHeld(ctx, w.resolve(t)) + if err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + state := w.state(t) + if fails { + if !slices.Equal(state.Written(), []string{tavily}) || !slices.ContainsFunc(recorded, func(e string) bool { + return strings.HasPrefix(e, "Warning SecretRefreshFailed ") && strings.Contains(e, "tries again in 2s") + }) || warned(recorded, "SecretWithdrawalFailed") || result.RequeueAfter != 2*time.Second { + t.Errorf("the state owes %v, the next look is %s on and events %q; want the value owed, "+ + "SecretRefreshFailed promising 2s and no failed withdrawal, and 2s", + state.Written(), result.RequeueAfter, recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) || len(state.Written()) != 0 { + t.Errorf("the revisions are %v and the state owes %v; want r1 activated again for the rotated value, "+ + "and nothing owed", got, state.Written()) + } + }) + } +} + +// TestARevisionAdoptedInSyncIsRecordedAtItsEpoch pins that a revision of the +// rendered document the operator did not activate, a reload of its document +// here, is recorded at the epoch the fleet's activation names for it, and as +// found now (appliedAt), which the wait for the engine to apply it counts +// from. Recorded at the epoch of the revision it replaced, the engine read as +// having applied it once it applied that one, and the Applied event paired it +// with that epoch. A fleet that cannot be read, or that answers without its +// activation, records nothing and claims no race: the reconcile fails and the +// next one reads both again. A fleet whose activation names another revision +// records nothing either, and is a lost race (engine.ErrRevisionAdvanced), +// which converge reads again from the start. +func TestARevisionAdoptedInSyncIsRecordedAtItsEpoch(t *testing.T) { + ctx := context.Background() + found := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + w := newWithdrawal(t, runningLetsGo) + company := w.run.company + company.Status.Config.Epoch = new(int64(1)) + company.Status.Config.AppliedAt = &metav1.Time{Time: found.Add(-6 * time.Hour)} + w.again(t, company) + w.run.now = func() time.Time { return found } + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + active, err := activeConfig(ctx, w.client) + if err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err != nil { + t.Fatal(err) + } + config := w.statusWritten().Status.Config + if config.RevisionID != "r2" || config.Epoch == nil || *config.Epoch != 2 { + epoch := "none" + if config.Epoch != nil { + epoch = strconv.FormatInt(*config.Epoch, 10) + } + t.Errorf("status.config = revision %s at epoch %s; want r2 at its own epoch, 2", config.RevisionID, epoch) + } + if config.AppliedAt == nil || !config.AppliedAt.Equal(&metav1.Time{Time: found}) { + t.Errorf("status.config.appliedAt = %v, want when the operator found r2, %s, not its last write", config.AppliedAt, found) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.fleetDown = true + w.engine.mu.Unlock() + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + w.run.config != nil { + t.Errorf("inSync over a fleet that cannot be read = %v, recording %+v; want an error that claims no race and "+ + "nothing recorded", err, w.run.config) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.fleetDown, w.engine.noActivation = false, true + w.engine.mu.Unlock() + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err == nil || errors.Is(err, engine.ErrRevisionAdvanced) || + strings.Contains(err.Error(), "instead") || w.run.config != nil { + t.Errorf("inSync over a fleet answering without its activation = %v, recording %+v; want an error that claims no "+ + "race and nothing recorded", err, w.run.config) + } + + w.again(t, company) + w.engine.mu.Lock() + w.engine.noActivation = false + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded again"); err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); !errors.Is(err, engine.ErrRevisionAdvanced) || w.run.config != nil { + t.Errorf("inSync of r2 while the fleet's activation names r3 = %v, recording %+v; want a lost race "+ + "(ErrRevisionAdvanced) and nothing recorded", err, w.run.config) + } +} + +// TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline pins that the wait for the +// engine to apply a revision the operator adopts counts from when it found +// it, not from the operator's last write, another revision's: a reload the +// engine has not applied yet is Applying however old that write is, with no +// ApplyFailed event, and ApplyFailed only applyDeadline after the operator +// found it, in words that do not say the operator activated it. +func TestAnAdoptedRevisionIsHeldToItsOwnApplyDeadline(t *testing.T) { + ctx := context.Background() + found := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + w := newWithdrawal(t, runningLetsGo) + company := w.run.company + company.Status.Config.Epoch = new(int64(1)) + company.Status.Config.AppliedAt = &metav1.Time{Time: found.Add(-6 * time.Hour)} + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.engine.mu.Lock() + w.engine.appliedLag = 1 + w.engine.mu.Unlock() + look := func(at time.Time) { + t.Helper() + w.run.now = func() time.Time { return at } + active, err := activeConfig(ctx, w.client) + if err != nil { + t.Fatal(err) + } + if err := w.run.inSync(ctx, w.client, active, w.run.company.Status.Config.AppliedHash); err != nil { + t.Fatal(err) + } + h, err := w.client.Health(ctx) + w.run.verifyApplied(ctx, w.client, h, err, ctrl.Result{}) + } + + w.again(t, company) + look(found) + if got := w.run.configApplied; got.Reason != crewletaiv1alpha1.ReasonApplying { + t.Fatalf("ConfigApplied of a reload the engine has not applied, six hours after the operator's last write = %+v, "+ + "want Applying", got) + } + if warned(drained(w.events), crewletaiv1alpha1.ReasonApplyFailed) { + t.Error("an ApplyFailed event for a reload the operator found a moment ago") + } + + company = w.statusWritten() + meta.SetStatusCondition(&company.Status.Conditions, *w.run.configApplied) + w.again(t, company) + look(found.Add(applyDeadline)) + if got := w.run.configApplied; got.Reason != crewletaiv1alpha1.ReasonApplyFailed || + !strings.Contains(got.Message, "found it activated") { + t.Errorf("ConfigApplied of that reload %s after the operator found it = %+v, want ApplyFailed saying the operator "+ + "found it activated", applyDeadline, got) + } +} + +// TestAValueWrittenBackStaysOwedThroughAFailedWrite pins that a value the sync +// writes back, which leaves the hash as it was, stays a reason to write the +// document until a revision activates after it, however the reconcile that +// wrote it back ends. The store lost the operator's value; a person's reload +// activated a revision reading the store without it; the reconcile that wrote +// it back, whose own re-activation by the engine failed, could not get its +// write past the dry run. The next reconcile's sync has nothing to write, and +// the document, its values and the engine's revision all match: judged by what +// that sync wrote, the engine was in sync, and it went on running the revision +// that read the store without the value. +func TestAValueWrittenBackStaysOwedThroughAFailedWrite(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.dryRunDown = true + w.engine.mu.Unlock() + if err := w.enforce(t); err == nil { + t.Fatal("enforce() = nil over a dry run the engine could not answer") + } + if !w.engine.holds(tavily) || !slices.Equal(w.state(t).Written(), []string{tavily}) { + t.Fatalf("after the write-back: store holds %s = %t, state owes %v; want it held and owed", + tavily, w.engine.holds(tavily), w.state(t).Written()) + } + + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.dryRunDown = false + w.engine.mu.Unlock() + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3", "r4"}) { + t.Fatalf("revisions = %v, want r4 written over the reload, which read the store without %s", got, tavily) + } + if summary := w.engine.summaryOf(); !strings.Contains(summary, "Secret ns/"+requiredSecret) { + t.Errorf("r4's summary = %q, want it to name the Secret written back", summary) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("state owes %v after r4 activated, want nothing", got) + } +} + +// settled runs reconciles of w's Company, each from the status the one before +// it wrote, until one activates nothing, and returns the engine's revisions. +func (w *withdrawal) settled(t *testing.T) []string { + t.Helper() + ids := w.engine.ids() + for range 4 { + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + if now := w.engine.ids(); !slices.Equal(now, ids) { + ids = now + continue + } + return ids + } + t.Fatalf("the reconciles go on activating revisions: %v", ids) + return nil +} + +// TestASaveLostAfterAConfirmedWriteWritesNothingAgain pins that a write of the +// document the engine confirmed settles every value it read, whether or not +// the reconcile that made it got to save so. The store lost a value, the sync +// wrote it back, the engine's own re-activation at the write failed, and the +// operator's write activated a revision reading it; +// the save of the state after that write is lost, and the next reconcile +// starts from the state saved before the write, with the status that recorded +// the write or, where the reconcile went no further, the status from before +// it. Settled only by the save the reconcile lost, the value stayed owed, and +// the next reconcile wrote the same document again, an epoch every node +// applied for nothing. +func TestASaveLostAfterAConfirmedWriteWritesNothingAgain(t *testing.T) { + for _, tc := range []struct { + name string + statusLost bool + }{ + {name: "the status records the write"}, + {name: "the status is the one from before the write", statusLost: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + ids := w.settled(t) + before := w.run.company.DeepCopy() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.afterApply = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + if err := w.enforce(t); !errors.Is(err, errStateWrite) { + t.Fatalf("the reconcile that wrote the value back ends with %v, want the save after its write", err) + } + written := w.engine.ids() + if len(written) != len(ids)+1 || !w.engine.holds(tavily) { + t.Fatalf("revisions %v and the store holds %s: %t, want one write after %v and the value back", + written, tavily, w.engine.holds(tavily), ids) + } + + w.failWrite.Store(0) + next := w.statusWritten() + if tc.statusLost { + next = before + } + w.again(t, next) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, written) { + t.Fatalf("revisions = %v, want %v: the write that read the value runs", got, written) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: the revision that runs read every value", got) + } + if config := w.statusWritten().Status.Config; config.RevisionID != written[len(written)-1] { + t.Errorf("status.config records %s, want the write that runs, %s", config.RevisionID, written[len(written)-1]) + } + }) + } +} + +// TestAWriteBackWhoseSaveIsLostStaysOwed pins that a value the sync writes +// back stays owed when the save after the write is lost. The store lost the +// operator's value and a person's reload activated a revision reading the +// store without it; the sync wrote it back, the engine's own re-activation at +// the write failed, and the sync could not save that it had written it. +// Recorded only after the write, nothing said so: the next sync found the +// store holding the operator's own value under the fingerprint it had, wrote +// nothing, and the operator took the reload for in sync while the engine went +// on running it without the value. +func TestAWriteBackWhoseSaveIsLostStaysOwed(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.settled(t) + tavily := secretsync.StoreName(secretref.Key{Namespace: "ns", Name: requiredSecret, Key: requiredKey}) + w.engine.mu.Lock() + delete(w.engine.store, tavily) + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + reloaded := w.engine.ids() + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + // The save after the value is written is lost. + w.engine.mu.Lock() + w.engine.afterPut = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + if err := w.enforce(t); !errors.Is(err, errStateWrite) { + t.Fatalf("the reconcile that wrote the value back ends with %v, want the save after the write", err) + } + if !w.engine.holds(tavily) || !slices.Equal(w.engine.ids(), reloaded) { + t.Fatalf("the store holds %s: %t, revisions %v; want it written back and nothing activated after %v", + tavily, w.engine.holds(tavily), w.engine.ids(), reloaded) + } + + w.failWrite.Store(0) + w.again(t, w.statusWritten()) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); len(got) != len(reloaded)+1 { + t.Fatalf("revisions = %v, want the document written over the reload, which read the store without %s", got, tavily) + } + if summary := w.engine.summaryOf(); !strings.Contains(summary, "Secret ns/"+requiredSecret) { + t.Errorf("the write's summary = %q, want it to name the Secret written back", summary) + } + if got := w.state(t).Written(); len(got) != 0 { + t.Errorf("the state owes %v after the write activated, want nothing", got) + } +} + +// TestARevisionThatReadEveryValueIsInSyncWhateverHashWasRecorded pins that the +// hash the operator recorded is no part of whether the engine is in sync: a +// revision of the rendered document that activated after every value the sync +// wrote runs the document with every value, and is recorded with the hash of +// what it runs. Held to the recorded hash, a write whose record was never +// saved, of a document whose own hash moved, was written again. +func TestARevisionThatReadEveryValueIsInSyncWhateverHashWasRecorded(t *testing.T) { + w := newWithdrawal(t, runningLetsGo) + ids := w.settled(t) + stale := w.run.company.DeepCopy() + stale.Status.Config.AppliedHash = "the hash of another document" + w.again(t, stale) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, ids) { + t.Fatalf("revisions = %v, want %v: nothing written over a revision that runs the document", got, ids) + } + if w.run.config == nil || w.run.config.AppliedHash == stale.Status.Config.AppliedHash || w.run.config.AppliedHash == "" { + t.Errorf("status.config = %+v, want the hash of what runs recorded", w.run.config) + } +} + +// TestAFailedReloadIsReadBack pins that a request to activate the running +// revision again that fails is read back: a revision the engine runs above the +// epoch the withdrawal was marked by came after the withdrawal, whoever +// activated it, and settles it. Where its document is the one read, the +// engine runs it again without the value and it is recorded as the operator's +// reactivation, its event saying the request failed; where it is another +// document, a Warning says the operator activated nothing again and takes it +// to read every value as required. A request that fails with nothing +// activated since leaves the withdrawal owed and reported. Taken for a failed +// withdrawal whatever activated, the operator reported a value the engine no +// longer read and activated a revision again for nothing. +func TestAFailedReloadIsReadBack(t *testing.T) { + // What somebody activates before the request is refused. + const ( + nothing = iota + same + another + ) + for _, tc := range []struct { + name string + other int + }{ + {name: "nothing activated", other: nothing}, + {name: "the running document activated again", other: same}, + {name: "another document activated", other: another}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + switch tc.other { + case same: + w.engine.beforeReload = func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + case another: + w.engine.beforeReload = func(e *storeEngine) { e.activateOther(w.requiring) } + } + w.engine.mu.Unlock() + found := time.Date(2026, 10, 25, 9, 0, 0, 0, time.UTC) + w.run.now = func() time.Time { return found } + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + state := w.state(t) + if tc.other == nothing { + if !slices.Equal(state.Withdrawn(), []string{w.registry}) || len(state.Written()) != 0 || + !warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want the registry owed, r1 having "+ + "read every value, and the failure reported", state.Withdrawn(), state.Written(), recorded) + } + return + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Fatalf("the revisions are %v, want r1 and the r2 somebody activated", got) + } + if len(state.Withdrawn()) != 0 || len(state.Written()) != 0 || warned(recorded, "SecretWithdrawalFailed") { + t.Errorf("the state owes withdrawals %v and writes %v, and events %q; want both settled by r2, which "+ + "activated after them, and no failure reported", state.Withdrawn(), state.Written(), recorded) + } + if again := tc.other == same; state.Records("r2") != again || told(recorded, "Reactivated") != again || + warned(recorded, "Reactivated") == again { + t.Errorf("the state records r2: %t, events %q; want r2 recorded and a Normal Reactivated: %t, "+ + "a Warning otherwise", state.Records("r2"), recorded, again) + } + if tc.other != same { + return + } + if !slices.Equal(state.LetGo("r2"), []string{w.registry}) { + t.Errorf("the state lets %v go for r2, want the registry, as r1 does", state.LetGo("r2")) + } + // r1 was the operator's, so r2, its document again, is. + if config := w.run.config; config == nil || config.RevisionID != "r2" || config.Epoch == nil || + *config.Epoch != 2 || config.AppliedAt == nil || !config.AppliedAt.Time.Equal(found) { + t.Errorf("status.config = %+v, want r2 at epoch 2, applied when the operator found it, %s", config, found) + } + }) + } +} + +// TestAnActivationNoNodeCanOpenSettlesNothing pins that the fleet's +// activation settles a change only where it names the revision the read of +// the document returned, which GET /config opened: one a write activated +// between the two reads may be one no node can open, which reads the store +// for nobody. +func TestAnActivationNoNodeCanOpenSettlesNothing(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + w.again(t, w.statusWritten()) + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { + e.activateOther(e.revisions[len(e.revisions)-1].document) + e.revisions[len(e.revisions)-1].unopened = keyringLost + } + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.state(t).Withdrawn(); !slices.Equal(got, []string{w.registry}) { + t.Errorf("the state owes %v, want the registry: r2, which the pointer names, is one no node opens", got) + } +} + +// TestAnActivationAfterAWithdrawalSettlesIt pins that a revision the engine +// runs above the epoch a withdrawal was marked by settles it, whoever +// activated it: a person's reload after a request of the operator's failed +// read the store without the value, and nothing is activated again for it. +func TestAnActivationAfterAWithdrawalSettlesIt(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.state(t).Withdrawn(); !slices.Equal(got, []string{w.registry}) { + t.Fatalf("the state owes %v after a request that failed, want the registry", got) + } + w.engine.mu.Lock() + w.engine.reloadDown = false + w.engine.mu.Unlock() + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Errorf("the revisions are %v, want r1 and the person's r2: nothing activated again", got) + } + if got := w.state(t).Withdrawn(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: r2 read the store without the registry", got) + } +} + +// TestAHeldReconcileSavesTheMarksItsReadSetBeforeItReloads pins that a +// held reconcile saves the epoch its read of the fleet gave a change before +// it asks for the reload that is to settle it. The withdrawal was made, the +// fleet's activation pointer could not be read after it, and the request that +// was to settle it failed, so it was saved unread. The next held reconcile +// reads the pointer, which marks it, and reloads, and the save after the +// reload is lost. Marked in memory alone, the withdrawal was loaded unread +// again, which no activation's epoch is above, and the reconcile after that +// activated the running revision a third time for a withdrawal the second had +// settled. +func TestAHeldReconcileSavesTheMarksItsReadSetBeforeItReloads(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.mu.Lock() + // The fleet answers the read that says what every pod runs, before the + // withdrawal, and not the one after it that is to mark it. + w.engine.beforeFleet = func(e *storeEngine) { + e.beforeFleet = func(e *storeEngine) { e.fleetDown = true } + } + w.engine.reloadDown = true + // The engine's own re-activation at the removal fails: the operator asks. + w.engine.reloadAnswer = failedReload("unavailable", "the coordination store did not answer") + w.engine.mu.Unlock() + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if state := w.state(t); !slices.Equal(state.Withdrawn(), []string{w.registry}) || !state.Unobserved() { + t.Fatalf("the state owes %v, unread: %t; want the registry, which no read of the fleet followed", + state.Withdrawn(), state.Unobserved()) + } + + w.engine.mu.Lock() + w.engine.fleetDown, w.engine.reloadDown = false, false + w.engine.afterReload = func(*storeEngine) { w.failWrite.Store(w.stateWrites.Load() + 1) } + w.engine.mu.Unlock() + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); !errors.Is(err, errStateWrite) { + t.Fatalf("withdrawHeld() = %v, want the save after the reload lost", err) + } + reloaded := w.engine.ids() + if !slices.Equal(reloaded, []string{"r1", "r2"}) { + t.Fatalf("the revisions are %v, want r1 activated again as r2", reloaded) + } + + w.failWrite.Store(0) + w.again(t, w.statusWritten()) + if _, err := w.run.withdrawHeld(ctx, w.resolve(t)); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, reloaded) { + t.Errorf("the revisions are %v, want %v: r2 activated after the withdrawal, and nothing again", got, reloaded) + } + if got := w.state(t).Withdrawn(); len(got) != 0 { + t.Errorf("the state owes %v, want nothing: r2 read the store without the registry", got) + } +} + +// TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument pins that inSync +// judges the revision an attempt of converge read against a fleet read after +// that read, in the same attempt (keepServing reads it afresh). An attempt +// whose fleet names a later revision than its document read starts over (a +// lost race); judged against the answer the first attempt read, the second +// attempt's read, later again, would look like a second lost race, and the +// reconcile would fail over an engine that was fine. +func TestAnAttemptReadsTheFleetAfterItsOwnReadOfTheDocument(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + if err := w.enforce(t); err != nil { + t.Fatal(err) + } + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2"}) { + t.Fatalf("revisions after the write = %v, want [r1 r2]", got) + } + w.again(t, w.statusWritten()) + if _, err := w.client.ReloadConfig(ctx, "reloaded by a person"); err != nil { + t.Fatal(err) + } + reloadOnce := func(e *storeEngine) { e.activateOther(e.revisions[len(e.revisions)-1].document) } + w.engine.mu.Lock() + w.engine.beforeFleet = func(e *storeEngine) { + reloadOnce(e) + e.beforeConfig = reloadOnce + } + w.engine.mu.Unlock() + + if _, err := w.run.converge(ctx, w.client); err != nil { + t.Fatalf("converge() = %v, want the second attempt to adopt the revision it read", err) + } + config := w.statusWritten().Status.Config + if got := w.engine.ids(); !slices.Equal(got, []string{"r1", "r2", "r3", "r4", "r5"}) { + t.Errorf("revisions = %v, want [r1 r2 r3 r4 r5]: nothing written, only the reloads", got) + } + if config.RevisionID != "r5" || config.Epoch == nil || *config.Epoch != 5 { + t.Errorf("status.config = %+v, want r5 at epoch 5, the revision the second attempt read", config) + } +} diff --git a/internal/controller/sentence_test.go b/internal/controller/sentence_test.go new file mode 100644 index 00000000..5c9470ef --- /dev/null +++ b/internal/controller/sentence_test.go @@ -0,0 +1,268 @@ +package controller + +import ( + "context" + "errors" + "go/ast" + "go/parser" + "go/token" + "net/http" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + crewletaiv1alpha1 "github.com/crewlet/crewlet-operator/api/v1alpha1" + "github.com/crewlet/crewlet-operator/internal/engine" + "github.com/crewlet/crewlet-operator/internal/secretsync" +) + +// The three shapes of what a message quotes at its end: the engine's words, +// written as a sentence with its full stop (its 503 unavailable, as crewlet's +// main and crewlet#142 word it under a Retry-After); the engine's words with +// its hint, which the client puts in parentheses after the message, both +// sentences with their full stops (the 503 a node answers while it starts, +// detail and hint as the engine's httpjson.Sentence ends them); and an error +// of Go's, which has none. +var ( + quotedSentence = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "unavailable", + Message: "Can't answer right now. Try again in a moment, or ask another node.", RetryAfter: 2 * time.Second} + quotedHint = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: "starting", + Message: "This node is recovering what it holds (its store, its broker and its copy of the estate) " + + "and serves nothing until it has started.", + Hint: `Retry after the Retry-After: once the node serves, /health stops answering status "starting" ` + + `and /ready stops naming reason "starting".`, + RetryAfter: 10 * time.Second} + errQuotedClause = errors.New("dial tcp 10.0.0.7:8080: connect: connection refused") + // quotedNoWait is the engine's 503 for a condition no wait clears, as + // crewlet#142 words it: quotedSentence's code, with no Retry-After and a + // sentence saying so. + quotedNoWait = &engine.APIError{StatusCode: http.StatusServiceUnavailable, Code: quotedSentence.Code, + Message: "This node has been evicted from the fleet. Have an administrator readmit it."} +) + +// endsOnce reports whether message ends with exactly one full stop: one not +// after another, or one a closing parenthesis follows, the end of a quoted +// hint, with nothing after it. +func endsOnce(message string) bool { + if body, quoted := strings.CutSuffix(message, ")"); quoted { + return strings.HasSuffix(body, ".") && !strings.HasSuffix(body, "..") + } + return strings.HasSuffix(message, ".") && !strings.HasSuffix(message, "..") && !strings.HasSuffix(message, ".).") +} + +func TestEndedEndsASentenceOnce(t *testing.T) { + for _, c := range []struct{ text, want string }{ + {"the engine refused it", "the engine refused it."}, + {"Try again in a moment.", "Try again in a moment."}, + {"Is it running?", "Is it running?"}, + {"It stopped!", "It stopped!"}, + {" padded ", "padded."}, + {"", ""}, + {"the value (Secret ns/a)", "the value (Secret ns/a)."}, + {"It failed. (Try again later.)", "It failed. (Try again later.)"}, + {"It failed (try again later)", "It failed (try again later)."}, + {"Is it running? (Ask the node!)", "Is it running? (Ask the node!)"}, + {")", ")."}, + } { + if got := ended(c.text); got != c.want { + t.Errorf("ended(%q) = %q, want %q", c.text, got, c.want) + } + } +} + +// TestAConditionThatQuotesAnErrorEndsItOnce holds every condition that ends +// with an error the operator met to one full stop, whether the error ends in +// the engine's sentence or in Go's words. +func TestAConditionThatQuotesAnErrorEndsItOnce(t *testing.T) { + for _, err := range []error{quotedSentence, quotedHint, quotedNoWait, errQuotedClause} { + t.Run(err.Error(), func(t *testing.T) { + run := &configRun{CompanyConfigReconciler: &CompanyConfigReconciler{}, company: &crewletaiv1alpha1.Company{}} + run.unavailable(err) + seat := &crewletaiv1alpha1.Seat{Namespace: "ns", Name: seatAna, Generation: 1} + p := newPart(seat) + run.fleetErr = err + run.decideReady(p, seat) + ready := seat.Status.Conditions[0] + for name, message := range map[string]string{ + "ConfigApplied": run.configApplied.Message, + "EngineHealthy": engineHealthy(nil, err).Message, + "IdentityClaimed": identityClaimed(engine.Health{}, err, nil, "", "").Message, + "a Seat's Ready": ready.Message, + } { + if !endsOnce(message) { + t.Errorf("%s does not end with one full stop: %q", name, message) + } + } + }) + } +} + +// TestANoWait503PromisesNoRetry holds the condition and the event that quote +// the engine's 503 for a condition no wait clears to saying so: no retry in so +// many seconds, which the engine's own words beside it contradict, but that +// somebody clears it in the engine while the operator goes on asking. A 503 +// under a Retry-After keeps its promise of a retry when the engine said. +func TestANoWait503PromisesNoRetry(t *testing.T) { + ctx := context.Background() + for _, c := range []struct { + name string + err error + noWait bool + }{ + {"no wait clears it", quotedNoWait, true}, + {"a wait clears it", quotedSentence, false}, + } { + t.Run(c.name, func(t *testing.T) { + run := &configRun{CompanyConfigReconciler: &CompanyConfigReconciler{}, company: &crewletaiv1alpha1.Company{}} + run.unavailable(c.err) + w := newWithdrawal(t, runningLetsGo) + w.run.withdrawalFailed(ctx, c.err) + event := strings.Join(drained(w.events), "\n") + for name, message := range map[string]string{ + "ConfigApplied": run.configApplied.Message, "SecretWithdrawalFailed": event, + } { + promised, cleared := strings.Contains(message, "tries again in"), + strings.Contains(message, "somebody has to clear it in the engine") + if promised == c.noWait || cleared != c.noWait { + t.Errorf("%s promises a retry: %t, and says somebody clears it: %t; want %t and %t: %s", + name, promised, cleared, !c.noWait, c.noWait, message) + } + } + }) + } +} + +// TestAFailedApplyAccountEndsWhatItQuotesOnce holds ApplyFailed's account to +// one full stop after what it quotes: the engine's answer when its record of +// the attempt cannot be read, and its account of an apply that failed, which +// is a clause or a sentence as the engine had it. +func TestAFailedApplyAccountEndsWhatItQuotesOnce(t *testing.T) { + written := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, c := range []struct { + name, fleet string + status int + }{ + {name: "a record that cannot be read", status: http.StatusServiceUnavailable, + fleet: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`}, + {name: "an account the engine ends", fleet: fleet(2, "r2", "error", "Something on this node failed.")}, + {name: "an account the engine does not end", fleet: fleet(2, "r2", "error", "engine: apply: no sandbox cell")}, + } { + t.Run(c.name, func(t *testing.T) { + cl := (&fakeEngine{health: health("isolated", "isolated", 1), fleet: c.fleet, fleetStatus: c.status}).client(t) + run, _ := verifyRun(written.Add(time.Second), written, crewletaiv1alpha1.ReasonApplied, nil) + h, err := cl.Health(context.Background()) + run.verifyApplied(context.Background(), cl, h, err, ctrl.Result{}) + account, _, _ := strings.Cut(run.configApplied.Message, " It reports status") + if run.configApplied.Reason != crewletaiv1alpha1.ReasonApplyFailed || !endsOnce(account) { + t.Errorf("ConfigApplied is %s: %q; want ApplyFailed, its account ending with one full stop", + run.configApplied.Reason, run.configApplied.Message) + } + }) + } +} + +// TestASyncThatFailsEndsWhatItQuotesOnce holds SecretsSynced to one full stop +// after the engine's refusal of a write to its secret store. +func TestASyncThatFailsEndsWhatItQuotesOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + w.engine.secretsDown = true + ref := types.NamespacedName{Namespace: w.run.company.Namespace, Name: secretSyncStateName(w.run.company)} + if _, _, err := w.run.sync(ctx, w.client, w.resolve(t), w.state(t), ref); err == nil { + t.Fatal("the sync succeeded against a store that refused it") + } + if c := w.run.secretsSynced; c == nil || c.Status != metav1.ConditionFalse || !endsOnce(c.Message) { + t.Errorf("SecretsSynced is %+v, want False, its message ending with one full stop", c) + } +} + +// TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce holds the Warning of an +// activation the operator could not read back to one full stop after the +// engine's answer to the read, before the sentence that follows it. +func TestAReloadThatCannotBeReadBackEndsWhatItQuotesOnce(t *testing.T) { + ctx := context.Background() + w := newWithdrawal(t, runningLetsGo) + // The operator asks for the activation: the engine did not at the removal. + w.engine.reloadAnswer = new(map[string]any(nil)) + state, res, err := secretsync.Sync(ctx, w.client, w.resolve(t), w.state(t), operatorLogin, discardState) + if err != nil { + t.Fatal(err) + } + w.engine.afterReload = func(e *storeEngine) { e.configDown = true } + if _, err := w.run.reactivate(ctx, w.client, nil, res.Reload, res.ReloadAction, state, w.ref()); err != nil { + t.Fatal(err) + } + recorded := drained(w.events) + i := slices.IndexFunc(recorded, func(e string) bool { return strings.HasPrefix(e, "Warning Reactivated ") }) + if i < 0 { + t.Fatalf("events %q, want a Warning Reactivated", recorded) + } + if strings.Contains(recorded[i], "..") { + t.Errorf("the Warning quotes the engine with two full stops: %q", recorded[i]) + } +} + +// quotedAtTheEnd is a format whose last verb quotes text and ends the +// sentence: "…: %s." and "…: %v.". +var quotedAtTheEnd = regexp.MustCompile(`: %[sv]\.$`) + +// TestQuotedEndingsGoThroughEnded holds the package's messages to ended: no +// message ends a sentence with a full stop of its own, neither as +// `text + "."` nor as a format ending ": %s." or ": %v.". Either is right only +// while what comes before it has none, and an error that carries the +// engine's words, which the engine ends with one, makes it two. A quote in +// the middle of a message (a pause's reason, a read that failed, followed by +// a sentence of the operator's) is not a shape this can see: those end the +// sentence that quotes them through ended too, and the tests beside this one +// hold each. +func TestQuotedEndingsGoThroughEnded(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + var found []string + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + t.Fatal(err) + } + // joined are the "." joins inside a longer join, such as a host + // name's labels: a full stop that ends nothing. + joined := map[ast.Expr]bool{} + ast.Inspect(f, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncDecl: + // ended's own body is where the rule is kept. + return n.Name.Name != "ended" + case *ast.BinaryExpr: + if inner, ok := n.X.(*ast.BinaryExpr); ok && n.Op == token.ADD { + joined[inner] = true + } + if lit, ok := n.Y.(*ast.BasicLit); ok && n.Op == token.ADD && lit.Value == `"."` && !joined[n] { + found = append(found, fset.Position(n.Pos()).String()+`: ends a message with a "." of its own`) + } + case *ast.BasicLit: + if text, err := strconv.Unquote(n.Value); n.Kind == token.STRING && err == nil && quotedAtTheEnd.MatchString(text) { + found = append(found, fset.Position(n.Pos()).String()+": ends a quote with a full stop of its own") + } + } + return true + }) + } + if len(found) == 0 { + return + } + t.Errorf("end these through ended:\n%s", strings.Join(found, "\n")) +} diff --git a/internal/engine/client.go b/internal/engine/client.go index 7779f3be..c8987e35 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -13,9 +13,10 @@ // that takes a held human seat away a *SeatHeldError, a lost compare-and-set // matches ErrRevisionAdvanced, a refused token ErrUnauthorized, a token the // engine will not let do this ErrForbidden, a write a managed document -// refuses this token ErrConfigManaged, a request the engine asks to be sent -// again ErrUnavailable, a draining or unconfigured engine ErrNotReady, and -// anything else an *APIError with the engine's code and message. +// refuses this token ErrConfigManaged, a revision the engine cannot open +// ErrRevisionUnreadable, a request the engine could not answer now +// ErrUnavailable, a draining or unconfigured engine ErrNotReady, and anything +// else an *APIError with the engine's code and message. // // No error this package returns carries a secret value: the engine never // echoes one, and a secret written with WriteSecret or PutSecrets travels @@ -229,11 +230,18 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { } `json:"unplaceable"` UnmannedRoles []string `json:"unmanned_roles"` TargetEpoch int64 `json:"target_epoch"` + Activation *struct { + RevisionID string `json:"revision_id"` + Epoch int64 `json:"epoch"` + } `json:"activation"` } if err := c.call(ctx, request{method: http.MethodGet, path: "/fleet"}, http.StatusOK, &answer); err != nil { return Fleet{}, err } f := Fleet{ThisNode: answer.ThisNode, UnmannedRoles: answer.UnmannedRoles, TargetEpoch: answer.TargetEpoch} + if a := answer.Activation; a != nil && a.RevisionID != "" { + f.Activation = &Activation{RevisionID: a.RevisionID, Epoch: a.Epoch} + } for i := range answer.Nodes { n := &answer.Nodes[i] f.Nodes = append(f.Nodes, FleetNode{ @@ -270,7 +278,10 @@ func (c *Client) Fleet(ctx context.Context) (Fleet, error) { // goes on running the revision it ran before, and starts on the revision its // own store names where that store outlives a restart, as a data pod's does // (a seat pod's does not: it starts on nothing and applies the fleet's -// revision). Before the first apply it fails with ErrNoActiveRevision. +// revision). Before the first apply it fails with ErrNoActiveRevision, and +// on an engine that cannot open the fleet's revision with +// ErrRevisionUnreadable, which names it (Unsealed reads the one a whole +// document replaces). // // An engine that cannot read the activation pointer from its coordination // store answers 503 with a Retry-After, which is ErrUnavailable: "the fleet @@ -352,8 +363,20 @@ func (c *Client) Revision(ctx context.Context, id string) (Revision, error) { // // A document the engine refuses fails with a *ValidationError locating each // problem; a stale req.IfMatch, or a write that lost a race with another -// writer, with ErrRevisionAdvanced; req.CreateOnly on a configured engine with -// ErrAlreadyConfigured. +// writer, with ErrRevisionAdvanced (a 412 when req.IfMatch is set, a 409 when +// it is not); req.CreateOnly on a configured engine with ErrAlreadyConfigured; +// a write while the engine cannot open the fleet's revision, which the write +// reads first, with ErrRevisionUnreadable, except over one stored without a +// seal (Unsealed), which the write (a dry run included) replaces, reading +// nothing from it: that revision stays the write's base, so a req.IfMatch is +// judged against it as against any other, and a write without one replaces +// it unconditionally; and a write whose activation the engine sent and could +// not confirm with ErrUnavailable naming the revision it stored, which +// UnconfirmedActivation reads. The fleet may run that revision or may not: the +// caller keeps it until a read of the fleet's activation (Fleet.Activation) +// says which, and sends the write again, on the condition it was first sent +// on, only once that read shows the fleet still runs the revision the write +// was built on. func (c *Client) ApplyConfig(ctx context.Context, req ApplyRequest) (ApplyResult, error) { if req.IfMatch != "" && req.CreateOnly { return ApplyResult{}, errors.New("apply the company document: IfMatch and CreateOnly are mutually exclusive: " + diff --git a/internal/engine/client_test.go b/internal/engine/client_test.go index 6006090d..69a54b8b 100644 --- a/internal/engine/client_test.go +++ b/internal/engine/client_test.go @@ -33,6 +33,10 @@ const ( baseRevision = "rev-1" + // revisionAdvanced is the code of a write or a reload that lost its race + // to another writer. + revisionAdvanced = "revision_advanced" + // poolLabel and dataPool are the placement label the operator gives a // data pod, and its value. poolLabel = "crewlet.ai/pool" @@ -114,6 +118,17 @@ func answer(status int, body string) http.HandlerFunc { } } +// answerAfter writes body, JSON, with status and a Retry-After of seconds: +// a 503 the engine writes where a wait clears what it met. The engine words a +// 503 by that header (httpjson.UnavailableWith), so a double that leaves it +// off answers with a message the engine never writes beside that body. +func answerAfter(status int, seconds, body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", seconds) + answer(status, body)(w, r) + } +} + // page writes body as a proxy in front of the engine would: not JSON. func page(status int) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { @@ -397,18 +412,34 @@ func TestUnlocatedDocumentRefusalIsStillAValidationError(t *testing.T) { } func TestErrorMapping(t *testing.T) { + // staleWrite is the write the operator makes: conditioned on the + // revision it read, which another write has moved past. + staleWrite := write + staleWrite.IfMatch = "rev-1" cases := []struct { name string + req *engine.ApplyRequest handler http.HandlerFunc sentinel error wantCode string wantStatus int }{ { + // A failed precondition: the request named the revision it + // was built on, and it is not the active one. name: "stale if-match", - handler: answer(http.StatusConflict, + req: &staleWrite, + handler: answer(http.StatusPreconditionFailed, `{"error":"revision_advanced","current_revision_id":"rev-9","your_base":"\"rev-1\""}`), - sentinel: engine.ErrRevisionAdvanced, wantCode: "revision_advanced", wantStatus: http.StatusConflict, + sentinel: engine.ErrRevisionAdvanced, wantCode: revisionAdvanced, wantStatus: http.StatusPreconditionFailed, + }, + { + // A conflict: a write that named no revision lost the race + // to another. The code is the same, and so is the sentinel. + name: "a lost race with no if-match", + handler: answer(http.StatusConflict, + `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`), + sentinel: engine.ErrRevisionAdvanced, wantCode: revisionAdvanced, wantStatus: http.StatusConflict, }, { name: "create-only on a configured engine", @@ -434,7 +465,7 @@ func TestErrorMapping(t *testing.T) { { name: "a token without the grant", handler: answer(http.StatusForbidden, `{"error":"unauthorized","grants":["admin"],`+ - `"message":"You don't have the access this needs (Admin, Worklet, Doclet or Artifacts). Ask an admin for it.",`+ + `"message":"You don't have the access this needs (Admin). Ask an admin for it.",`+ `"reason":"no_grant"}`), sentinel: engine.ErrForbidden, wantCode: "unauthorized", wantStatus: http.StatusForbidden, }, @@ -457,13 +488,59 @@ func TestErrorMapping(t *testing.T) { wantStatus: http.StatusForbidden, }, { + name: "a revision sealed under a key the keyring lost", + handler: answer(http.StatusConflict, unreadableRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unreadable_revision", wantStatus: http.StatusConflict, + }, + { + name: "a revision stored without a seal", + handler: answer(http.StatusConflict, unsealedRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_revision", wantStatus: http.StatusConflict, + }, + { + name: "the fleet's revision sealed under a key this node lacks", + handler: answer(http.StatusConflict, unreadableFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unreadable_fleet_revision", wantStatus: http.StatusConflict, + }, + { + name: "the fleet's revision without a seal", + handler: answer(http.StatusConflict, unsealedFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unsealed_fleet_revision", wantStatus: http.StatusConflict, + }, + { + name: "a revision sealed in a form this build does not read", + handler: answer(http.StatusConflict, unknownSealRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unknown_seal_revision", wantStatus: http.StatusConflict, + }, + { + name: "the fleet's revision sealed in a form this build does not read", + handler: answer(http.StatusConflict, unknownSealFleetRevision), + sentinel: engine.ErrRevisionUnreadable, wantCode: "unknown_seal_revision", wantStatus: http.StatusConflict, + }, + { + name: "a proxy's 409", + handler: page(http.StatusConflict), + wantStatus: http.StatusConflict, + }, + { + // The authority table's unknown: two seconds, and the code's + // own sentence, which invites the wait. name: "an engine that could not decide", - handler: answer(http.StatusServiceUnavailable, `{"error":"unavailable",`+ + handler: answerAfter(http.StatusServiceUnavailable, "2", `{"error":"unavailable",`+ `"message":"Can't answer right now. Try again in a moment, or ask another node."}`), sentinel: engine.ErrUnavailable, wantCode: "unavailable", wantStatus: http.StatusServiceUnavailable, }, { - // A 503 asks for the request again whoever wrote it. + // A condition no wait clears: who holds the human seats a + // write removes, read from the identity directory, refused on + // this node until somebody acts. No Retry-After, and a + // message that says waiting won't change it. + name: "a condition no wait clears", + handler: answer(http.StatusServiceUnavailable, seatHoldersUnavailableNoWait), + sentinel: engine.ErrUnavailable, wantCode: "seat_holders_unavailable", wantStatus: http.StatusServiceUnavailable, + }, + { + // A 503 is retried whoever wrote it. name: "a proxy's 503", handler: page(http.StatusServiceUnavailable), sentinel: engine.ErrUnavailable, wantStatus: http.StatusServiceUnavailable, }, @@ -477,13 +554,17 @@ func TestErrorMapping(t *testing.T) { sentinels := []error{ engine.ErrRevisionAdvanced, engine.ErrAlreadyConfigured, engine.ErrNoActiveRevision, engine.ErrUnauthorized, engine.ErrNotReady, engine.ErrConfigManaged, engine.ErrForbidden, - engine.ErrUnavailable, + engine.ErrUnavailable, engine.ErrRevisionUnreadable, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", tc.handler) - _, err := apply(f, write) + req := write + if tc.req != nil { + req = *tc.req + } + _, err := apply(f, req) var apiErr *engine.APIError if !errors.As(err, &apiErr) { t.Fatalf("ApplyConfig() = %v, want an *APIError", err) @@ -502,14 +583,147 @@ func TestErrorMapping(t *testing.T) { } } +// unreadableRevision, unsealedRevision, unreadableFleetRevision, +// unsealedFleetRevision, unknownSealRevision and unknownSealFleetRevision are +// the engine's answers to a read of a revision it cannot open, this node's +// copy or one the fleet's activation pointer carries, as crewlet's configapi +// words them (RefuseUnreadable, UnreadableRevisionError) and its +// httpjson.Envelope sends them, the detail and the hint each ended as a +// sentence (httpjson.Sentence): the code's own sentence, the revision, what +// happened as the detail, and the remedy as the hint. A seal the engine's +// build does not read is one code for either origin, told apart by its detail +// and its hint. Each names refusedRevision. +const ( + refusedRevision = "rev-4" + + unreadableRevision = `{"error":"unreadable_revision","message":"That revision is sealed under a key this ` + + `deployment no longer holds, so it cannot be read. Put the key back in the node's keyring first.",` + + `"revision_id":"rev-4","detail":"This node's keyring doesn't hold the key that revision is sealed under.",` + + `"hint":"It is sealed under a key that is no longer in the keyring; restore that key to the node's ` + + `secrets.keys first."}` + unsealedRevision = `{"error":"unsealed_revision","message":"That revision is stored without a seal, which ` + + `a build older than the mandatory keyring wrote, and a node reads only sealed revisions because the seal ` + + `is what authenticates one. The hint says what brings its document back.","revision_id":"rev-4",` + + `"detail":"This node's copy of that revision is stored without a seal, so nothing vouches for who wrote it.",` + + `"hint":"It is stored without a seal, so nothing vouches for who wrote it and it is neither shown, edited nor ` + + `reverted to; to replace it, an admin sends the whole document from your own copy with PUT /config, or ` + + `imports it with ` + "`crewlet config import`" + `, either of which stores it sealed."}` + unreadableFleetRevision = `{"error":"unreadable_fleet_revision","message":"This node doesn't hold the key ` + + `the fleet sealed its revision under, so it can't read it. The hint says what fixes it.",` + + `"revision_id":"rev-4","detail":"This node doesn't hold the key the fleet sealed this revision under.",` + + `"hint":"Add the key the fleet seals under to this node's secrets.keys and restart it."}` + unsealedFleetRevision = `{"error":"unsealed_fleet_revision","message":"The fleet's revision ` + + `arrived without a seal, so this node won't read it. The hint says what to do.","revision_id":"rev-4",` + + `"detail":"The fleet's revision arrived without a seal, so this node won't read it.",` + + `"hint":"Every node seals what it activates, so this body came from outside the fleet and is neither shown ` + + `nor edited; to replace it, an admin sends the whole document with PUT /config, or imports it with ` + + "`crewlet config import`" + `, either of which stores it sealed and activates it."}` + unknownSealRevision = `{"error":"unknown_seal_revision","message":"That revision is sealed in a form this ` + + `build of Crewlet doesn't read, most likely by a newer build, so this node can't read or replace it. The ` + + `hint says what fixes it.","revision_id":"rev-4",` + + `"detail":"This node's copy of that revision is sealed in a form this build of Crewlet doesn't read.",` + + `"hint":"A newer build of Crewlet most likely sealed it here, and no write here replaces it, since that ` + + `would discard it unread; run that build on this node again."}` + unknownSealFleetRevision = `{"error":"unknown_seal_revision","message":"That revision is sealed in a form ` + + `this build of Crewlet doesn't read, most likely by a newer build, so this node can't read or replace it. ` + + `The hint says what fixes it.","revision_id":"rev-4",` + + `"detail":"The fleet's revision is sealed in a form this build of Crewlet doesn't read.",` + + `"hint":"A newer build of Crewlet most likely sealed it, and no write here replaces it, since that would ` + + `discard it unread; upgrade this node to the build the rest of the fleet runs."}` +) + +// TestAnUnreadableRevisionNamesItAndItsRemedy reads the engine's answer to a +// read of the fleet's document its keyring does not open: ErrRevisionUnreadable +// and no other refusal, naming the revision, with the engine's account as the +// message and its remedy as the hint, which is everything a person has to act +// on, since no wait and no request clears it. +func TestAnUnreadableRevisionNamesItAndItsRemedy(t *testing.T) { + for _, body := range []string{unreadableRevision, unsealedRevision, unreadableFleetRevision, unsealedFleetRevision, + unknownSealRevision, unknownSealFleetRevision} { + f := newFakeEngine(t) + f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) + f.handle("GET /config", answer(http.StatusConflict, body)) + _, err := f.client().ActiveConfig(context.Background()) + var apiErr *engine.APIError + if !errors.Is(err, engine.ErrRevisionUnreadable) || !errors.As(err, &apiErr) || + errors.Is(err, engine.ErrUnavailable) || errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("ActiveConfig() = %v, want ErrRevisionUnreadable and no other refusal", err) + } + var want struct{ Detail, Hint string } + if err := json.Unmarshal([]byte(body), &want); err != nil { + t.Fatal(err) + } + if apiErr.RevisionID != refusedRevision || apiErr.Message != want.Detail || apiErr.Hint != want.Hint || + apiErr.RetryAfter != 0 { + t.Errorf("APIError = {revision %q, message %q, hint %q, retry after %s}, want {rev-4, the detail, "+ + "the hint, 0}", apiErr.RevisionID, apiErr.Message, apiErr.Hint, apiErr.RetryAfter) + } + } +} + +// TestUnsealedIsTheOneRevisionAWriteReplaces holds Unsealed to the one cause +// the engine lets a whole document replace: a revision stored without a seal, +// this node's copy or the fleet's, named by the refusal. A revision sealed +// under a key the keyring lacks, or in a form the engine's build does not +// read, is somebody's document, which the engine refuses every write over; +// read as unsealed, the operator would send a write the engine refuses, or, +// were the engine to take it, discard that document unread. A refusal that +// names no revision is no base to condition a write on. +func TestUnsealedIsTheOneRevisionAWriteReplaces(t *testing.T) { + for _, c := range []struct { + name, body string + unsealed bool + }{ + {"this node's copy without a seal", unsealedRevision, true}, + {"the fleet's revision without a seal", unsealedFleetRevision, true}, + {"a key the keyring lost", unreadableRevision, false}, + {"a key the fleet seals under that this node lacks", unreadableFleetRevision, false}, + {"this node's copy sealed in a form its build does not read", unknownSealRevision, false}, + {"the fleet's revision sealed in a form this build does not read", unknownSealFleetRevision, false}, + {"a revision without a seal the engine does not name", + strings.Replace(unsealedRevision, `"revision_id":"rev-4",`, "", 1), false}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":4}`)) + f.handle("GET /config", answer(http.StatusConflict, c.body)) + _, err := f.client().ActiveConfig(context.Background()) + if !errors.Is(err, engine.ErrRevisionUnreadable) { + t.Fatalf("ActiveConfig() = %v, want ErrRevisionUnreadable", err) + } + revision, ok := engine.Unsealed(err) + want := "" + if c.unsealed { + want = refusedRevision + } + if ok != c.unsealed || revision != want { + t.Errorf("Unsealed() = %q, %t; want %q, %t", revision, ok, want, c.unsealed) + } + }) + } + for _, err := range []error{nil, errors.New("dial tcp: connection refused"), + &engine.APIError{StatusCode: http.StatusConflict, RevisionID: refusedRevision}} { + if revision, ok := engine.Unsealed(err); ok { + t.Errorf("Unsealed(%v) = %q, true; want false: it is no refusal of a revision without a seal", err, revision) + } + } +} + +// TestRevisionAdvancedNamesTheCurrentRevision reads both revisions a lost +// compare-and-set names, from the answer the operator meets: its writes are +// conditioned on the revision they were built on, so the engine refuses one +// another write moved past with 412. func TestRevisionAdvancedNamesTheCurrentRevision(t *testing.T) { f := newFakeEngine(t) - f.handle("PUT /config", answer(http.StatusConflict, + f.handle("PUT /config", answer(http.StatusPreconditionFailed, `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`)) - _, err := apply(f, write) + req := write + req.IfMatch = "rev-1" + _, err := apply(f, req) var apiErr *engine.APIError - if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != "rev-9" || apiErr.StoredRevisionID != "rev-10" { - t.Fatalf("ApplyConfig() = %v, want the current and the stored revision named", err) + if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != "rev-9" || apiErr.StoredRevisionID != "rev-10" || + apiErr.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("ApplyConfig() = %v, want a 412 naming the current and the stored revision", err) } } @@ -553,7 +767,7 @@ func TestRefusalMessageIsWhatTheEngineSaid(t *testing.T) { func TestAuthorityRefusalNamesItsGrants(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", answer(http.StatusForbidden, `{"error":"unauthorized","grants":["admin"],`+ - `"message":"You don't have the access this needs (Admin, Worklet, Doclet or Artifacts). Ask an admin for it.",`+ + `"message":"You don't have the access this needs (Admin). Ask an admin for it.",`+ `"reason":"no_grant"}`)) _, err := apply(f, write) var apiErr *engine.APIError @@ -565,25 +779,42 @@ func TestAuthorityRefusalNamesItsGrants(t *testing.T) { } } +// The engine's refusal of a write that removes a human seat when the node +// could not read the identity directory to see who holds it, under a code of +// its own (seat_holders_unavailable), as it words it under a wait +// (seatHoldersUnavailableSoon, beside a two-second Retry-After) and where no +// wait clears it (seatHoldersUnavailableNoWait, with none: the node answering +// was evicted, and its detail says who acts). +const ( + seatHoldersUnavailableSoon = `{"detail":"This node couldn't read who holds the human seats this change ` + + `removes, so nothing was written. Try again in a moment.","error":"seat_holders_unavailable",` + + `"message":"Can't check who holds the seats this change removes right now, so nothing was written. ` + + `Try again in a moment.","seats":["founder"]}` + seatHoldersUnavailableNoWait = `{"detail":"The node answering this has been evicted from the fleet, so it ` + + `can't answer. Try another node, or have an administrator readmit the one that refused.",` + + `"error":"seat_holders_unavailable","message":"Can't check here who holds the seats this change removes, ` + + `so nothing was written. Waiting won't change that.","seats":["founder"]}` +) + // TestUnavailableCarriesTheEnginesRetryAfter reads when the engine asked to // be sent a request again: a 503's Retry-After in whole seconds, which is -// what the engine writes. A date is not read, nor is the header on anything -// but a 503. +// what the engine writes, and none where it wrote none, which it words as a +// wait that changes nothing. A date is not read, nor is the header on +// anything but a 503. func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { - const identityUnavailable = `{"error":"identity_unavailable",` + - `"hint":"This write removes a human seat, and this node could not read the identity directory to see ` + - `whether anybody holds it; nothing was written.",` + - `"message":"Can't check who you are right now. Try again in a moment.","seats":["founder"]}` for _, c := range []struct { name string status int header string + body string want time.Duration }{ - {name: "seconds", status: http.StatusServiceUnavailable, header: "2", want: 2 * time.Second}, - {name: "none", status: http.StatusServiceUnavailable}, - {name: "a date", status: http.StatusServiceUnavailable, header: "Fri, 16 Oct 2026 23:59:00 GMT"}, - {name: "not a 503", status: http.StatusConflict, header: "2"}, + {name: "seconds", status: http.StatusServiceUnavailable, header: "2", body: seatHoldersUnavailableSoon, + want: 2 * time.Second}, + {name: "none", status: http.StatusServiceUnavailable, body: seatHoldersUnavailableNoWait}, + {name: "a date", status: http.StatusServiceUnavailable, header: "Fri, 16 Oct 2026 23:59:00 GMT", + body: seatHoldersUnavailableSoon}, + {name: "not a 503", status: http.StatusConflict, header: "2", body: seatHoldersUnavailableSoon}, } { t.Run(c.name, func(t *testing.T) { f := newFakeEngine(t) @@ -591,7 +822,7 @@ func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { if c.header != "" { w.Header().Set("Retry-After", c.header) } - answer(c.status, identityUnavailable)(w, r) + answer(c.status, c.body)(w, r) }) _, err := apply(f, write) var apiErr *engine.APIError @@ -605,6 +836,44 @@ func TestUnavailableCarriesTheEnginesRetryAfter(t *testing.T) { } } +// TestWaitlessIsTheEnginesNoWait503 pins which answers Waitless reads as the +// engine's 503 for a condition no wait clears: one under a code of its own and +// no Retry-After, as crewlet#142 words it ("waiting won't change that"), and +// no other. A 503 with a Retry-After is one a wait clears; one whose write +// may have landed carries none and says to retry it; and one naming no code +// is a proxy's. +func TestWaitlessIsTheEnginesNoWait503(t *testing.T) { + const noWait = `{"error":"unavailable","message":"This can't be done here, and waiting won't change that.",` + + `"detail":"This node has been evicted from the fleet. Have an administrator readmit it."}` + for _, c := range []struct { + name string + handler http.HandlerFunc + want bool + }{ + {"a condition no wait clears", answer(http.StatusServiceUnavailable, noWait), true}, + {"a seat-holder read no wait clears", answer(http.StatusServiceUnavailable, seatHoldersUnavailableNoWait), true}, + {"a condition a wait clears", answerAfter(http.StatusServiceUnavailable, "2", + `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node."}`), false}, + {"a write that may have landed", answer(http.StatusServiceUnavailable, + `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node.",`+ + `"outcome":"unknown","op_id":"op-1"}`), false}, + {"a proxy's 503", page(http.StatusServiceUnavailable), false}, + {"a refusal that is no 503", answer(http.StatusConflict, `{"error":"revision_advanced"}`), false}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + f.handle("PUT /config", c.handler) + _, err := apply(f, write) + if got := engine.Waitless(err); got != c.want { + t.Errorf("Waitless(%v) = %t, want %t", err, got, c.want) + } + }) + } + if engine.Waitless(nil) { + t.Error("Waitless(nil) = true, want false") + } +} + // seatFounder is the human seat the seat_held tests find a person on, and // stageActive the stage of a person or a service account who has finished // enrolling, both as the engine names them. @@ -620,15 +889,15 @@ const ( func TestSeatHeldNamesEachHolderButNotTheirLogin(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", answer(http.StatusConflict, `{"detail":"This change removes human seats that `+ - `people still hold: ana (held by an open invitation (inv-1)); founder (held by jane.doe@nimbus.example); `+ - `release (held by service account svc:release).",`+ + `somebody still holds: ana (held by an open invitation (inv-1)), founder (held by jane.doe@nimbus.example) `+ + `and release (held by service account svc:release).",`+ `"error":"seat_held",`+ `"held":{"ana":{"invitation":"inv-1","expires_at":"2026-10-23T09:00:00Z"},`+ `"founder":{"person":"p-1","kind":"person","login":"jane.doe@nimbus.example","stage":"active"},`+ `"release":{"person":"p-2","kind":"machine","login":"svc:release","stage":"active"}},`+ `"hint":"Move each person to another human seat (PATCH /iam/people/{person} with seat) or remove them, `+ `unbind a service account (PATCH /iam/people/{person} with no seat), cancel an open invitation `+ - `(DELETE /iam/invitations/{id}) — each an admin's gesture — then send the write again.",`+ + `(DELETE /iam/invitations/{id}), each an admin's gesture, then send the write again.",`+ `"message":"This change removes a seat someone holds or is invited to. Move or remove them, or cancel `+ `the invitation, first. The detail says who."}`)) _, err := apply(f, write) @@ -679,8 +948,12 @@ func TestStartingWriteIsAskedAgainWhenTheEngineSays(t *testing.T) { f := newFakeEngine(t) f.handle("PUT /config", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Retry-After", "10") - answer(http.StatusServiceUnavailable, `{"error":"starting","message":"This node is starting.",`+ - `"hint":"retry after the Retry-After; /health stops answering status \"starting\" once the node serves"}`)(w, r) + answer(http.StatusServiceUnavailable, `{"error":"starting",`+ + `"message":"This node is still starting and is not serving yet. Try again in a moment.",`+ + `"detail":"This node is recovering what it holds (its store, its broker and its copy of the estate) `+ + `and serves nothing until it has started.",`+ + `"hint":"Retry after the Retry-After: once the node serves, /health stops answering status \"starting\" `+ + `and /ready stops naming reason \"starting\"."}`)(w, r) }) _, err := apply(f, write) var apiErr *engine.APIError @@ -731,13 +1004,20 @@ func TestPutSecretSendsTheRawValue(t *testing.T) { // TestPutSecretErrorsNeverContainTheValue holds the package's promise on every // way a write can fail: a refusal, an error answer, a proxy's page and a -// transport failure. +// transport failure. The engine's answers are as it words them: a value past +// what the route takes; a store too busy to take it, a condition of the node +// crewlet#142 answers 503 with, in words, where crewlet's main answered 500; +// and a fault of its own. func TestPutSecretErrorsNeverContainTheValue(t *testing.T) { const value = "sk-live-do-not-leak" cases := map[string]http.HandlerFunc{ - "refused": answer(http.StatusServiceUnavailable, - `{"error":"no_keyring","detail":"this node has no secrets.keys"}`), - "internal error": answer(http.StatusInternalServerError, `{"error":"internal_error"}`), + "refused": answer(http.StatusRequestEntityTooLarge, `{"error":"body_too_large","message":"The request body is `+ + `larger than this endpoint accepts. Send a smaller document, or split the change across more than one request."}`), + "a store too busy to take it": answerAfter(http.StatusServiceUnavailable, "2", `{"error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node.",`+ + `"detail":"This node was too busy to take this in time. Try again in a moment."}`), + "internal error": answer(http.StatusInternalServerError, `{"error":"internal_error",`+ + `"message":"Something went wrong inside the engine. The reason is in this node's log."}`), "a proxy's page": page(http.StatusBadGateway), } for name, h := range cases { @@ -782,7 +1062,7 @@ func TestPutSecretsSendsOneBody(t *testing.T) { "the re-activation lost a race": { reload: `{"error":"revision_advanced","detail":"another configuration write activated first", "stored_revision_id":"rev-3","node":"node-a"}`, - want: &engine.Reload{Error: "revision_advanced", Detail: "another configuration write activated first", + want: &engine.Reload{Error: revisionAdvanced, Detail: "another configuration write activated first", StoredRevisionID: "rev-3", Node: "node-a"}, }, } { @@ -1237,6 +1517,31 @@ func TestFleetReadsNodesAndSeatLeases(t *testing.T) { }); record != want || !record.Status.Failed() { t.Errorf("LastApply() = %+v, want %+v, failed", record, want) } + want := engine.Activation{RevisionID: "r7", Epoch: 7} + if a := got.Activation; a == nil || *a != want { + t.Errorf("Activation = %+v, want %+v", a, want) + } +} + +// TestFleetWithNoActivationNamesNone pins that a fleet whose activation +// pointer names nothing, which the engine answers as null both when nothing +// has been activated and when it could not read the pointer, has no +// Activation rather than one naming the empty revision. +func TestFleetWithNoActivationNamesNone(t *testing.T) { + for _, body := range []string{ + `{"this_node":"node-0","nodes":[],"target_epoch":0,"activation":null}`, + `{"this_node":"node-0","nodes":[],"target_epoch":0}`, + } { + f := newFakeEngine(t) + f.handle("GET /fleet", answer(http.StatusOK, body)) + got, err := f.client().Fleet(context.Background()) + if err != nil { + t.Fatal(err) + } + if got.Activation != nil { + t.Errorf("Fleet() of %s has Activation %+v, want none", body, got.Activation) + } + } } // TestRevisionReadsTheStoredDocument pins the read of one stored revision, @@ -1268,22 +1573,120 @@ func TestRevisionReadsTheStoredDocument(t *testing.T) { } // TestActiveConfigWhileTheFleetCannotBeRead pins what an engine that cannot -// read its activation pointer answers GET /config with: 503 with a -// Retry-After, never a guess, and the client's error is ErrUnavailable with -// that delay, which the config reconciler retries at rather than reporting a -// fault. +// read its activation pointer answers GET /config with, as crewlet#142 +// answers a condition of the node on a read: 503 unavailable with a +// Retry-After, the five seconds of its health tick, and the condition in words +// as the detail, never a guess. The client's error is ErrUnavailable with that +// delay and those words, which the config reconciler retries at, and quotes, +// rather than reporting a fault. func TestActiveConfigWhileTheFleetCannotBeRead(t *testing.T) { + const said = "This node couldn't reach the rest of the fleet. Try again in a moment." f := newFakeEngine(t) f.handle("GET /health", answer(http.StatusOK, `{"status":"ok","applied_epoch":3}`)) - f.handle("GET /config", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "2") - answer(http.StatusServiceUnavailable, `{"error":"unavailable","message":"try again shortly", -"reason":"the fleet's activation could not be read from the coordination store"}`)(w, r) - }) + f.handle("GET /config", answerAfter(http.StatusServiceUnavailable, "5", `{"error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node.","detail":"`+said+`"}`)) _, err := f.client().ActiveConfig(context.Background()) apiErr, ok := errors.AsType[*engine.APIError](err) - if !errors.Is(err, engine.ErrUnavailable) || !ok || apiErr.RetryAfter != 2*time.Second { - t.Fatalf("ActiveConfig() = %v, want ErrUnavailable asking again in 2s", err) + if !errors.Is(err, engine.ErrUnavailable) || !ok || apiErr.RetryAfter != 5*time.Second || apiErr.Message != said { + t.Fatalf("ActiveConfig() = %v, want ErrUnavailable asking again in 5s, in the engine's words", err) + } +} + +// unknownOutcome is the engine's answer to a write it stored and whose +// activation its coordination store did not answer (crewlet's +// internal/api/configapi, RefuseActivationUnknown): 503 unavailable with the +// unknown outcome, the revision it stored, the node whose history alone holds +// it, the revision the write was built on, and a hint saying how to look +// first, ended as httpjson.Sentence ends it. +func unknownOutcome(stored, node, base string) string { + hint := fmt.Sprintf("Revision %[1]s was stored on node %[2]s, and the coordination store did not answer its "+ + "activation, so whether the fleet runs it is unknown. GET /config names the active revision once the store "+ + "answers again. If it is not %[1]s, the revision is in node %[2]s's history alone: send GET "+ + "/config/revisions/%[1]s/diff, and POST /config/revisions/%[1]s/revert to make it the company, to that "+ + "node — by its own address where a shared one fronts several, since any other node holds no copy and "+ + "answers 404. Sending the write again makes another revision.", stored, node) + body, _ := json.Marshal(map[string]string{ + "error": "unavailable", "message": "Can't answer right now. Try again in a moment, or ask another node.", + "outcome": "unknown", "stored_revision_id": stored, "node": node, "your_base": base, "hint": hint, + }) + return string(body) +} + +// TestUnconfirmedActivationIsTheEnginesUnknownOutcome pins which answers to a +// write UnconfirmedActivation reads as one whose activation the engine sent +// and could not confirm: the 503 naming the revision it stored, and no +// other. A lost race names a stored revision too, but the engine knows it +// did not activate, and a 503 naming none stored nothing to look for. +// +// It also pins how that answer decodes: ErrUnavailable and not +// ErrRevisionAdvanced, the Retry-After of an undecided answer, and the +// engine's hint, which names the node whose history alone holds the revision. +// Read as a lost race it would send the operator to render again over a +// revision that may be its own; the wait is when a read may tell, never a +// cue to send the same write again, which from the same base loses to its +// own first attempt. +func TestUnconfirmedActivationIsTheEnginesUnknownOutcome(t *testing.T) { + for _, c := range []struct { + name, body string + status int + retryAfter string + stored string + }{ + {name: "an activation whose answer was lost", status: http.StatusServiceUnavailable, retryAfter: "2", + stored: "rev-10", body: unknownOutcome("rev-10", "node-0", "rev-9")}, + {name: "a write that lost the race", status: http.StatusConflict, + body: `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, + {name: "a seed another write beat", status: http.StatusPreconditionFailed, + body: `{"error":"already_configured","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`}, + {name: "an engine that could not store it", status: http.StatusServiceUnavailable, retryAfter: "2", + body: `{"error":"unavailable","message":"Can't answer right now. Try again in a moment, or ask another node.",` + + `"detail":"This node couldn't reach the rest of the fleet. Try again in a moment."}`}, + } { + t.Run(c.name, func(t *testing.T) { + f := newFakeEngine(t) + if c.retryAfter != "" { + f.handle("PUT /config", answerAfter(c.status, c.retryAfter, c.body)) + } else { + f.handle("PUT /config", answer(c.status, c.body)) + } + _, err := apply(f, write) + stored, ok := engine.UnconfirmedActivation(err) + if ok != (c.stored != "") || stored != c.stored { + t.Errorf("UnconfirmedActivation(%v) = %q, %t; want %q, %t", err, stored, ok, c.stored, c.stored != "") + } + if c.stored == "" { + return + } + var apiErr *engine.APIError + if !errors.As(err, &apiErr) || !errors.Is(err, engine.ErrUnavailable) || + errors.Is(err, engine.ErrRevisionAdvanced) { + t.Fatalf("ApplyConfig() = %v, want an *APIError matching ErrUnavailable and not ErrRevisionAdvanced", err) + } + var want struct{ Hint string } + if err := json.Unmarshal([]byte(c.body), &want); err != nil { + t.Fatal(err) + } + if apiErr.RetryAfter != 2*time.Second || apiErr.Hint != want.Hint || !strings.Contains(apiErr.Hint, "node-0") { + t.Errorf("APIError = {retry after %s, hint %q}, want {2s, the engine's hint naming node-0}", + apiErr.RetryAfter, apiErr.Hint) + } + }) + } + if stored, ok := engine.UnconfirmedActivation(nil); ok || stored != "" { + t.Errorf("UnconfirmedActivation(nil) = %q, %t; want nothing", stored, ok) + } +} + +// TestAReloadThatLosesItsRaceIsRevisionAdvanced pins that a reload, which +// names no revision, is a write that races others: one that loses its +// activation to another write is ErrRevisionAdvanced, as the engine answers it +// (409 revision_advanced, naming the revision it stored and the one that won). +func TestAReloadThatLosesItsRaceIsRevisionAdvanced(t *testing.T) { + f := newFakeEngine(t) + f.handle("POST /config/reload", answer(http.StatusConflict, + `{"error":"revision_advanced","current_revision_id":"rev-9","stored_revision_id":"rev-10"}`)) + if _, err := f.client().ReloadConfig(context.Background(), "a reload that loses"); !errors.Is(err, engine.ErrRevisionAdvanced) { + t.Errorf("ReloadConfig() = %v, want ErrRevisionAdvanced", err) } } @@ -1306,7 +1709,11 @@ func TestFleetLastApplyOfAnUnlistedNodeIsNone(t *testing.T) { // empty fleet: an empty one would read as every seat unplaced. func TestFleetRefusals(t *testing.T) { unavailable := newFakeEngine(t) - unavailable.handle("GET /fleet", answer(http.StatusServiceUnavailable, `{"error":"unavailable","message":"the lease table could not be read"}`)) + // The lease table is the coordination store's, and one that did not + // answer clears by the health tick's five seconds. + unavailable.handle("GET /fleet", answerAfter(http.StatusServiceUnavailable, "5", + `{"detail":"This node couldn't reach the rest of the fleet. Try again in a moment.","error":"unavailable",`+ + `"message":"Can't answer right now. Try again in a moment, or ask another node."}`)) if got, err := unavailable.client().Fleet(context.Background()); err == nil { t.Errorf("Fleet() of an engine answering 503 = %+v, want an error", got) } else if apiErr, ok := errors.AsType[*engine.APIError](err); !ok || apiErr.StatusCode != http.StatusServiceUnavailable { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index ca097ea9..e640cae3 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "net/http" "slices" "strings" "testing" @@ -234,8 +235,9 @@ func TestARecordOfTheApplyIsWaitedForAndAWrongOneFails(t *testing.T) { // TestEngineFleetNamesTheSeatsHolder pins the read of /fleet against the // engine: the answering node, its roles, broker and labels, the seat it holds -// because its labels match the seat's placement, and a seat whose placement -// no node matches, which no node holds and the engine calls unplaceable. +// because its labels match the seat's placement, a seat whose placement no +// node matches, which no node holds and the engine calls unplaceable, and the +// fleet's activation, which names the fleet's revision at its epoch. func TestEngineFleetNamesTheSeatsHolder(t *testing.T) { t.Parallel() ctx := context.Background() @@ -271,6 +273,17 @@ func TestEngineFleetNamesTheSeatsHolder(t *testing.T) { t.Errorf("bo: held %t, unplaceable %v; want held by nobody and unplaceable, as no node is labelled for its pool", held, fleet.Unplaceable) } + // The fleet's activation names the revision GET /config answers, at its + // epoch: the one read that says which epoch a revision the operator did + // not activate itself activated as, and that settles a write the engine + // could not confirm. + active, err := client.ActiveConfig(ctx) + if err != nil { + t.Fatal(err) + } + if a := fleet.Activation; a == nil || a.RevisionID != active.RevisionID || a.Epoch < 1 { + t.Errorf("Fleet().Activation = %+v, want the fleet's revision %s at its epoch", a, active.RevisionID) + } } // TestEngineApplyActivatesTheRevision pins the write's answer to what the engine @@ -340,6 +353,11 @@ func TestEngineStaleIfMatchIsRevisionAdvanced(t *testing.T) { if !errors.As(err, &apiErr) || apiErr.CurrentRevisionID != next.RevisionID { t.Fatalf("ApplyConfig() with a stale If-Match = %v, want the current revision %q named", err, next.RevisionID) } + // A failed precondition, as HTTP says of a condition the request + // stated: only a write that named no revision loses with a 409. + if apiErr.StatusCode != http.StatusPreconditionFailed { + t.Errorf("ApplyConfig() with a stale If-Match answered %d, want 412", apiErr.StatusCode) + } } // TestEngineDryRunLocatesProblems pins what attribution is built on: the engine diff --git a/internal/engine/errors.go b/internal/engine/errors.go index 13bf6f76..2f8ce45d 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -24,9 +24,13 @@ var ( // ErrRevisionAdvanced is a compare-and-set the caller lost: the active // revision is not the one an If-Match named, or another write activated - // between this write's read and its activation. Re-read the active - // configuration and render again; the engine may have kept this write's - // revision as inert history. + // between this write's read and its activation. The engine answers it + // 412 to a write whose If-Match named a revision, as the operator's + // writes over a running document do, and 409 to one that named none, as + // its activation of the running revision again (ReloadConfig) does; a + // create-only write that loses its race is ErrAlreadyConfigured instead. + // Re-read the active configuration and render again; the engine may have + // kept this write's revision as inert history. ErrRevisionAdvanced = errors.New("the engine's active configuration revision has advanced") // ErrAlreadyConfigured is a create-only write (ApplyRequest.CreateOnly) @@ -57,13 +61,47 @@ var ( // changes nothing. ErrForbidden = errors.New("the engine does not let the API token do this") - // ErrUnavailable is a request the engine could not answer now and asks - // to be sent again (503): it could not decide who is asking, or could - // not read what the request needs, such as who holds a seat a write - // takes away. The *APIError behind it carries RetryAfter when the - // engine said when. + // ErrUnavailable is a request the engine could not answer now (503): it + // could not decide who is asking, or could not read what the request + // needs, such as who holds a seat a write takes away or a store of its + // fleet's that did not answer. The *APIError behind it carries + // RetryAfter where the engine said when to ask again. Where it said + // nothing, no wait clears what it met, and its message says so: a + // condition of the node, such as a record it can't read yet, that + // somebody clears in the engine, whom the detail names where it can. + // + // A config write can be one too after the engine has stored its + // revision: the activation was sent to the fleet's coordination store, + // whose answer never came back, so the engine may run the revision or + // may not. That 503 carries the Retry-After of an undecided answer, when + // a read may tell, and names the revision in StoredRevisionID and the + // node that stored it in its hint: UnconfirmedActivation tells it from + // every other 503. What settles it is a read of the fleet's + // activation (Fleet.Activation), before the write is sent again: once + // the first did activate, the same write from the same base loses the + // race to it and is refused ErrRevisionAdvanced, and a write built on + // the revision it stored activates the same document a second time. ErrUnavailable = errors.New("the engine could not answer now") + // ErrRevisionUnreadable is a revision of the company document that the + // engine cannot open (409): sealed under a key the keyring no longer + // holds (unreadable_revision), or, met through the fleet's activation + // pointer, under a key this node was never given + // (unreadable_fleet_revision); sealed in a form this build of the engine + // does not read, most likely by a newer build, in either place + // (unknown_seal_revision); or stored without a seal, which no node reads + // (unsealed_revision, unsealed_fleet_revision). The engine answers it to + // a read of the document and to every write built from it, and sends no + // Retry-After: no wait opens the revision. A sealed one is somebody's + // document, which no write replaces, since that would discard it unread: + // what opens it is given to the engine, a key put in its keyring, which + // it reads when it starts, or the build that sealed it. One without a + // seal no key opens, and the one write that goes on over it is a whole + // document (ApplyConfig), which reads nothing from it: Unsealed reports + // it. The *APIError behind it carries the revision in RevisionID, the + // engine's account as its Message and the remedy as its Hint. + ErrRevisionUnreadable = errors.New("the engine cannot open a revision of its company document") + // ErrNotReady is an engine that is up but should not be sent work: it // is draining for a shutdown, has no company yet, or its posture is shed // or stuck. The *NotReadyError behind it names which. @@ -92,6 +130,21 @@ const ( // has its own sentinel. var codesForbidden = []string{"unauthorized", "token_refused", "seat_unavailable"} +// codesRevisionUnreadable are the refusals of a revision the engine cannot +// open, one per cause and origin because the remedies share nothing: a key +// put back in the keyring (unreadable_revision), a key the fleet seals under +// given to this node (unreadable_fleet_revision), the build that sealed it +// (unknown_seal_revision, one code for either origin, since that remedy is +// the same for both), or a whole document replacing it (codesUnsealed). +var codesRevisionUnreadable = append([]string{ + "unreadable_revision", "unreadable_fleet_revision", "unknown_seal_revision", +}, codesUnsealed...) + +// codesUnsealed are the refusals of a revision stored without a seal, this +// node's copy (unsealed_revision) or the fleet's (unsealed_fleet_revision): +// the one cause a write replaces (Unsealed). +var codesUnsealed = []string{"unsealed_revision", "unsealed_fleet_revision"} + // codesWithProblems are the refusals of a document: the engine located each // failure in it, so they become a *ValidationError. var codesWithProblems = map[string]struct{}{ @@ -105,9 +158,10 @@ var codesWithProblems = map[string]struct{}{ // engine's (*NotReadyError). // // It matches ErrUnauthorized, ErrForbidden, ErrRevisionAdvanced, -// ErrAlreadyConfigured, ErrNoActiveRevision, ErrConfigManaged and -// ErrUnavailable with errors.Is where its status and code say so, so a caller -// can branch on the sentinel and still read the engine's message. +// ErrAlreadyConfigured, ErrNoActiveRevision, ErrConfigManaged, +// ErrRevisionUnreadable and ErrUnavailable with errors.Is where its status +// and code say so, so a caller can branch on the sentinel and still read the +// engine's message. type APIError struct { // StatusCode is the HTTP status. StatusCode int @@ -131,10 +185,18 @@ type APIError struct { // named one (revision_advanced, already_configured). CurrentRevisionID string - // StoredRevisionID is the revision a write that lost the race stored and - // left inert, when the engine kept one. + // StoredRevisionID is the revision a write stored, when the engine kept + // one and did not confirm activating it: inert history beside a lost + // race (ErrRevisionAdvanced, ErrAlreadyConfigured), or, on a 503 + // (ErrUnavailable, read by UnconfirmedActivation), the revision whose + // activation the engine sent and could not confirm, which the fleet may + // run or may not. StoredRevisionID string + // RevisionID is the revision a refusal is about, when the engine named + // one: the revision it cannot open (ErrRevisionUnreadable). + RevisionID string + // ManagedBy are the token ids that may change the company document, // when the engine refused this write as config_managed. ManagedBy []string @@ -153,6 +215,10 @@ type APIError struct { // request is sent again, from a 503's Retry-After; zero when it did // not say. RetryAfter time.Duration + + // Outcome is what the engine knows of a write a 503 answers: "unknown" + // for one it may or may not have landed, empty for every other answer. + Outcome string } func (e *APIError) Error() string { @@ -187,12 +253,18 @@ func (e *APIError) Is(target error) bool { // says nothing about what the engine lets the token do. return e.StatusCode == http.StatusForbidden && slices.Contains(codesForbidden, e.Code) case ErrUnavailable: - // By status alone: a 503 is "ask again later" whoever wrote it, - // and the engine writes it under several codes (unavailable, - // identity_unavailable and the stores' own), each retried the - // same way. A draining engine's 503 is a *NotReadyError instead. + // By status alone: a 503 is "not now" whoever wrote it, and the + // engine writes it under several codes (unavailable, + // identity_unavailable, seat_holders_unavailable and the stores' + // own), each retried the same way, whether it said when or that + // no wait clears it: nothing else tells the caller that somebody + // has cleared it. A draining engine's 503 is a *NotReadyError + // instead. return e.StatusCode == http.StatusServiceUnavailable case ErrRevisionAdvanced: + // By code alone: the engine writes it 412 for a write that named + // the revision it was built on and 409 for one that named none, + // and either way what moved is the company's revision. return e.Code == codeRevisionAdvanced case ErrAlreadyConfigured: return e.Code == codeAlreadyConfigured @@ -205,10 +277,74 @@ func (e *APIError) Is(target error) bool { // Matched by code and status together: a 403 without the code is // a proxy's, and says nothing about the document. return e.StatusCode == http.StatusForbidden && e.Code == codeConfigManaged + case ErrRevisionUnreadable: + // By code alone: only the engine writes these codes, so a 409 + // without one (a proxy's) says nothing about its keyring. + return slices.Contains(codesRevisionUnreadable, e.Code) } return false } +// Unsealed is the engine's active revision, one it stores without a seal, and +// true, when err is the engine's refusal to open it (ErrRevisionUnreadable +// under unsealed_revision or unsealed_fleet_revision, naming the revision). +// The fleet's pointer names it, and no node runs it: each runs the revision it +// applied before. +// +// It is the one revision the engine cannot open that a write replaces: a +// whole document sent with ApplyConfig goes on over it, reading nothing from +// it, so no masked value is restored from it and no field carried, and the +// engine judges the write as one replacing nothing. The revision is still the +// write's base: a dry run names it as BaseRevisionID, and IfMatch is judged +// against it. Every other write is built from the active document and is +// refused over it as a read is. A revision sealed in any way the engine cannot +// open (unreadable_revision, unreadable_fleet_revision, +// unknown_seal_revision) is refused for every write, ApplyConfig included, +// since replacing it would discard a document somebody sealed, and is not +// reported here. Nor is a refusal that names no revision, on which no write +// can be conditioned; the engine always names one. +func Unsealed(err error) (string, bool) { + var apiErr *APIError + if !errors.As(err, &apiErr) || !slices.Contains(codesUnsealed, apiErr.Code) || apiErr.RevisionID == "" { + return "", false + } + return apiErr.RevisionID, true +} + +// UnconfirmedActivation is the revision a config write stored whose activation +// the engine sent to its fleet and could not confirm, and true, when err is +// that answer: a 503 naming the revision it stored (ErrUnavailable with a +// StoredRevisionID). A lost race names a stored revision too, but it is no +// 503, and the engine knows that one did not activate. +// +// The fleet may run the revision or may not, and only a read of the fleet's +// activation (Fleet.Activation) says which: a caller that wrote it keeps it +// until a read does, and sends the write again only once one says it did +// not land. +func UnconfirmedActivation(err error) (string, bool) { + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusServiceUnavailable || apiErr.StoredRevisionID == "" { + return "", false + } + return apiErr.StoredRevisionID, true +} + +// Waitless reports whether err is the engine's 503 for a condition no wait +// clears: one under a code of its own with no Retry-After, which the engine +// sends only when it knows waiting changes nothing (a record a node can't +// read yet, a node the fleet evicted), and words so ("This can't be done +// here, and waiting won't change that."). Somebody clears it in the engine, +// whom its detail names where it can tell. A write whose outcome the engine +// does not know is not one, though it carries no Retry-After either: it may +// have landed, and the engine's answer says to try it again or ask another +// node (UnconfirmedActivation reads the one a config write gets). Nor is a +// 503 that names no code, which something in front of the engine wrote. +func Waitless(err error) bool { + var apiErr *APIError + return errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusServiceUnavailable && apiErr.Code != "" && + apiErr.RetryAfter == 0 && apiErr.Outcome != outcomeUnknown +} + // NotReadyError is an engine that answered but will not take work now. type NotReadyError struct { // Reason is the engine's own word for why: draining, unconfigured, shed @@ -422,8 +558,10 @@ type refusal struct { Problems []Problem `json:"problems"` CurrentRevisionID string `json:"current_revision_id"` StoredRevisionID string `json:"stored_revision_id"` + RevisionID string `json:"revision_id"` ManagedBy []string `json:"managed_by"` Held map[string]heldAnswer `json:"held"` + Outcome string `json:"outcome"` } // heldAnswer is one holder of a seat as the engine's seat_held refusal sends @@ -483,10 +621,12 @@ func errorFromResponse(status int, header http.Header, body []byte) error { Hint: r.Hint, CurrentRevisionID: r.CurrentRevisionID, StoredRevisionID: r.StoredRevisionID, + RevisionID: r.RevisionID, ManagedBy: r.ManagedBy, Reason: r.Reason, Grants: r.Grants, RetryAfter: retryAfter(status, header), + Outcome: r.Outcome, } } diff --git a/internal/engine/seats.go b/internal/engine/seats.go index 2961a0cb..531b1cac 100644 --- a/internal/engine/seats.go +++ b/internal/engine/seats.go @@ -182,7 +182,8 @@ type pauseAnswer struct { Detail string `json:"detail"` } -// The outcomes a pause tool answers with (Crewlet's statelog outcomes). +// The outcomes a pause tool answers with, the second also a 503's of a write +// the engine may or may not have landed (Crewlet's statelog outcomes). const ( outcomeApplied = "applied" outcomeUnknown = "unknown" diff --git a/internal/engine/types.go b/internal/engine/types.go index ae42a4c6..91d6dd3d 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -236,8 +236,10 @@ type ApplyRequest struct { Summary string // IfMatch makes the write conditional on the active revision being this - // one; a stale one fails with ErrRevisionAdvanced before anything is - // stored. Empty is unconditional. Mutually exclusive with CreateOnly. + // one: a stale one fails with ErrRevisionAdvanced, a 412, before anything + // is stored; a write another activated past once it was stored fails + // with it too. Empty is unconditional. Mutually exclusive with + // CreateOnly. IfMatch string // CreateOnly makes the write land only on an engine, and a fleet, with no @@ -262,8 +264,10 @@ type ApplyResult struct { Epoch int64 // BaseRevisionID is, for a dry run, the revision the check was built on, - // empty when nothing was active. A caller that dry-runs and then applies - // with IfMatch set to it applies exactly what it checked. + // empty when nothing was active; one stored without a seal, which the + // write replaces having read nothing from it (Unsealed), included. A + // caller that dry-runs and then applies with IfMatch set to it applies + // exactly what it checked. BaseRevisionID string // Warnings are what the engine will run but a person should know about, @@ -481,6 +485,30 @@ type Fleet struct { // TargetEpoch is the activation epoch the fleet converges on, zero when // nothing has been activated or the engine could not read it. TargetEpoch int64 + + // Activation is the fleet's activation pointer as the engine read it: + // the revision it names and that revision's activation epoch, read in + // one read with TargetEpoch. Nil when nothing has been activated or the + // engine could not read the pointer, which it does not tell apart. + // + // It is the one answer to which epoch a revision the operator did not + // activate itself, such as a reload of its document, activated as: + // GET /config names the revision and not its epoch, and /health the + // epoch a node has applied, which may be another revision's. + // And it is the one answer to whether a write the engine could not + // confirm activated (ErrUnavailable naming a StoredRevisionID): the + // pointer names that revision or it does not. + Activation *Activation +} + +// Activation is one activation of the fleet's company document. +type Activation struct { + // RevisionID is the revision the activation made active. + RevisionID string + + // Epoch is the activation's epoch: every engine has applied the + // revision once its applied epoch reaches it. + Epoch int64 } // Holder returns the lease of the seat with handle and the id the engine diff --git a/internal/render/bootstrap.go b/internal/render/bootstrap.go index 4e700ec8..1e11f7bb 100644 --- a/internal/render/bootstrap.go +++ b/internal/render/bootstrap.go @@ -190,6 +190,43 @@ var ErrNoAddress = errors.New("a data pod's crewlet.yaml needs the addresses the // caller holds the pods back on it rather than start one that crash-loops. var ErrSignInNotAcknowledged = errors.New("password-only sign-in off loopback needs the class's acknowledgement") +// ErrVolumeTooSmall is a data pod's Tier A asked for with a data volume whose +// share, the stream, is under what a fresh node of Crewlet reserves on it +// (freshNodeFloor). The EngineClass API admits no such size, so what reaches +// it is a volume claimed under an older, smaller floor, kept by a running +// company (its claims cannot change but by an expansion) or stamped by a +// class stored before the floor rose, which the API server's ratcheting still +// lets update. Crewlet refuses to boot on it wherever it has a state log to +// create, and an engine that keeps a log the stream does not hold yet has +// one at every boot, so the caller holds the pods back on it, naming the +// volume, rather than roll an engine that cannot boot; growing the claim is +// what clears it. A *VolumeError carries the sizes. +var ErrVolumeTooSmall = errors.New("crewlet cannot reserve its state logs on the stream a data volume this small gives it") + +// VolumeError is ErrVolumeTooSmall for one volume: its size, the stream three +// quarters of it would give the engine, what a fresh node reserves on it, and +// the least volume that holds that. +type VolumeError struct { + // Volume is the data volume's size, the claim the pods run on. + Volume resource.Quantity + // Stream is the stream.store_max_bytes Volume gives. + Stream int64 + // Floor is what a fresh node of the class reserves on its stream. + Floor int64 + // Least is the least volume, in whole gibibytes, whose stream holds + // Floor. + Least resource.Quantity +} + +func (e *VolumeError) Error() string { + return fmt.Sprintf("%v: a data volume of %s gives the engine a stream of %d bytes, under the %d a fresh node "+ + "reserves on it; a volume needs at least %s", ErrVolumeTooSmall, e.Volume.String(), e.Stream, e.Floor, + e.Least.String()) +} + +// Unwrap makes a VolumeError ErrVolumeTooSmall for errors.Is. +func (e *VolumeError) Unwrap() error { return ErrVolumeTooSmall } + // ErrKeyringInvalid is a Tier A asked for with a keyring the engine could not // use: no key, an active key it does not hold, an id that is not a valid key // id, or material that is not base64 of 32 bytes. The keyring is a Secret's, @@ -375,6 +412,42 @@ const ( streamShareDenominator = 4 ) +// What a fresh node of Crewlet reserves on its event stream before it boots +// (its engine.BrokerFloorFor): its nine state logs' ceilings at their +// floors, and, with the objects in the stream's own bucket, that bucket +// beside them at a quarter of the stream. Crewlet refuses to boot a node on +// a stream that cannot hold them where it has a log to create: one starting +// on an empty stream directory, or one whose engine keeps a log its stream +// does not hold yet. TestContractStreamFloor holds both to the figure +// `crewlet validate` names, and TestStreamFloorFiguresAreTheEngines holds +// every hand-written comment and page that quotes one to these. +const ( + freshNodeLogsFloor int64 = 8120172544 + freshNodeBucketFloor int64 = 10826896725 +) + +// freshNodeFloor is what a fresh node of class reserves on its stream: its +// state logs at their floors where the class keeps its files in S3, and the +// object store's bucket beside them where it keeps them in the stream's own +// bucket, the default. +func freshNodeFloor(class *v1alpha1.EngineClass) int64 { + if class.Spec.Objects.Backend == v1alpha1.ObjectBackendS3 { + return freshNodeLogsFloor + } + return freshNodeBucketFloor +} + +// leastDataVolume is the least data volume, in whole gibibytes, whose share +// (streamShare) holds floor: four quarters of the floor's three, a quarter +// rounded up since the share is taken in whole quarters, then the gibibyte +// rounded up. For freshNodeLogsFloor it is 11Gi and for freshNodeBucketFloor +// 14Gi, the EngineClass's floors on data.storage.size, which +// TestContractStreamFloor holds to it. +func leastDataVolume(floor int64) resource.Quantity { + volume := (floor + streamShareNumerator - 1) / streamShareNumerator * streamShareDenominator + return *resource.NewQuantity((volume+1<<30-1)>>30<<30, resource.BinarySI) +} + // storeMaxBytesCeiling is the largest stream.store_max_bytes Crewlet starts // on (config.StoreMaxBytesCeiling, 64 TiB), a guard against a unit mistake // rather than a policy: it is more than an order of magnitude above what the @@ -456,6 +529,8 @@ const keyringKeyBytes = 32 // // It fails when an address is missing (ErrNoAddress); when an address is one // Crewlet refuses to start on (an *AddressError, ErrAddressRefused); when the +// data volume gives a stream under what a fresh node reserves on it (a +// *VolumeError, ErrVolumeTooSmall); when the // class lets people sign in with a password alone at an ExternalURL that is // not on loopback without acknowledging it (ErrSignInNotAcknowledged); and // when the keyring cannot be used (ErrKeyringInvalid): no keys, an active key @@ -468,6 +543,9 @@ func Bootstrap(in *BootstrapInput) (*BootstrapDocument, error) { if err := refusedAddress(in.ExternalURL, in.PublicURL, in.ArtifactsURL, in.AllowedOrigins); err != nil { return nil, err } + if stream, floor := streamShare(in.VolumeSize), freshNodeFloor(in.Class); stream < floor { + return nil, &VolumeError{Volume: in.VolumeSize, Stream: stream, Floor: floor, Least: leastDataVolume(floor)} + } signIn := in.Class.Spec.SignIn acknowledged := signIn.AcceptInsecure != nil && *signIn.AcceptInsecure if signIn.SecondFactor == v1alpha1.SecondFactorOptional && !acknowledged && !loopback(in.ExternalURL) { diff --git a/internal/render/bootstrap_test.go b/internal/render/bootstrap_test.go index e12c24cc..99d79cc8 100644 --- a/internal/render/bootstrap_test.go +++ b/internal/render/bootstrap_test.go @@ -653,6 +653,16 @@ func TestSeatPodsCarryNoSignIn(t *testing.T) { } } +// The least whole-GiB data volumes whose stream share holds what a fresh node +// reserves, by where a class keeps its files: the broker's bucket beside the +// logs, or S3 (leastDataVolume of freshNodeBucketFloor and +// freshNodeLogsFloor), which the EngineClass's floors on data.storage.size +// are. +const ( + leastBucketVolume = "14Gi" + leastLogsVolume = "11Gi" +) + // TestBootstrapBoundsTheStreamByTheVolume pins the event stream's share of // the volume: three quarters of the claim the pod runs on, whatever the // class says now, since the claim is what the pod was given, and never more @@ -663,7 +673,7 @@ func TestBootstrapBoundsTheStreamByTheVolume(t *testing.T) { size string want int64 }{ - {size: "12Gi", want: 9 << 30}, + {size: leastBucketVolume, want: 14 << 30 / 4 * 3}, {size: "85Ti", want: 85 << 40 / 4 * 3}, {size: "93824992236884", want: 70368744177663}, {size: "93824992236888", want: storeMaxBytesCeiling}, @@ -694,6 +704,56 @@ func TestBootstrapBoundsTheStreamByTheVolume(t *testing.T) { } } +// TestBootstrapRefusesAVolumeUnderTheStreamFloor holds the render to what a +// fresh node of Crewlet reserves on its stream, for each place a class keeps +// its files: a volume whose share is under it, which a company keeps from an +// older floor, is refused as a *VolumeError naming the volume, the stream it +// gives, what a fresh node reserves and the least whole-GiB volume that holds +// it, rather than rendered into a crewlet.yaml no node with a state log to +// create boots on; the least such volume renders. +func TestBootstrapRefusesAVolumeUnderTheStreamFloor(t *testing.T) { + for _, c := range []struct { + bootstrap, size, least string + stream, floor int64 + }{ + {bootstrap: caseNATS, size: "12Gi", stream: 12 << 30 / 4 * 3, floor: freshNodeBucketFloor, least: leastBucketVolume}, + {bootstrap: caseNATS, size: "13Gi", stream: 13 << 30 / 4 * 3, floor: freshNodeBucketFloor, least: leastBucketVolume}, + // A byte under the least volume whose share holds the reservation. + {bootstrap: caseNATS, size: "14435862299", stream: 14435862299 / 4 * 3, floor: freshNodeBucketFloor, + least: leastBucketVolume}, + {bootstrap: caseNATS, size: "0", stream: 0, floor: freshNodeBucketFloor, least: leastBucketVolume}, + {bootstrap: "s3", size: "9Gi", stream: 9 << 30 / 4 * 3, floor: freshNodeLogsFloor, least: leastLogsVolume}, + {bootstrap: "s3", size: "10826896727", stream: 10826896727 / 4 * 3, floor: freshNodeLogsFloor, least: leastLogsVolume}, + } { + t.Run(c.bootstrap+"/"+c.size, func(t *testing.T) { + in := bootstrapCaseNamed(t, c.bootstrap).input(t) + in.VolumeSize = resource.MustParse(c.size) + doc, err := Bootstrap(in) + refused, ok := errors.AsType[*VolumeError](err) + if !ok || !errors.Is(err, ErrVolumeTooSmall) || doc != nil { + t.Fatalf("Bootstrap of a %s volume = %v, %v; want a *VolumeError", c.size, doc, err) + } + if !refused.Volume.Equal(in.VolumeSize) || refused.Stream != c.stream || refused.Floor != c.floor || + refused.Least.String() != c.least { + t.Errorf("the refusal of a %s volume names %s, a stream of %d under %d and a least volume of %s; "+ + "want %s, %d under %d and %s", c.size, refused.Volume.String(), refused.Stream, refused.Floor, + refused.Least.String(), c.size, c.stream, c.floor, c.least) + } + }) + } + for _, c := range []struct{ bootstrap, size string }{ + {bootstrap: caseNATS, size: "14435862300"}, {bootstrap: caseNATS, size: leastBucketVolume}, + {bootstrap: "s3", size: "10826896728"}, {bootstrap: "s3", size: leastLogsVolume}, + } { + in := bootstrapCaseNamed(t, c.bootstrap).input(t) + in.VolumeSize = resource.MustParse(c.size) + if _, err := Bootstrap(in); err != nil { + t.Errorf("Bootstrap refuses a %s volume with files in %s, whose share holds a fresh node: %v", c.size, + c.bootstrap, err) + } + } +} + // TestDataPodsKeepTheirLeafListenerForPinnedSeats holds the data pods' leaf // listener open while the revision the engine runs pins a seat to a seat pod, // whatever the class says now: the engine keeps running that revision while diff --git a/internal/render/company.go b/internal/render/company.go index 410c7185..ca70df22 100644 --- a/internal/render/company.go +++ b/internal/render/company.go @@ -485,11 +485,19 @@ func (r *renderer) company() companyDoc { RootSpace: spec.Knowledge.RootSpace, Vectors: spec.Knowledge.Vectors, }, + Chat: chatDoc{Backend: chatBackend(spec.Chat.Backend)}, TokenBudget: tokenBudget(spec.TokenBudget), } if days := spec.Tracker.Worklet.InboxRetentionDays; days != 0 { doc.Tracker.Worklet = &trackerWorkletDoc{InboxRetentionDays: days} } + if chatlet := spec.Chat.Chatlet; chatlet != (v1alpha1.ChatletSettings{}) { + doc.Chat.Chatlet = &chatletDoc{ + MessageRetentionDays: chatlet.MessageRetentionDays, + DefaultChannelPrivate: chatlet.DefaultChannelPrivate, + TypingStatus: workingStatus(chatlet.TypingStatus), + } + } doc.Providers = r.providersDoc() doc.Assistant = r.assistant() doc.Integrations = r.integrations() diff --git a/internal/render/company_test.go b/internal/render/company_test.go index 6b5f07eb..b99f48cd 100644 --- a/internal/render/company_test.go +++ b/internal/render/company_test.go @@ -35,10 +35,10 @@ func companyCases() []companyCase { return []companyCase{ // The example company every page of the docs uses. {name: sampleCompany, exposed: true}, - {name: "minimal", advisories: agentsAlone}, + {name: caseMinimal, advisories: agentsAlone}, {name: "minimal-agent", advisories: agentsAlone}, {name: "org-chart"}, - {name: "slack", exposed: true, advisories: agentsAlone}, + {name: caseSlack, exposed: true, advisories: agentsAlone}, {name: "slack-phrases", exposed: true, advisories: agentsAlone}, {name: "github-enterprise", exposed: true, advisories: agentsAlone}, {name: "github-minimal", exposed: true, advisories: agentsAlone}, @@ -51,6 +51,7 @@ func companyCases() []companyCase { {name: caseAtlassian, exposed: true}, {name: caseSeatPools}, {name: caseOptionalKeys, advisories: []advisory{advisoryNoUnitChannel}}, + {name: "chat"}, } } diff --git a/internal/render/contract_test.go b/internal/render/contract_test.go index 5447e8de..b585c290 100644 --- a/internal/render/contract_test.go +++ b/internal/render/contract_test.go @@ -1215,6 +1215,68 @@ func TestContractGettingStarted(t *testing.T) { "-config", write(t, "crewlet.yaml", tierA.YAML), "-company", write(t, "company.yaml", doc.YAML)) } +// TestContractContactAdvisoryFollowsTheChatSurface holds what the Seat API +// and the guides say of a Human seat with no chat account to the engine: on a +// company that talks on Slack, here through its Slack Integration, Crewlet +// warns at the seat's contact of a Human seat that names no Slack account, a +// GitHub login not counting, since it says nothing of whether a Slack mention +// reaches the person; and it says nothing of one that names its Slack member +// id. On Chatlet it warns of neither, which TestContractGettingStarted +// holds. +func TestContractContactAdvisoryFollowsTheChatSurface(t *testing.T) { + tierA, err := Bootstrap(bootstrapCases()[0].input(t)) + if err != nil { + t.Fatal(err) + } + config := write(t, "crewlet.yaml", tierA.YAML) + for _, c := range []struct { + name string + contact v1alpha1.SeatContact + warns bool + }{ + {name: "a GitHub login alone", contact: v1alpha1.SeatContact{GitHubLogin: "jane-doe"}, warns: true}, + {name: "a Slack member id", contact: v1alpha1.SeatContact{SlackUserID: "U0JANEDOE", GitHubLogin: "jane-doe"}}, + } { + t.Run(c.name, func(t *testing.T) { + in := companyCaseNamed(t, caseSlack).input(t) + person := v1alpha1.Seat{ + Namespace: in.Company.Namespace, Name: "jane", + Spec: v1alpha1.SeatSpec{ + CompanyRef: v1alpha1.CompanyReference{Name: in.Company.Name}, + Kind: v1alpha1.SeatKindHuman, + Contact: c.contact, + }, + } + in.Seats = append(in.Seats, person) + doc, err := Company(in) + if err != nil { + t.Fatal(err) + } + i := slices.IndexFunc(doc.Sources, func(s Source) bool { + return s.Object == ObjectRef{Kind: KindSeat, Name: person.Name} + }) + if i < 0 { + t.Fatalf("the document renders Seat %s nowhere", person.Name) + } + var want []advisory + if c.warns { + want = append(want, advisory{ + path: doc.Sources[i].Path.String() + ".contact", + says: fmt.Sprintf("the human seat %q has no account on Slack", person.Name), + }) + } + env := bootstrapEnv(tierA) + for _, k := range doc.Refs { + env[secretsync.StoreName(k)] = dummyCredential(k) + } + for _, name := range doc.Provisioned { + env[name] = "minted-" + name + } + validate(t, env, want, "-config", config, "-company", write(t, "company.yaml", doc.YAML)) + }) + } +} + // TestContractScheduleRefusalsReachTheSchedule has the engine's dry run // refuse each of a schedule's mistakes no API rule can see, and checks that // the document attributes every problem to the Schedule, which is what the @@ -1287,6 +1349,75 @@ func TestContractScheduleRefusalsReachTheSchedule(t *testing.T) { } } +// TestContractChatSettingsFollowTheEngine has the engine's dry run judge the +// chat settings no API rule can, because each turns on the company's +// Integrations and seats. Chatlet beside a Slack Integration would split the +// company's conversations between two surfaces, Vendor on a company with no +// chat Integration has nothing to talk through, and a chatlet block with no +// backend beside a Slack Integration sets a policy for a chat the company does +// not run: each is refused where Crewlet says, and the document attributes it +// to the Company, which is where the controller reports it. A chatlet block +// with no backend on a company with no chat Integration is Chatlet's policy, +// which the API admits as the engine does, and the engine takes it. +func TestContractChatSettingsFollowTheEngine(t *testing.T) { + client := enginetest.Start(t).Client() + chatlet := v1alpha1.ChatletSettings{DefaultChannelPrivate: new(true)} + for _, c := range []struct { + name string + from string + chat v1alpha1.ChatSettings + refusedAt string + }{ + { + name: "Chatlet beside a Slack Integration", from: caseSlack, + chat: v1alpha1.ChatSettings{Backend: v1alpha1.ChatChatlet}, refusedAt: "chat.backend", + }, + { + name: "Vendor with no chat Integration", from: caseMinimal, + chat: v1alpha1.ChatSettings{Backend: v1alpha1.ChatVendor}, refusedAt: "chat.backend", + }, + { + name: "a chatlet block with no backend beside a Slack Integration", from: caseSlack, + chat: v1alpha1.ChatSettings{Chatlet: chatlet}, refusedAt: "chat.chatlet", + }, + { + name: "a chatlet block with no backend and no chat Integration", from: caseMinimal, + chat: v1alpha1.ChatSettings{Chatlet: chatlet}, + }, + } { + t.Run(c.name, func(t *testing.T) { + in := companyCaseNamed(t, c.from).input(t) + in.Company.Spec.Chat = c.chat + doc, err := Company(in) + if err != nil { + t.Fatal(err) + } + _, err = client.ApplyConfig(t.Context(), engine.ApplyRequest{Document: doc.YAML, DryRun: true}) + if c.refusedAt == "" { + if err != nil { + t.Fatalf("the dry run refused a chat the API admits: %v", err) + } + return + } + verr, ok := errors.AsType[*engine.ValidationError](err) + if !ok { + t.Fatalf("the dry run answered %v, want a refusal", err) + } + if len(verr.Problems) == 0 { + t.Fatalf("the refusal names no problem: %v", verr) + } + want := ObjectRef{Kind: KindCompany, Name: in.Company.Name} + for _, p := range verr.Problems { + got, ok := doc.Attribute(PathOf(p.Segments)) + if path := PathOf(p.Segments).String(); path != c.refusedAt || !ok || got != want { + t.Errorf("the engine's problem at %q (%s) is attributed to %v, %v; want %s, on %v", + path, p.Message, got, ok, c.refusedAt, want) + } + } + }) + } +} + // TestContractExecutorRefusalsReachTheirObject has the engine's dry run // refuse the mistakes about a seat's executor that the render leaves to it, // as no object's own spec shows them, and checks that the document attributes diff --git a/internal/render/credentials_test.go b/internal/render/credentials_test.go index 6efa152f..094f3e7e 100644 --- a/internal/render/credentials_test.go +++ b/internal/render/credentials_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "slices" "strings" "testing" @@ -332,6 +333,12 @@ func engineConfigValidator(t *testing.T) func(entries ...any) field.ErrorList { } } +// engineConfigEntry is an engineConfig entry setting the top-level key of the +// company document to an empty block. +func engineConfigEntry(key string) map[string]any { + return map[string]any{keyName: key, keyValue: map[string]any{}} +} + // TestEngineConfigCannotNameACredentialBlock: engineConfig passes a // top-level block of the company document through as written, so it may // never name a block Crewlet holds a credential under, or a literal could @@ -365,22 +372,41 @@ func TestEngineConfigCannotNameACredentialBlock(t *testing.T) { } } -// engineConfigEntry is a Company.spec.engineConfig entry setting key to an -// empty block. -func engineConfigEntry(key string) map[string]any { - return map[string]any{keyName: key, "value": map[string]any{}} -} - -// TestEngineConfigCannotNameATypedKey holds the rule on -// Company.spec.engineConfig to typedKeys: an entry under a key the render -// writes from a field would be merged over it, or replace it. -func TestEngineConfigCannotNameATypedKey(t *testing.T) { +// TestEngineConfigRefusesEveryTypedKey: the render writes every top-level key +// a field or object of this API owns, and merges a Company's engineConfig +// entries over the result, so an entry naming such a key would replace what +// the render wrote; and Diff tells a key the render writes from one an +// engineConfig entry does by typedKeys. So typedKeys must be every top-level +// key the company document's type renders, and the CRD's rule on +// spec.engineConfig must refuse each of them, while a tuning block the API +// does not type is let through. A key typed without both fails here. +func TestEngineConfigRefusesEveryTypedKey(t *testing.T) { + doc := reflect.TypeFor[companyDoc]() + rendered := make([]string, 0, doc.NumField()) + for f := range doc.Fields() { + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + rendered = append(rendered, name) + } + slices.Sort(rendered) + typed := slices.Sorted(slices.Values(typedKeys)) + if !slices.Equal(rendered, typed) { + t.Errorf("the company document renders the top-level keys %v, and typedKeys is %v", rendered, typed) + } validate := engineConfigValidator(t) + refused := func(key string) bool { + return slices.ContainsFunc(validate(engineConfigEntry(key)), func(e *field.Error) bool { + return strings.HasPrefix(e.Detail, engineConfigRefusal) + }) + } for _, key := range typedKeys { - errs := validate(engineConfigEntry(key)) - if !slices.ContainsFunc(errs, func(e *field.Error) bool { return strings.HasPrefix(e.Detail, engineConfigRefusal) }) { - t.Errorf("the render writes %s from the API, and the CRD lets engineConfig set it: add it to the rule "+ - "on Company.spec.engineConfig", key) + if !refused(key) { + t.Errorf("the render writes %s itself, and the CRD lets engineConfig set it too: add it to the rule on "+ + "Company.spec.engineConfig", key) + } + } + for _, key := range []string{"turn_engine", "learning", "scheduling", "artifacts"} { + if refused(key) { + t.Errorf("engineConfig may not set %s, a tuning block the API does not type", key) } } } diff --git a/internal/render/diff.go b/internal/render/diff.go index 56499336..4167c437 100644 --- a/internal/render/diff.go +++ b/internal/render/diff.go @@ -14,8 +14,8 @@ import ( // objects of the API render, the same list the API refuses in a Company's // engineConfig. Every other top-level key is an engineConfig entry. var typedKeys = []string{ - "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "token_budget", - keyProviders, keyIntegrations, keyMCPServers, keyRoles, keyUnits, keySeatAccessMigrated, + "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "chat", "token_budget", + "assistant", keyProviders, keyIntegrations, keyMCPServers, keyRoles, keyUnits, keySeatAccessMigrated, } // keySeatAccessMigrated is the document's list of the integration kinds whose diff --git a/internal/render/document.go b/internal/render/document.go index 6e5b3a56..a43f8cff 100644 --- a/internal/render/document.go +++ b/internal/render/document.go @@ -14,6 +14,7 @@ type companyDoc struct { Timezone string `json:"timezone,omitempty"` Tracker trackerDoc `json:"tracker,omitzero"` Knowledge knowledgeDoc `json:"knowledge,omitzero"` + Chat chatDoc `json:"chat,omitzero"` TokenBudget tokenBudgetDoc `json:"token_budget,omitzero"` Assistant assistantDoc `json:"assistant,omitzero"` Providers providersDoc `json:"providers,omitzero"` @@ -43,6 +44,17 @@ type knowledgeDoc struct { Vectors *bool `json:"vectors,omitempty"` } +type chatDoc struct { + Backend string `json:"backend,omitempty"` + Chatlet *chatletDoc `json:"chatlet,omitempty"` +} + +type chatletDoc struct { + MessageRetentionDays *int32 `json:"message_retention_days,omitempty"` + DefaultChannelPrivate *bool `json:"default_channel_private,omitempty"` + TypingStatus string `json:"typing_status,omitempty"` +} + type tokenBudgetDoc struct { Day int64 `json:"day,omitempty"` Week int64 `json:"week,omitempty"` diff --git a/internal/render/enums.go b/internal/render/enums.go index 1818d4a5..a8e34afc 100644 --- a/internal/render/enums.go +++ b/internal/render/enums.go @@ -72,6 +72,18 @@ func knowledgeBackend(b v1alpha1.KnowledgeBackend) string { return "" } +func chatBackend(b v1alpha1.ChatBackend) string { + switch b { + case v1alpha1.ChatChatlet: + return "chatlet" + case v1alpha1.ChatVendor: + return "vendor" + case v1alpha1.ChatNone: + return crewletNone + } + return "" +} + func embeddingProviderType(t v1alpha1.EmbeddingProviderType) string { switch t { case v1alpha1.EmbeddingProviderOpenAI, "": diff --git a/internal/render/helpers_test.go b/internal/render/helpers_test.go index 4bb7e344..01a5ac2e 100644 --- a/internal/render/helpers_test.go +++ b/internal/render/helpers_test.go @@ -30,6 +30,14 @@ const ( testUID = types.UID("3f2a9c1e-6b1d-4e7a-9a52-0c8d5f1e2b47") ) +// caseSlack is the company case whose people talk on Slack, and caseMinimal +// the one that connects nothing, which the contract tests start from for a +// company on a vendor's chat and one on Chatlet. +const ( + caseSlack = "slack" + caseMinimal = "minimal" +) + // The names of the Nimbus sample objects the tests break and look up. const ( sampleCompany = "nimbus" @@ -80,6 +88,9 @@ const ( // a unit's, an engineConfig block's. const keyName = "name" +// keyValue is the key an engineConfig entry holds its block under. +const keyValue = "value" + // keyLLM is the key of a model chain in the company document: a seat's, and // the assistant's. const keyLLM = "llm" diff --git a/internal/render/streamfloor_test.go b/internal/render/streamfloor_test.go index e38e719e..ea98c72c 100644 --- a/internal/render/streamfloor_test.go +++ b/internal/render/streamfloor_test.go @@ -1,6 +1,7 @@ package render import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -14,17 +15,6 @@ import ( "sigs.k8s.io/yaml" ) -// What a fresh node of Crewlet reserves on its event stream before it boots -// (its engine.BrokerFloorFor): its nine state logs' ceilings at their -// floors, and, with the objects in the stream's own bucket, that bucket -// beside them at a quarter of the stream. TestContractStreamFloor holds both -// to the figure `crewlet validate` names, and TestStreamFloorFiguresAreTheEngines -// holds every hand-written comment and page that quotes one to these. -const ( - freshNodeLogsFloor int64 = 8120172544 - freshNodeBucketFloor int64 = 10826896725 -) - // The EngineClass rules that floor the data volume, as controller-gen writes // them into the CRD: the size's own rule, the least any class boots on, and // the class's rule for a class whose objects are in the stream's bucket. @@ -44,7 +34,10 @@ var ( // as an advisory at stream.store_max_bytes rather than a refusal: the setting // is valid (above config.StoreMaxBytesFloor), and what refuses is the broker, // when the node reserves its logs and its object bucket on an empty stream -// directory. +// directory. And each floor is the least volume, in whole gibibytes, whose +// share holds what a fresh node reserves (leastDataVolume), which is where +// the render refuses a volume as one no node boots on (Bootstrap's +// *VolumeError). func TestContractStreamFloor(t *testing.T) { s3Floor, natsFloor := engineClassFloors(t) for _, c := range []struct { @@ -68,16 +61,45 @@ func TestContractStreamFloor(t *testing.T) { }}}, } { t.Run(c.bootstrap+"/"+v.size.String(), func(t *testing.T) { + // Rendered at the floor, its stream then set to v.size's share: + // the render refuses a volume under the floor (a *VolumeError), + // and what this holds is the engine's word on such a stream. in := bootstrapCaseNamed(t, c.bootstrap).input(t) - in.VolumeSize = v.size + in.VolumeSize = c.floor doc, err := Bootstrap(in) if err != nil { t.Fatal(err) } - validate(t, bootstrapEnv(doc), v.warns, "-tier", "bootstrap", write(t, "crewlet.yaml", doc.YAML)) + var tierA map[string]any + if err := yaml.Unmarshal(doc.YAML, &tierA); err != nil { + t.Fatal(err) + } + tierA["stream"].(map[string]any)["store_max_bytes"] = streamShare(v.size) + encoded, err := json.Marshal(tierA) + if err != nil { + t.Fatal(err) + } + written, err := yamlOf(encoded) + if err != nil { + t.Fatal(err) + } + validate(t, bootstrapEnv(doc), v.warns, "-tier", "bootstrap", write(t, "crewlet.yaml", written)) }) } } + for _, c := range []struct { + bootstrap string + floor resource.Quantity + reserves int64 + }{ + {bootstrap: caseNATS, floor: natsFloor, reserves: freshNodeBucketFloor}, + {bootstrap: "s3", floor: s3Floor, reserves: freshNodeLogsFloor}, + } { + if least := leastDataVolume(c.reserves); c.floor.Cmp(least) != 0 { + t.Errorf("the EngineClass's %s floor of %s is not leastDataVolume of %d, %s, where the render refuses "+ + "a volume", c.bootstrap, c.floor.String(), c.reserves, least.String()) + } + } } // engineClassFloors reads the two floors on an EngineClass's data volume out @@ -133,6 +155,7 @@ func bootstrapCaseNamed(t *testing.T, name string) *bootstrapCase { // the engine, so this is what keeps them the engine's. var streamFloorQuotes = map[string][]int64{ engineClassGo: {freshNodeLogsFloor, freshNodeBucketFloor}, + commonTypesGo: {freshNodeLogsFloor, freshNodeBucketFloor}, "docs/concepts/company-document.md": {freshNodeLogsFloor, freshNodeBucketFloor}, troubleshootingMd: {freshNodeLogsFloor, freshNodeBucketFloor}, "internal/engine/enginetest/enginetest.go": {freshNodeBucketFloor}, diff --git a/internal/render/testdata/company/chat.input.yaml b/internal/render/testdata/company/chat.input.yaml new file mode 100644 index 00000000..c5ee53c4 --- /dev/null +++ b/internal/render/testdata/company/chat.input.yaml @@ -0,0 +1,39 @@ +# A company on Chatlet, named rather than derived, with every setting of its +# policy: a horizon on its messages, rooms private unless their creator says +# otherwise, and a working indicator only where a person waits. Its one unit +# declares the room its team talks in, and its one person sits in that team. +apiVersion: crewlet.ai/v1alpha1 +kind: Company +metadata: + name: talky +spec: + engineClassName: basic + companyName: Talky + chat: + backend: Chatlet + chatlet: + messageRetentionDays: 90 + defaultChannelPrivate: true + typingStatus: Addressed +--- +apiVersion: crewlet.ai/v1alpha1 +kind: Unit +metadata: + name: team +spec: + companyRef: + name: talky + displayName: Team + channel: team +--- +apiVersion: crewlet.ai/v1alpha1 +kind: Seat +metadata: + name: founder +spec: + companyRef: + name: talky + kind: Human + displayName: Founder + unitRef: + name: team diff --git a/internal/render/testdata/company/chat.yaml b/internal/render/testdata/company/chat.yaml new file mode 100644 index 00000000..759ed795 --- /dev/null +++ b/internal/render/testdata/company/chat.yaml @@ -0,0 +1,22 @@ +chat: + backend: chatlet + chatlet: + default_channel_private: true + message_retention_days: 90 + typing_status: addressed +name: Talky +seat_access_migrated: +- atlassian +- datadog +- github +- gitlab +- mattermost +- slack +units: +- channel: team + id: team + name: Team + roles: + - handle: founder + kind: human + name: Founder diff --git a/internal/secretsync/doc.go b/internal/secretsync/doc.go index 37977159..2c133f51 100644 --- a/internal/secretsync/doc.go +++ b/internal/secretsync/doc.go @@ -73,7 +73,9 @@ // reads its store only when a revision activates, so each name removed // is recorded (State.Withdrawn) until one has, for the caller to make // one where none is applied, or the one it applied started with the -// value still there. +// value still there. Which activation came after a removal, or after a +// value written (State.Written), is told by the fleet's activation +// epoch read after it (State.Observed, State.Ran), whoever activated it. // // # What the operator owns in the store // diff --git a/internal/secretsync/engine_test.go b/internal/secretsync/engine_test.go index 8df9ca03..dd63d5aa 100644 --- a/internal/secretsync/engine_test.go +++ b/internal/secretsync/engine_test.go @@ -47,7 +47,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { } desired := desiredOf(map[string]string{slack: "xoxb", signing: "sig"}) - state, res, err := Sync(ctx, eng, desired, State{}, self) + state, res, err := Sync(ctx, eng, desired, State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -76,7 +76,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { // The engine's listing is all a second Sync has to go on besides the // state, so this is where a mismatch in how the client reports the // writer would show up as a rewrite on every reconcile. - state, res, err = Sync(ctx, eng, desired, state, self) + state, res, err = Sync(ctx, eng, desired, state, self, discard) if err != nil { t.Fatalf("second Sync: %v", err) } @@ -91,7 +91,7 @@ func TestSyncAgainstTheEngine(t *testing.T) { if err := other.PutSecret(ctx, slack, []byte("typed-by-hand")); err != nil { t.Fatalf("overwriting %s: %v", slack, err) } - _, res, err = Sync(ctx, eng, desired, state, self) + _, res, err = Sync(ctx, eng, desired, state, self, discard) if err != nil { t.Fatalf("third Sync: %v", err) } @@ -124,7 +124,7 @@ func TestContractASyncIsOneActivation(t *testing.T) { if err != nil { t.Fatal(err) } - state, res, err := Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb", signing: "signing"}), State{}, self) + state, res, err := Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb", signing: "signing"}), State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -140,7 +140,7 @@ func TestContractASyncIsOneActivation(t *testing.T) { t.Fatalf("the engine runs revision %s, want the re-activation %s", active.RevisionID, res.Reload.RevisionID) } - _, res, err = Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb-rotated", signing: "signing"}), state, self) + _, res, err = Sync(ctx, eng, desiredOf(map[string]string{read: "k1", slack: "xoxb-rotated", signing: "signing"}), state, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -182,7 +182,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { self := engine.TokenLogin(enginetest.OperatorTokenID) served := StoreName(secretref.Key{Namespace: nimbusNS, Name: "nimbus-e2b", Key: apiKeyKey}) - state, _, err := Sync(ctx, eng, desiredOf(map[string]string{served: enginetest.SandboxKeyValue}), State{}, self) + state, _, err := Sync(ctx, eng, desiredOf(map[string]string{served: enginetest.SandboxKeyValue}), State{}, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } @@ -226,7 +226,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { desired.Keep(r.RevisionID) keeping := desired keeping.KeepServed(p.RevisionID, served) - state, res, err := Sync(ctx, eng, keeping, state, self) + state, res, err := Sync(ctx, eng, keeping, state, self, discard) if err != nil { t.Fatalf("Sync keeping the served revision: %v", err) } @@ -243,7 +243,7 @@ func TestSyncKeepsWhatARefusedRevisionRuns(t *testing.T) { return servingP() }) - if _, res, err = Sync(ctx, eng, desired, state, self); err != nil || !slices.Equal(res.Deleted, []string{served}) { + if _, res, err = Sync(ctx, eng, desired, state, self, discard); err != nil || !slices.Equal(res.Deleted, []string{served}) { t.Fatalf("Sync for the fleet's revision alone = %+v, %v; want %s deleted", res, err, served) } if err := e.Restart(t); err == nil { diff --git a/internal/secretsync/resolve.go b/internal/secretsync/resolve.go index b44c268f..2fa4fa93 100644 --- a/internal/secretsync/resolve.go +++ b/internal/secretsync/resolve.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crewlet/crewlet-operator/internal/engine" "github.com/crewlet/crewlet-operator/internal/secretref" ) @@ -99,6 +100,13 @@ type Desired struct { // and no value, so Sync refuses it rather than delete every entry it // does not push. partial bool + // keepsListed is whether running, which KeepListed named then, is a + // revision nobody can read, for which every owned name the store lists + // is kept, each read as required (withListed). A bool rather than the + // revision's id beside running, and beside partial, which it shares a + // word with, keeps a Desired, a value every reconcile copies, inside the + // size a value parameter is held to. + keepsListed bool } // Names returns the names Desired pushes, sorted. @@ -192,7 +200,28 @@ func (d *Desired) keep(revision string, names []string) { d.kept = kept } -// Running is the revision Keep named, empty for none. +// KeepListed is Keep for a revision whose document nobody can read, one the +// engine stores without a seal: revision is its id, and every name the store +// lists that Owns accepts is kept, so Sync and Withdraw delete none that +// Desired pushes no value for, an absent name (Missing) and a released one +// (Release) included, whatever the State they are given records of the +// revision's reading, while Sync still pushes every value Desired holds. +// +// Which names such a revision references is what nobody can read, and it is +// not what the engine's nodes run: each runs the revision it applied before, +// which it reads again from the store when it restarts, so not even a record +// of how this revision reads a name says what they need. A name deleted while +// such a revision is the active one is a value a node may start without at +// its next boot, read as required by a revision the operator cannot see. The +// store stays as it is until a revision somebody can read runs, which the +// write that replaces this one is; the sync after that write removes what it +// lets go. +func (d *Desired) KeepListed(revision string) { + d.running = revision + d.keepsListed = true +} + +// Running is the revision Keep or KeepListed named, empty for none. func (d Desired) Running() string { return d.running } @@ -214,13 +243,30 @@ func (d Desired) Unread() []string { return slices.Sorted(maps.Keys(d.unread)) } -// Holders are the revisions kept (Keep, KeepServed) that reference name and -// read it as required, as prev records their reading (State.LetGo), sorted: -// those that keep an absent name in the store. +// Holders are the revisions kept (Keep, KeepServed, KeepListed) that +// reference name, an absent name the store holds (Result.Retained), and read +// it as required, as prev records their reading (State.LetGo), sorted: those +// that keep it in the store. The revision KeepListed named keeps every name +// the store lists that Owns accepts, and reads each as required whatever prev +// records, since nobody can read how it reads one, so it holds every such +// name. func (d Desired) Holders(name string, prev State) []string { + out := d.holders(name, prev) + if d.keepsListed && Owns(name) && !slices.Contains(out, d.running) { + out = append(out, d.running) + slices.Sort(out) + } + return out +} + +// holders is Holders by what d keeps alone: the revision KeepListed named +// holds only the names withListed kept for it, those the store lists, so that +// a name it no longer lists, which a run that stopped before saving removed, +// is not taken for one it keeps. +func (d Desired) holders(name string, prev State) []string { var out []string for revision, names := range d.kept { - if _, ok := names[name]; ok && !prev.letsGo(revision, name) { + if _, ok := names[name]; ok && (d.unreadable(revision) || !prev.letsGo(revision, name)) { out = append(out, revision) } } @@ -228,6 +274,29 @@ func (d Desired) Holders(name string, prev State) []string { return out } +// unreadable reports whether revision is the one KeepListed named, whose +// document nobody can read. +func (d Desired) unreadable(revision string) bool { + return d.keepsListed && revision == d.running +} + +// withListed is d with every name listed that Owns accepts kept for the +// revision KeepListed named, where it named one, and d itself otherwise. It +// copies what it changes (keep), so the caller's Desired keeps what it was. +func (d Desired) withListed(listed []engine.SecretInfo) Desired { + if !d.keepsListed { + return d + } + names := make([]string, 0, len(listed)) + for _, info := range listed { + if Owns(info.Name) { + names = append(names, info.Name) + } + } + d.keep(d.running, names) + return d +} + // Release names the references the revision the engine runs lets go without // whose Secret or key does not exist, as Absent reports them of // State.LetGoKeys of that revision: Sync and Withdraw remove the entry of @@ -253,11 +322,12 @@ func (d *Desired) Release(absent ...*MissingError) { } // retains reports whether name, absent, stays in the store: a revision kept -// references it (Keep, KeepServed) and reads it as required, which it does -// unless prev records it as letting the name go without (State.LetGo), or a -// node runs a revision the operator does not know (KeepUnread). +// references it (Keep, KeepServed, KeepListed) and reads it as required, +// which it does unless prev records it as letting the name go without +// (State.LetGo), or a node runs a revision the operator does not know +// (KeepUnread). func (d Desired) retains(name string, prev State) bool { - return len(d.unread) > 0 || len(d.Holders(name, prev)) > 0 + return len(d.unread) > 0 || len(d.holders(name, prev)) > 0 } // keeps reports whether a revision kept references name, or a node runs a @@ -276,10 +346,11 @@ func (d Desired) keeps(name string) bool { // letGo reports whether a revision kept references name and, as prev // records its reading, lets it go: what makes a released name one to remove, -// where no other revision kept retains it. +// where no other revision kept retains it. The revision KeepListed named lets +// nothing go: nobody can read how it reads a name, whatever prev records. func (d Desired) letGo(name string, prev State) bool { for revision, names := range d.kept { - if _, ok := names[name]; ok && prev.letsGo(revision, name) { + if _, ok := names[name]; ok && !d.unreadable(revision) && prev.letsGo(revision, name) { return true } } diff --git a/internal/secretsync/state.go b/internal/secretsync/state.go index f725c478..24b38977 100644 --- a/internal/secretsync/state.go +++ b/internal/secretsync/state.go @@ -6,10 +6,13 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "maps" + "math" "slices" + "strconv" "strings" corev1 "k8s.io/api/core/v1" @@ -38,10 +41,14 @@ const ( fingerprintKeyLen = sha256.Size // withdrawnField is the data key of the state Secret that lists the - // withdrawn names (State.Withdrawn), one per line. Like - // fingerprintKeyField it has a dash, so Owns never accepts it, and so - // have the three after it. + // withdrawn names (State.Withdrawn), and writtenField the one that lists + // the written names (State.Written): one per line, each followed, after + // a space, by the activation epoch it waits for an activation above, + // where a read of the fleet's activation pointer has followed it + // (State.Observed). Like fingerprintKeyField each has a dash, so Owns + // never accepts it, and so have the five after them. withdrawnField = "withdrawn-names" + writtenField = "written-names" // revisionField and letGoField are the data keys of the state Secret // that hold the revision whose reading of the store the State records @@ -51,15 +58,32 @@ const ( revisionField = "activated-revision" letGoField = "optional-keys" retainedField = "retained-names" + + // pendingField is the data key of the state Secret that holds the + // readings of the documents written whose activation a read is still to + // confirm (State.Pending), as one JSON object keyed by the hash the + // caller gave each. + pendingField = "pending-activations" + + // awaitField is the data key of the state Secret that lists the + // revisions the changes of the store await (State.Awaiting): one change + // and one revision per line, the name, a space and the revision. + awaitField = "awaited-revisions" ) // State is what the operator last pushed into one engine's secret store: a // fingerprint of the value of every name it pushed, and the key the -// fingerprints are computed under; the names it withdrew (Withdrawn); one +// fingerprints are computed under; the names it withdrew (Withdrawn), and +// the ones it wrote (Written), that no activation is known to have read the +// store after, with the activation epoch each waits past (Observed, Ran); one // revision of the engine's document whose reading of the store it knows, with // the Secret keys that revision lets go without (Activated, Adopted, LetGo); -// and the names it last kept for the running revision although their Secret -// or key is gone (Retained). +// the names it last kept for the running revision although their Secret or +// key is gone (Retained); the reading of each document the caller wrote +// whose revision a later read may find running without the caller having +// recorded it (Pending, Confirmed); and the revisions the engine stored for +// activations asked for after a change and could not confirm, any of which +// found running settles it (Awaiting, Found). // // It records nothing of what an engine node runs or starts on, which the // caller reads from the engine's fleet alone: a node applies the fleet's @@ -80,18 +104,68 @@ type State struct { // value its methods return changed copies of, and each copy shares it. key string fingerprints map[string][]byte - withdrawn map[string]struct{} + // withdrawn and written are the store changes no activation is known to + // have read, by name: each the epoch of the fleet's activation pointer + // read first after it, which an activation the engine runs at a greater + // epoch came after, and unread where no read has followed it yet. + withdrawn map[string]int64 + written map[string]int64 + // awaits are the revisions Awaiting recorded for those changes, by + // name: each a revision the engine stored for an activation asked for + // after the change, sorted. An entry for a name the State records no + // change of is none: a change settled otherwise leaves nothing to await. + awaits map[string][]string + // readings is how the documents the State knows of read the store, nil + // for none (readingRecord). + readings *readingRecord + retained map[string]struct{} +} + +// readingRecord is how the documents a State knows of read the store: the +// revision whose reading it records, and the documents the caller wrote whose +// revision a read may yet find running (Pending), whose reading becomes the +// revision's once one does (Confirmed). Behind one pointer, so that a State, +// which callers pass by value, stays inside the size a value parameter is +// held to; and never changed once made: a State that changes it makes another +// (withReadings). +type readingRecord struct { // revision is the revision whose reading the State records, empty for // none, and letGo the Secret keys its document lets go without, by // store name. revision string letGo map[string]secretref.Key - retained map[string]struct{} + // pending are the readings Pending recorded: the Secret keys each + // document lets go without, by store name, by the hash the caller gave + // the document. + pending map[string]map[string]secretref.Key +} + +// reading is what s records of how documents read the store, the zero record +// for nothing. +func (s State) reading() readingRecord { + if s.readings == nil { + return readingRecord{} + } + return *s.readings +} + +// withReadings is s recording r, which it does not change afterwards. +func (s State) withReadings(r readingRecord) State { + s.readings = nil + if r.revision != "" || len(r.letGo) > 0 || len(r.pending) > 0 { + s.readings = &r + } + return s +} + +// pending is what s records under Pending, nil for nothing. +func (s State) pending() map[string]map[string]secretref.Key { + return s.reading().pending } // Withdrawn are the names Sync or Withdraw removed from the store because the // optional reference each stands for has no Secret or key any more, since a -// revision last activated, sorted. +// revision activated after the removal, sorted. // // The engine reads its store when a revision activates, and starts every tool // server and executor run with what it read then: removing a name changes @@ -101,25 +175,216 @@ type State struct { // holds. So the caller activates a revision once a name is withdrawn, the // next document or the running one again, and then records it with // Activated, or with Reactivated where it activated none because the -// revision the engine runs needs no activation for it. Kept in the saved -// State, a withdrawal survives a failed activation and a restarted operator. +// revision the engine runs needs no activation for it; and any activation the +// engine is read running above the epoch the withdrawal waits past settles it +// too (Ran), whoever made it. Kept in the saved State, a withdrawal survives a +// failed activation, a lost save after it and a restarted operator. func (s State) Withdrawn() []string { return slices.Sorted(maps.Keys(s.withdrawn)) } +// Written are the names Sync wrote a value under that no activation is known +// to have read the store after, sorted: every value it pushed, whether new or +// written back over one somebody else wrote or the store lost. +// +// A written value changes nothing the engine runs until a revision activates, +// as a withdrawal does not, and a value written back leaves the fingerprints, +// and so the caller's hash of what it applied, where they were: since the +// value was replaced, a revision may have activated reading the other one, +// which it goes on running. So a revision of the caller's document is one it +// runs with every value only once a revision has activated after the last +// write (Activated, Reloaded, Ran). Sync saves the mark before the value is +// written, so a write survives a write of the document that fails or loses +// its race, a save lost after the value was written, and a restarted +// operator, which the caller's own record of what one reconcile wrote did not. +func (s State) Written() []string { + return slices.Sorted(maps.Keys(s.written)) +} + +// Reloaded is s once a revision has activated after every write and +// withdrawal it records, a revision whose document the caller does not know +// included: the engine read the store with each value written and without +// each withdrawn, so both are settled. The revision it records stays +// recorded. +func (s State) Reloaded() State { + s.withdrawn, s.written, s.awaits = nil, nil, nil + return s +} + // Reactivated is s with every withdrawal it records settled: a revision has // activated since, so the engine has read the store without them, or the // revision the engine runs is one no activation is owed for (see LetGo). // The revision it records stays recorded. func (s State) Reactivated() State { s.withdrawn = nil + s.awaits = s.awaited() + return s +} + +// unread is the mark of a change of the store no read of the fleet's +// activation pointer has followed yet: no activation is above it, so none +// settles it but one the caller makes, and the first read after it lowers it +// to the epoch read (Observed). +const unread = math.MaxInt64 + +// Observed is s once the caller has read the fleet's activation pointer at +// epoch after every change of the store s records: each change no read has +// followed yet waits for an activation above epoch, which the pointer can +// only name once a write after this read moved it, so after the change; one +// a read followed already keeps the earlier epoch. Zero is a pointer that +// names nothing. +// +// The epoch is read after the changes, never before: one read before a change +// would settle it by an activation the pointer named before it, which read +// the store without it. And it is read before the activation the caller asks +// for, and saved with s before that is sent, so that a save lost after the +// activation leaves a change that activation settles (Ran) rather than one it +// waits past. +func (s State) Observed(epoch int64) State { + s.withdrawn = observed(s.withdrawn, epoch) + s.written = observed(s.written, epoch) + return s +} + +// Ran is s once the caller has read the fleet's activation pointer at epoch, +// naming a revision it has seen opened (the one GET /config answered): every +// change that waits for an activation above a lower epoch is settled, whoever +// made that activation and whatever its document, since it came after the +// change, and every node reads the store at its attempt to apply it, a node +// that refuses it included, whose running revision resolves its references +// from what that attempt read. A revision no node can open (one sealed under +// a key the keyring lost, or stored without a seal) is not one they read the +// store for, however the pointer names it, and settles nothing. +func (s State) Ran(epoch int64) State { + s.withdrawn = ran(s.withdrawn, epoch) + s.written = ran(s.written, epoch) + s.awaits = s.awaited() + return s +} + +// Awaiting is s once the caller has asked the engine for an activation after +// every change of the store s records, and the engine stored revision for it +// and could not confirm activating it: should a read find revision running, +// it activated after every one of those changes and every node read the +// store when it applied it, so it settles them (Found), whatever the fleet's +// activation pointer says. A change a later read of the pointer has followed +// is settled by any activation above it (Ran), and needs no such record; one +// no read has followed (Observed) needs it, since no epoch the pointer names +// later can settle it, and the pointer is likeliest to be unreadable when the +// engine loses an activation's answer. A later change of a name waits for an +// activation of its own. +func (s State) Awaiting(revision string) State { + if revision == "" { + return s + } + awaits := s.awaited() + for _, marks := range []map[string]int64{s.withdrawn, s.written} { + for name := range marks { + if revisions := awaits[name]; !slices.Contains(revisions, revision) { + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = slices.Sorted(slices.Values(append(slices.Clone(revisions), revision))) + } + } + } + s.awaits = awaits return s } +// Found is s once the caller has read the engine running revision, as a +// revision its nodes can open: every change that awaits it (Awaiting) is +// settled, as Ran settles one an activation above its epoch came after. +func (s State) Found(revision string) State { + awaits := s.awaits + if revision == "" || len(awaits) == 0 { + return s + } + read := func(name string, _ int64) bool { return slices.Contains(awaits[name], revision) } + s.withdrawn = without(s.withdrawn, read) + s.written = without(s.written, read) + s.awaits = s.awaited() + return s +} + +// SameMarks reports whether s and o record the same changes of the store, +// each with the same mark and awaiting the same revisions: whether a save of +// one leaves what a save of the other did. +func (s State) SameMarks(o State) bool { + return maps.Equal(s.withdrawn, o.withdrawn) && maps.Equal(s.written, o.written) && + maps.EqualFunc(s.awaited(), o.awaited(), slices.Equal) +} + +// awaited is what s awaits (Awaiting) for the changes it records, in a map of +// its own, nil for none: an entry for a name settled otherwise is dropped. +func (s State) awaited() map[string][]string { + var awaits map[string][]string + for name, revisions := range s.awaits { + _, withdrawn := s.withdrawn[name] + _, written := s.written[name] + if len(revisions) == 0 || !withdrawn && !written { + continue + } + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = revisions + } + return awaits +} + +// Unobserved reports whether s records a change of the store no read of the +// fleet's activation pointer has followed yet (Observed). +func (s State) Unobserved() bool { + for _, marks := range []map[string]int64{s.withdrawn, s.written} { + for _, epoch := range marks { + if epoch == unread { + return true + } + } + } + return false +} + +// observed is marks with every one above epoch lowered to it, in a map of its +// own: the State's maps are values other States share. +func observed(marks map[string]int64, epoch int64) map[string]int64 { + if marks == nil { + return nil + } + next := make(map[string]int64, len(marks)) + for name, mark := range marks { + next[name] = min(mark, epoch) + } + return next +} + +// ran is marks less every one an activation at epoch is above, in a map of +// its own, nil for none. +func ran(marks map[string]int64, epoch int64) map[string]int64 { + return without(marks, func(_ string, mark int64) bool { return epoch > mark }) +} + +// without is marks less every one settled reports, in a map of its own, nil +// for none. +func without(marks map[string]int64, settled func(name string, mark int64) bool) map[string]int64 { + var left map[string]int64 + for name, mark := range marks { + if settled(name, mark) { + continue + } + if left == nil { + left = map[string]int64{} + } + left[name] = mark + } + return left +} + // Activated is s once the operator has activated revision, whose document -// lets go without the Secret keys optional: every withdrawal is settled, as -// Reactivated, and LetGo answers optional for revision from now on, and -// nothing for any other. +// lets go without the Secret keys optional: every withdrawal and every write +// is settled, as Reloaded, and LetGo answers optional for revision from now +// on, and nothing for any other. // // The caller records every activation it makes and knows the document of: an // apply, with render.Document.Optional of the document it applied, and an @@ -127,7 +392,7 @@ func (s State) Reactivated() State { // revision it read, once it has confirmed that the revision activated again // is that one. func (s State) Activated(revision string, optional []secretref.Key) State { - return s.Reactivated().Adopted(revision, optional) + return s.Reloaded().Adopted(revision, optional) } // Adopted is s recording revision, whose document lets go without the Secret @@ -136,23 +401,89 @@ func (s State) Activated(revision string, optional []secretref.Key) State { // optional, but did not activate it after the withdrawals it owes, so the // engine may have read the store with them. func (s State) Adopted(revision string, optional []secretref.Key) State { - s.revision = revision - s.letGo = make(map[string]secretref.Key, len(optional)) + r := s.reading() + r.revision = revision + r.letGo = make(map[string]secretref.Key, len(optional)) for _, k := range optional { - s.letGo[StoreName(k)] = k + r.letGo[StoreName(k)] = k } - return s + return s.withReadings(r) +} + +// Pending is s recording the reading of a document the caller is about to +// write, which lets go without the Secret keys optional, by hash, the hash the +// caller gives that document, which must tell it from every other document it +// writes: should a read find the revision the engine stored for the write +// running before the caller has recorded its activation (its answer lost, or +// the caller's record of it), Confirmed records that revision's reading from +// it. The caller records it, and saves s, before the write is sent, since a +// write whose answer is lost leaves nothing else to say how its document reads +// the store. +func (s State) Pending(hash string, optional []secretref.Key) State { + letGo := make(map[string]secretref.Key, len(optional)) + for _, k := range optional { + letGo[StoreName(k)] = k + } + pending := maps.Clone(s.pending()) + if pending == nil { + pending = map[string]map[string]secretref.Key{} + } + pending[hash] = letGo + r := s.reading() + r.pending = pending + return s.withReadings(r) +} + +// Pends reports whether s records the reading of the document hash names +// (Pending). +func (s State) Pends(hash string) bool { + _, ok := s.pending()[hash] + return ok +} + +// Confirmed is s recording revision, which the caller found running the +// document whose reading Pending recorded under hash, as reading the store +// that way, as Adopted records a revision of a document the caller renders: +// without settling a withdrawal or a write, which only a read of the epoch the +// revision activated as settles (Ran). ok is false where s records no reading +// under hash, and s is returned unchanged. +func (s State) Confirmed(revision, hash string) (next State, ok bool) { + letGo, ok := s.pending()[hash] + if !ok || revision == "" { + return s, false + } + r := s.reading() + r.revision, r.letGo = revision, letGo + return s.withReadings(r), true +} + +// KeepPending is s recording only the readings Pending recorded under hashes: +// the caller's record of the rest says no revision it has not recorded can be +// that write. +func (s State) KeepPending(hashes []string) State { + var kept map[string]map[string]secretref.Key + for _, hash := range hashes { + if letGo, ok := s.pending()[hash]; ok { + if kept == nil { + kept = map[string]map[string]secretref.Key{} + } + kept[hash] = letGo + } + } + r := s.reading() + r.pending = kept + return s.withReadings(r) } // Recorded is the revision whose reading s records, empty for none. func (s State) Recorded() string { - return s.revision + return s.reading().revision } // Records reports whether s records how revision reads the store: whether // LetGo answers for it. func (s State) Records(revision string) bool { - return revision != "" && revision == s.revision + return revision != "" && revision == s.reading().revision } // LetGo are the store names the document of revision lets go without, every @@ -172,7 +503,7 @@ func (s State) LetGo(revision string) []string { if !s.Records(revision) { return nil } - return slices.Sorted(maps.Keys(s.letGo)) + return slices.Sorted(maps.Keys(s.reading().letGo)) } // LetGoKeys are the Secret keys LetGo names the store entries of, sorted as @@ -183,41 +514,76 @@ func (s State) LetGoKeys(revision string) []secretref.Key { if !s.Records(revision) { return nil } - return slices.SortedFunc(maps.Values(s.letGo), secretref.Compare) + return slices.SortedFunc(maps.Values(s.reading().letGo), secretref.Compare) } // letsGo reports whether the document of revision lets go without name, as // LetGo says. func (s State) letsGo(revision, name string) bool { - _, ok := s.letGo[name] + _, ok := s.reading().letGo[name] return ok && s.Records(revision) } // Retained are the names the last Sync or Withdraw left in the store, // although the optional reference each stands for has no Secret or key any // more, because the revision the engine runs reads them as required, or is -// one whose reading the State does not record (Result.Retained), sorted. The caller compares them with the Retained of -// the State it started from, to tell a name it newly keeps from one it has -// already said it keeps. +// one whose reading the State does not record (Result.Retained), sorted. The +// caller compares them with the Retained of the State it started from, to +// tell a name it newly keeps from one it has already said it keeps. func (s State) Retained() []string { return slices.Sorted(maps.Keys(s.retained)) } -// withdraw records name as withdrawn in s, which it changes in place. +// withdraw records name as withdrawn in s, which it changes in place: a +// change no read has followed, whatever an earlier withdrawal of it waited +// past. func (s *State) withdraw(name string) { if s.withdrawn == nil { - s.withdrawn = map[string]struct{}{} + s.withdrawn = map[string]int64{} + } + s.withdrawn[name] = unread + s.changedAgain(name) +} + +// write records a value written under name in s, which it changes in place, +// as withdraw records a withdrawal. +func (s *State) write(name string) { + if s.written == nil { + s.written = map[string]int64{} } - s.withdrawn[name] = struct{}{} + s.written[name] = unread + s.changedAgain(name) } -// carried is a State that starts from s: its key, its record of withdrawals -// and of the activated revision, and what it last retained, in maps of its -// own where a Sync or a Withdraw changes them in place, and fingerprints. +// changedAgain records in s, which it changes in place, that name changed +// after every activation Awaiting recorded for it: none of them read it now. +func (s *State) changedAgain(name string) { + if _, ok := s.awaits[name]; ok { + awaits := maps.Clone(s.awaits) + delete(awaits, name) + if len(awaits) == 0 { + awaits = nil + } + s.awaits = awaits + } +} + +// deleted records name removed from the store in s, which it changes in +// place: what was written under it is no value any revision reads now, and +// a withdrawal, where it is one, is what an activation owes for it. +func (s *State) deleted(name string) { + delete(s.fingerprints, name) + delete(s.written, name) +} + +// carried is a State that starts from s: its key, its record of withdrawals, +// of writes and the revisions they await, and of how documents read the +// store, and what it last retained, in maps of its own where a Sync or a +// Withdraw changes them in place, and fingerprints. func (s State) carried(fingerprints map[string][]byte) State { return State{ - key: s.key, fingerprints: fingerprints, withdrawn: maps.Clone(s.withdrawn), - revision: s.revision, letGo: s.letGo, retained: s.retained, + key: s.key, fingerprints: fingerprints, withdrawn: maps.Clone(s.withdrawn), written: maps.Clone(s.written), + awaits: s.awaits, readings: s.readings, retained: s.retained, } } @@ -331,13 +697,78 @@ func LoadState(ctx context.Context, r client.Reader, ref types.NamespacedName, o } // A line that is no name the operator owns, or no Secret key, is // dropped, as a malformed fingerprint is: only a hand edit writes one, - // and no entry of it can be the operator's to have withdrawn, let go or - // retained. - s.withdrawn = ownedLines(secret.Data[withdrawnField]) - s.revision = string(secret.Data[revisionField]) - s.letGo = keyLines(secret.Data[letGoField]) + // and no entry of it can be the operator's to have withdrawn, written, + // let go or retained. + s.withdrawn = markLines(secret.Data[withdrawnField]) + s.written = markLines(secret.Data[writtenField]) + s.awaits = awaitLines(secret.Data[awaitField]) + s.awaits = s.awaited() s.retained = ownedLines(secret.Data[retainedField]) - return s, nil + return s.withReadings(readingRecord{ + revision: string(secret.Data[revisionField]), letGo: keyLines(secret.Data[letGoField]), + pending: pendingOf(secret.Data[pendingField]), + }), nil +} + +// pendingRecord is one reading State.Pending recorded, as the state Secret +// holds it: Secret keys as secretref.Key prints them. +type pendingRecord struct { + Optional []string `json:"optional"` +} + +// pendingOf reads the readings State.Pending recorded from data, nil for none. +// What does not decode is dropped, as a malformed line of the other fields +// is: only a hand edit writes it, and the cost is the one State.Confirmed +// would have saved, a revision read as requiring every name it references. +func pendingOf(data []byte) map[string]map[string]secretref.Key { + var records map[string]pendingRecord + if json.Unmarshal(data, &records) != nil { + return nil + } + var pending map[string]map[string]secretref.Key + for hash, r := range records { + if hash == "" { + continue + } + letGo := map[string]secretref.Key{} + for _, line := range r.Optional { + if k, ok := parseKey(line); ok { + letGo[StoreName(k)] = k + } + } + if pending == nil { + pending = map[string]map[string]secretref.Key{} + } + pending[hash] = letGo + } + return pending +} + +// pendingData is s's pending readings as the state Secret holds them, keyed +// by hash, and nothing for none, as the other lists are; encoding/json writes +// the keys sorted, so a State saves the same bytes whatever order its maps +// hold them in. +func (s State) pendingData() []byte { + if len(s.pending()) == 0 { + return []byte{} + } + records := make(map[string]pendingRecord, len(s.pending())) + for hash, letGo := range s.pending() { + keys := slices.SortedFunc(maps.Values(letGo), secretref.Compare) + optional := make([]string, len(keys)) + for i, k := range keys { + optional[i] = k.String() + } + records[hash] = pendingRecord{Optional: optional} + } + data, err := json.Marshal(records) + if err != nil { + // A map of strings to a struct of strings always encodes; were it + // ever not to, nothing pending is the State's zero, which costs a + // revision read as requiring every name and nothing else. + return []byte{} + } + return data } // keyLines are the lines of data that are Secret keys as secretref.Key prints @@ -377,6 +808,81 @@ func parseKey(line string) (secretref.Key, bool) { return k, true } +// markLines are the lines of data that are names Owns accepts, each with the +// epoch after it, nil for none. A name with no epoch after it, or one that is +// not a number no lower than zero, is unread: only a hand edit writes the +// second, and reading it as an epoch could settle a change no activation +// read. +func markLines(data []byte) map[string]int64 { + var marks map[string]int64 + for line := range strings.Lines(string(data)) { + name, epoch, read := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + if !Owns(name) { + continue + } + mark := int64(unread) + if n, err := strconv.ParseInt(epoch, 10, 64); read && err == nil && n >= 0 { + mark = n + } + if marks == nil { + marks = map[string]int64{} + } + marks[name] = mark + } + return marks +} + +// marked is marks one per line, sorted by name, each followed by its epoch +// where a read has followed it, never nil. +func marked(marks map[string]int64) []byte { + out := []byte{} + for _, name := range slices.Sorted(maps.Keys(marks)) { + out = append(out, name...) + if mark := marks[name]; mark != unread { + out = append(out, ' ') + out = strconv.AppendInt(out, mark, 10) + } + out = append(out, '\n') + } + return out +} + +// awaitLines are the lines of data that are a name Owns accepts and a +// revision, the revisions of each name sorted, nil for none. A line that is +// not one is dropped: only a hand edit writes it, and the cost is the one a +// lost record of an unconfirmed activation costs, an activation asked for +// again. +func awaitLines(data []byte) map[string][]string { + var awaits map[string][]string + for line := range strings.Lines(string(data)) { + name, revision, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + if !ok || !Owns(name) || revision == "" || strings.ContainsAny(revision, " \t") || + slices.Contains(awaits[name], revision) { + continue + } + if awaits == nil { + awaits = map[string][]string{} + } + awaits[name] = append(awaits[name], revision) + } + for name := range awaits { + slices.Sort(awaits[name]) + } + return awaits +} + +// awaitedLines is awaits one change and one revision per line, sorted by +// name and then revision, never nil. +func awaitedLines(awaits map[string][]string) []byte { + out := []byte{} + for _, name := range slices.Sorted(maps.Keys(awaits)) { + for _, revision := range awaits[name] { + out = append(out, name+" "+revision+"\n"...) + } + } + return out +} + // ownedLines are the lines of data that are names Owns accepts, as a set, // nil for none. func ownedLines(data []byte) map[string]struct{} { @@ -427,25 +933,29 @@ func SaveState(ctx context.Context, c client.Client, ref types.NamespacedName, o if err != nil { return fmt.Errorf("finding the kind of the owner of the secret sync state %s: %w", ref, err) } - data := make(map[string][]byte, len(s.fingerprints)+5) + data := make(map[string][]byte, len(s.fingerprints)+7) maps.Copy(data, s.fingerprints) if len(s.key) > 0 { data[fingerprintKeyField] = []byte(s.key) } // Written when they are empty too, so this field manager owns them: a // list it left out would stay where a hand edit put it, and every - // reconcile would activate a revision for names it withdrew long ago, - // or take a revision whose reading it does not know for one that lets - // a name go. - data[withdrawnField] = lines(s.Withdrawn()) - data[revisionField] = []byte(s.revision) - keys := slices.SortedFunc(maps.Values(s.letGo), secretref.Compare) + // reconcile would activate a revision for names it withdrew or wrote + // long ago, or take a revision whose reading it does not know for one + // that lets a name go. + data[withdrawnField] = marked(s.withdrawn) + data[writtenField] = marked(s.written) + data[awaitField] = awaitedLines(s.awaited()) + reading := s.reading() + data[revisionField] = []byte(reading.revision) + keys := slices.SortedFunc(maps.Values(reading.letGo), secretref.Compare) printed := make([]string, len(keys)) for i, k := range keys { printed[i] = k.String() } data[letGoField] = lines(printed) data[retainedField] = lines(s.Retained()) + data[pendingField] = s.pendingData() secret := corev1ac.Secret(ref.Name, ref.Namespace). WithLabels(map[string]string{ "app.kubernetes.io/name": "crewlet", diff --git a/internal/secretsync/state_test.go b/internal/secretsync/state_test.go index 8f17e5eb..fd843792 100644 --- a/internal/secretsync/state_test.go +++ b/internal/secretsync/state_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/hmac" "crypto/sha256" + "encoding/json" "errors" "fmt" "maps" @@ -153,6 +154,134 @@ func TestSaveStateKeepsTheWithdrawnNames(t *testing.T) { } } +// TestSaveStateKeepsTheWrittenNames: the names written since a revision +// last activated after them survive a save and a load, each with the epoch +// it waits past or none, so a write of the document that fails, or an +// operator that restarts, leaves them owed exactly as they were; a line that +// names no entry the operator owns, which only a hand edit writes, is +// dropped, one whose epoch is no number no lower than zero waits for a read +// as one no read followed does, and an activation empties the list. +func TestSaveStateKeepsTheWrittenNames(t *testing.T) { + ctx := context.Background() + c := newStateClient(t) + eng := newFakeEngine() + state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), State{}) + state, _ = mustSync(t, eng, desiredOf(map[string]string{nameA: "a1", nameB: "b1"}), state.Observed(3)) + if err := SaveState(ctx, c, stateRef, company(), state); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + if want := nameA + " 3\n" + nameB + "\n"; string(secret.Data[writtenField]) != want { + t.Fatalf("written-names = %q, want %q", secret.Data[writtenField], want) + } + secret.Data[writtenField] = append(secret.Data[writtenField], []byte("not-a-store-name 4\n"+nameC+" -1\n")...) + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if !slices.Equal(loaded.Written(), []string{nameA, nameB, nameC}) { + t.Fatalf("Written() after a load = %v, want [%s %s %s]", loaded.Written(), nameA, nameB, nameC) + } + if got := loaded.Ran(4).Written(); !slices.Equal(got, []string{nameB, nameC}) { + t.Fatalf("Written() after a load and an activation at epoch 4 = %v, want [%s %s]", got, nameB, nameC) + } + + if err := SaveState(ctx, c, stateRef, company(), loaded.Activated(revision1, nil)); err != nil { + t.Fatalf("SaveState: %v", err) + } + if got := stateSecret(t, c).Data[writtenField]; len(got) != 0 { + t.Errorf("the state Secret keeps the written names %q after a revision activated", got) + } +} + +// TestARevisionStoredForAnActivationSettlesTheChangesItWasAskedAfter: the +// engine stored a revision for an activation the operator asked for after a +// withdrawal and a write and could not confirm activating it, and no read of +// the fleet's pointer followed either change, so no epoch the pointer names +// later settles them (Ran). Finding that revision running settles both +// (Found), and finding any other settles neither; a change made after the +// activation was asked for waits for one of its own; what is awaited survives +// a save and a load, a line a hand edit wrote is dropped, and nothing of it +// is kept once the changes are settled otherwise. Without the record, the +// operator activated the same document again for changes that activation had +// already read. +func TestARevisionStoredForAnActivationSettlesTheChangesItWasAskedAfter(t *testing.T) { + ctx := context.Background() + c := newStateClient(t) + state := State{key: newFingerprintKey()} + state.withdraw(nameA) + state.write(nameB) + awaiting := state.Awaiting(revision2) + if awaiting.SameMarks(state) { + t.Error("SameMarks() = true for a state that awaits a revision beside one that awaits none") + } + if got := awaiting.Ran(9); !slices.Equal(got.Withdrawn(), []string{nameA}) || !slices.Equal(got.Written(), []string{nameB}) { + t.Fatalf("Ran(9) settled %v and %v of changes no read followed; want both owed", got.Withdrawn(), got.Written()) + } + if got := awaiting.Found(revision1); len(got.Withdrawn()) != 1 || len(got.Written()) != 1 { + t.Errorf("Found(%s) left %v and %v, want both owed: another revision is no proof the request landed", + revision1, got.Withdrawn(), got.Written()) + } + if got := awaiting.Found(revision2); len(got.Withdrawn()) != 0 || len(got.Written()) != 0 || len(got.awaited()) != 0 { + t.Errorf("Found(%s) left %v, %v and awaits %v, want both settled and nothing awaited", + revision2, got.Withdrawn(), got.Written(), got.awaited()) + } + + later := awaiting.carried(nil) + later.write(nameB) + if got := later.Found(revision2); len(got.Withdrawn()) != 0 || !slices.Equal(got.Written(), []string{nameB}) { + t.Errorf("after %s was written again, Found(%s) left %v and %v; want the later write owed alone", + nameB, revision2, got.Withdrawn(), got.Written()) + } + + if err := SaveState(ctx, c, stateRef, company(), awaiting.Awaiting(revision1)); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + if want := nameA + " " + revision1 + "\n" + nameA + " " + revision2 + "\n" + nameB + " " + revision1 + "\n" + nameB + + " " + revision2 + "\n"; string(secret.Data[awaitField]) != want { + t.Fatalf("%s = %q, want %q", awaitField, secret.Data[awaitField], want) + } + secret.Data[awaitField] = append(secret.Data[awaitField], []byte("not-a-store-name r3\n"+nameA+"\n"+nameC+" r3\n")...) + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + want := map[string][]string{nameA: {revision1, revision2}, nameB: {revision1, revision2}} + if !maps.EqualFunc(loaded.awaited(), want, slices.Equal) { + t.Fatalf("awaits after a load = %v, want %v: a malformed line, and one for a change the state does not record, "+ + "dropped", loaded.awaited(), want) + } + if got := loaded.Found(revision1); len(got.Withdrawn()) != 0 || len(got.Written()) != 0 { + t.Errorf("Found(%s) after a load left %v and %v, want both settled", revision1, got.Withdrawn(), got.Written()) + } + + for name, settled := range map[string]State{ + "Reloaded": loaded.Reloaded(), + "Reactivated": loaded.Reactivated(), + "Ran": loaded.Observed(4).Ran(5), + } { + if err := SaveState(ctx, c, stateRef, company(), settled); err != nil { + t.Fatalf("SaveState: %v", err) + } + saved := awaitLines(stateSecret(t, c).Data[awaitField]) + for awaited := range saved { + if !slices.Contains(settled.Withdrawn(), awaited) && !slices.Contains(settled.Written(), awaited) { + t.Errorf("after %s the state Secret holds %s for %s, a change it settled", name, awaitField, awaited) + } + } + if name == "Reactivated" && !slices.Equal(saved[nameB], []string{revision1, revision2}) { + t.Errorf("after Reactivated %s awaits %v, want the write still awaiting both revisions", nameB, saved[nameB]) + } + } +} + // TestActivationIsRecordedForItsRevisionAlone: what Activated records of the // operator's last activation answers for that revision and for no other, an // empty one included, the Secret keys it lets go as well as their names, @@ -221,7 +350,7 @@ func TestActivationIsRecordedForItsRevisionAlone(t *testing.T) { if err := SaveState(ctx, c, stateRef, company(), State{key: next.key}); err != nil { t.Fatalf("SaveState: %v", err) } - for _, field := range []string{revisionField, letGoField, retainedField} { + for _, field := range []string{revisionField, letGoField, retainedField, pendingField, awaitField} { if got, ok := stateSecret(t, c).Data[field]; !ok || len(got) != 0 { t.Errorf("the state Secret holds %s = %q (present: %t), want it empty, and owned", field, got, ok) } @@ -253,6 +382,129 @@ func TestAdoptionRecordsARevisionWithoutSettlingAWithdrawal(t *testing.T) { } } +// TestAPendingReadingRecordsTheRevisionFoundRunningIt: the reading of a +// document about to be written is recorded under the hash the caller gives it +// (Pending), and once the caller finds a revision running that document +// (Confirmed) the State records that revision's reading from it, as Adopted +// records a revision of a document the caller renders: LetGo answers the +// document's optional keys for it, and no withdrawal or write is settled, +// since only a read of the epoch the revision activated as says it came after +// them. The readings stay for the caller to drop, a hash never pending +// confirms nothing, KeepPending keeps only the hashes it names, and a Sync +// carries what is pending. +func TestAPendingReadingRecordsTheRevisionFoundRunningIt(t *testing.T) { + const hash2, hash3 = "hash-of-revision-2", "hash-of-revision-3" + base := State{key: newFingerprintKey()}.Activated(revision1, nil) + base.withdraw(nameA) + base.write(nameB) + pending := base.Pending(hash2, []secretref.Key{keyC}).Pending(hash3, nil) + if len(base.pending()) != 0 || base.Pends(hash2) { + t.Fatalf("Pending changed the State it was made from: %v", base.pending()) + } + if got := slices.Sorted(maps.Keys(pending.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Fatalf("pending = %v, want [%s %s]", got, hash2, hash3) + } + if !pending.Pends(hash2) || !pending.Pends(hash3) || pending.Pends(revision2) { + t.Errorf("Pends(%s), Pends(%s), Pends(%s) = %t, %t, %t; want true, true, false", hash2, hash3, revision2, + pending.Pends(hash2), pending.Pends(hash3), pending.Pends(revision2)) + } + + confirmed, ok := pending.Confirmed(revision2, hash2) + if !ok { + t.Fatalf("Confirmed(%s, %s) found no pending reading", revision2, hash2) + } + if confirmed.Recorded() != revision2 || !slices.Equal(confirmed.LetGo(revision2), []string{nameC}) { + t.Errorf("after the confirmation Recorded() = %s and LetGo(%s) = %v, want %s and [%s]", + confirmed.Recorded(), revision2, confirmed.LetGo(revision2), revision2, nameC) + } + if !slices.Equal(confirmed.Withdrawn(), []string{nameA}) || !slices.Equal(confirmed.Written(), []string{nameB}) { + t.Errorf("after the confirmation Withdrawn() = %v and Written() = %v, want [%s] and [%s]: a confirmation "+ + "settles nothing", confirmed.Withdrawn(), confirmed.Written(), nameA, nameB) + } + if got := slices.Sorted(maps.Keys(confirmed.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Errorf("the confirmation left %v pending, want [%s %s]: dropping them is the caller's", got, hash2, hash3) + } + if pending.Recorded() != revision1 { + t.Errorf("Confirmed changed the State it was made from: Recorded() = %s", pending.Recorded()) + } + for _, c := range []struct{ revision, hash string }{{revision2, "no-such-hash"}, {"", hash2}} { + if same, ok := pending.Confirmed(c.revision, c.hash); ok || same.Recorded() != revision1 { + t.Errorf("Confirmed(%q, %q) = %t, recorded %s; want nothing changed", c.revision, c.hash, ok, same.Recorded()) + } + } + kept := pending.KeepPending([]string{hash3, "no-such-hash"}) + if got := slices.Sorted(maps.Keys(kept.pending())); !slices.Equal(got, []string{hash3}) { + t.Errorf("KeepPending([%s no-such-hash]) keeps %v pending, want [%s]", hash3, got, hash3) + } + if len(pending.pending()) != 2 { + t.Errorf("KeepPending changed the State it was made from: %v", pending.pending()) + } + + eng := newFakeEngine() + synced, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), base.Pending(hash2, []secretref.Key{keyC})) + if after, ok := synced.Confirmed(revision2, hash2); !ok || !slices.Equal(after.LetGo(revision2), []string{nameC}) { + t.Errorf("Confirmed(%s, %s) after a Sync = %t, let go %v; want the reading carried", revision2, hash2, ok, + after.LetGo(revision2)) + } +} + +// TestPendingReadingsSurviveASaveAndALoad: what Pending records survives a +// save and a load, what only a hand edit writes (a line that is no Secret key, +// a reading under no hash) is dropped, and a record that is not JSON at all is +// nothing pending. +func TestPendingReadingsSurviveASaveAndALoad(t *testing.T) { + ctx := context.Background() + const hash2, hash3 = "hash-of-revision-2", "hash-of-revision-3" + pending := State{key: newFingerprintKey()}.Activated(revision1, nil). + Pending(hash2, []secretref.Key{keyC}).Pending(hash3, nil) + c := newStateClient(t) + if err := SaveState(ctx, c, stateRef, company(), pending); err != nil { + t.Fatalf("SaveState: %v", err) + } + secret := stateSecret(t, c) + var records map[string]map[string]any + if err := json.Unmarshal(secret.Data[pendingField], &records); err != nil { + t.Fatalf("the state Secret holds %s = %q, not JSON: %v", pendingField, secret.Data[pendingField], err) + } + records[hash2]["optional"] = append(records[hash2]["optional"].([]any), "not-a-key") + records[""] = map[string]any{"optional": []any{keyC.String()}} + var err error + if secret.Data[pendingField], err = json.Marshal(records); err != nil { + t.Fatal(err) + } + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + loaded, err := LoadState(ctx, c, stateRef, company()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if got := slices.Sorted(maps.Keys(loaded.pending())); !slices.Equal(got, []string{hash2, hash3}) { + t.Fatalf("pending after a load = %v, want [%s %s]", got, hash2, hash3) + } + if got := slices.Sorted(maps.Keys(loaded.pending()[hash2])); !slices.Equal(got, []string{nameC}) { + t.Errorf("the reading pending under %s after a load lets go %v, want [%s]", hash2, got, nameC) + } + if len(loaded.pending()[hash3]) != 0 { + t.Errorf("the reading pending under %s after a load lets go %v, want nothing", hash3, loaded.pending()[hash3]) + } + again, _ := loaded.Confirmed(revision2, hash2) + want, _ := pending.Confirmed(revision2, hash2) + if !slices.Equal(again.LetGoKeys(revision2), want.LetGoKeys(revision2)) { + t.Errorf("Confirmed(%s, %s) after a load lets go %v, want %v", revision2, hash2, again.LetGoKeys(revision2), + want.LetGoKeys(revision2)) + } + + secret = stateSecret(t, c) + secret.Data[pendingField] = []byte("{not json") + if err := c.Update(ctx, secret); err != nil { + t.Fatalf("editing the state: %v", err) + } + if loaded, err = LoadState(ctx, c, stateRef, company()); err != nil || len(loaded.pending()) != 0 { + t.Errorf("LoadState over a pending record that is not JSON = %v, %v; want nothing pending", loaded.pending(), err) + } +} + // TestSaveStateForgetsWhatTheStateDropped: an entry the state no longer has // must leave the Secret too, or a later load would vouch for a name the // store no longer holds. @@ -269,7 +521,8 @@ func TestSaveStateForgetsWhatTheStateDropped(t *testing.T) { t.Fatalf("SaveState: %v", err) } got := slices.Sorted(maps.Keys(stateSecret(t, c).Data)) - want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, revisionField, letGoField, retainedField})) + want := slices.Sorted(slices.Values([]string{nameA, fingerprintKeyField, withdrawnField, writtenField, revisionField, + letGoField, retainedField, pendingField, awaitField})) if !slices.Equal(got, want) { t.Fatalf("state Secret keys = %v, want %v", got, want) } diff --git a/internal/secretsync/sync.go b/internal/secretsync/sync.go index 2481aa27..4cd3c5eb 100644 --- a/internal/secretsync/sync.go +++ b/internal/secretsync/sync.go @@ -120,7 +120,9 @@ type Overwrite struct { // a value somebody else put there for its own and never notice. Then it // deletes every name the engine lists that Owns accepts and desired neither // pushes nor keeps, every absent name (Desired.Missing) but one a revision -// kept reads as required (Desired.Keep, Desired.KeepServed), which it leaves +// kept reads as required (Desired.Keep, Desired.KeepServed, and every name the +// store lists for a revision nobody can read, Desired.KeepListed), which it +// leaves // (Result.Retained, State.Retained), and every name a revision kept lets go // whose Secret or key is gone (Desired.Release) that no other revision kept // reads as required; it records each absent or released name it deleted as @@ -143,10 +145,11 @@ type Overwrite struct { // one activation however many requests carry it. Where the last request // re-activates nothing, because the revision reads none of its names, or its // re-activation fails, Result.Reload says so and the caller activates the -// revision itself. A value written deferred is recorded as written (a -// fingerprint in the State) only once the last write request has been -// answered: a sync that fails, or stops, before that leaves it to be written -// again by the next, whose last request activates it. The engine re-activates +// revision itself. A value written deferred gets its fingerprint in the +// State only once the last write request has been answered: a sync that +// fails, or stops, before that leaves it to be written again by the next, +// whose last request activates it. Every value is recorded as owed +// (State.Written) before the first request, as below. The engine re-activates // the revision at the removal of a value it reads too, one removal at a time, // which is what Result.Reload reports where one comes last. // @@ -160,7 +163,20 @@ type Overwrite struct { // state: a retry from it repeats only what is not known to have happened. // A name whose write failed has no fingerprint in it, because the engine may // or may not have stored the value. -func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, self string) (State, Result, error) { +// +// Before it writes the first value, Sync hands save the State with every name +// it is about to write recorded as written (State.Written), and the +// fingerprints as they were, and writes nothing unless save took it. A value +// changes nothing the engine runs until a revision activates after it, and +// one written back leaves its fingerprint where it was: recorded only after +// the write, a save lost between the two left a value no revision read and +// nothing that said so, and the next Sync, finding the store holding its own +// value under the fingerprint it had, had nothing left to write. A name saved +// as written whose value never reached the store owes an activation for +// nothing, which costs one; the other way cost the engine the value. +func Sync( + ctx context.Context, eng EngineSecrets, desired Desired, prev State, self string, save func(State) error, +) (State, Result, error) { var res Result if desired.partial { return prev, res, errors.New("syncing the engine's secret store: the values were not all read, " + @@ -170,6 +186,7 @@ func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, s if err != nil { return prev, res, fmt.Errorf("listing the engine's secret store: %w", err) } + desired = desired.withListed(listing) listed := make(map[string]engine.SecretInfo, len(listing)) for _, info := range listing { listed[info.Name] = info @@ -199,7 +216,11 @@ func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, s } } - if err := push(ctx, eng, desired, listed, &next, self, &res); err != nil { + writes := next.toWrite(desired, listed, self) + if err := next.writeAhead(writes, save); err != nil { + return next, res, err + } + if err := push(ctx, eng, desired, writes, listed, &next, self, &res); err != nil { return next, res, err } @@ -233,32 +254,35 @@ func Sync(ctx context.Context, eng EngineSecrets, desired Desired, prev State, s return next, res, fmt.Errorf("deleting %s, %s, from the engine's secret store: %w", name, what, err) } res.removed(removed) - delete(next.fingerprints, name) + next.deleted(name) res.Deleted = append(res.Deleted, name) if absent { next.withdraw(name) } } res.Retained, next.retained = retained, nameSet(retained) + // A value written that the store no longer holds, and that this sync + // did not write again, is none the next activation reads. + for name := range next.written { + if _, ok := next.fingerprints[name]; !ok { + delete(next.written, name) + } + } return next, res, nil } -// push writes every value of desired that the engine does not hold as the -// operator last wrote it, as Sync says, recording each written in next and -// in res, with the engine's last re-activation; a failed write stops it. +// push writes the values of desired named by writes (toWrite), as Sync +// says, recording each written in next and in res, with the engine's last +// re-activation; a failed write stops it. func push( - ctx context.Context, eng EngineSecrets, desired Desired, listed map[string]engine.SecretInfo, next *State, - self string, res *Result, + ctx context.Context, eng EngineSecrets, desired Desired, writes []string, listed map[string]engine.SecretInfo, + next *State, self string, res *Result, ) error { var single, batched []string fingerprints := map[string][]byte{} - for _, name := range desired.Names() { + for _, name := range writes { value := desired.values[name] fp := next.fingerprint(name, value) - info, ok := listed[name] - if ok && info.OperatorID == self && hmac.Equal(next.fingerprints[name], fp) { - continue - } // Until the write is answered the engine may or may not hold it. delete(next.fingerprints, name) fingerprints[name] = fp @@ -354,6 +378,43 @@ func sortWritten(res *Result) { slices.SortFunc(res.Overwritten, func(a, b Overwrite) int { return strings.Compare(a.Name, b.Name) }) } +// toWrite are the names of desired Sync writes, given what the engine lists: +// those it does not list, lists as written with a credential other than self, +// or holds under another fingerprint than s does. +func (s State) toWrite(desired Desired, listed map[string]engine.SecretInfo, self string) []string { + var writes []string + for _, name := range desired.Names() { + info, ok := listed[name] + if !ok || info.OperatorID != self || !hmac.Equal(s.fingerprints[name], s.fingerprint(name, desired.values[name])) { + writes = append(writes, name) + } + } + return writes +} + +// writeAhead records every name of writes as written in s, which it changes +// in place, once save has taken a State of its own recording them so, the +// fingerprints as they were: the writes Sync makes next leave what save took +// as it was. It records and saves nothing for no writes, and records nothing +// where save fails. +func (s *State) writeAhead(writes []string, save func(State) error) error { + if len(writes) == 0 { + return nil + } + ahead := s.carried(maps.Clone(s.fingerprints)) + for _, name := range writes { + ahead.write(name) + } + if err := save(ahead); err != nil { + return fmt.Errorf("recording the values about to be written into the engine's secret store, %s: %w", + strings.Join(writes, ", "), err) + } + for _, name := range writes { + s.write(name) + } + return nil +} + // Withdraw removes from the engine's store the entry of every absent name of // desired (Desired.Missing) that it lists, but one a revision kept reads as // required (Desired.Keep, Desired.KeepServed), and of every name a revision @@ -386,6 +447,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat if err != nil { return prev, res, fmt.Errorf("listing the engine's secret store: %w", err) } + desired = desired.withListed(listing) next := prev.carried(maps.Clone(prev.fingerprints)) listed := make(map[string]bool, len(listing)) var removing []string @@ -413,7 +475,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat for _, name := range gone { _, pushed := prev.fingerprints[name] if _, removed := desired.gone(name, prev); pushed && removed && !listed[name] { - delete(next.fingerprints, name) + next.deleted(name) next.withdraw(name) } } @@ -424,7 +486,7 @@ func Withdraw(ctx context.Context, eng EngineSecrets, desired Desired, prev Stat name, err) } res.removed(removed) - delete(next.fingerprints, name) + next.deleted(name) next.withdraw(name) res.Deleted = append(res.Deleted, name) } diff --git a/internal/secretsync/sync_test.go b/internal/secretsync/sync_test.go index 9bb2bece..0c1f6b03 100644 --- a/internal/secretsync/sync_test.go +++ b/internal/secretsync/sync_test.go @@ -228,13 +228,17 @@ func desiredOf(values map[string]string) Desired { func mustSync(t *testing.T, eng *fakeEngine, desired Desired, prev State) (State, Result) { t.Helper() - next, res, err := Sync(context.Background(), eng, desired, prev, self) + next, res, err := Sync(context.Background(), eng, desired, prev, self, discard) if err != nil { t.Fatalf("Sync: %v", err) } return next, res } +// discard is a save that takes every State Sync hands it ahead of a write, +// and keeps none. +func discard(State) error { return nil } + func assertStore(t *testing.T, eng *fakeEngine, want map[string]fakeEntry) { t.Helper() if !maps.Equal(eng.entries, want) { @@ -361,7 +365,7 @@ func TestSyncSplitsALargeSetIntoBatches(t *testing.T) { } boom := errors.New("engine unavailable") eng.failPut[second[0]] = boom - partial, res, err := Sync(context.Background(), eng, desiredOf(values), State{}, self) + partial, res, err := Sync(context.Background(), eng, desiredOf(values), State{}, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -403,7 +407,7 @@ func TestASyncWhoseLastWriteFailsActivatesItsValuesNextTime(t *testing.T) { eng.reads[name] = true } eng.failPut[last] = errors.New("engine unavailable") - partial, _, err := Sync(context.Background(), eng, desiredOf(values), State{}, self) + partial, _, err := Sync(context.Background(), eng, desiredOf(values), State{}, self, discard) if err == nil || len(eng.reloaded) != 0 || len(partial.fingerprints) != 0 { t.Fatalf("a failed last write = %v, activated %v, recorded %d names; want the error, no activation "+ "and nothing recorded", err, eng.reloaded, len(partial.fingerprints)) @@ -585,6 +589,85 @@ func TestSyncKeepsWhatTheRunningDocumentReferences(t *testing.T) { } } +// TestKeepListedDeletesNothingTheStoreLists: over a revision nobody can read +// (one the engine stores without a seal), Sync and Withdraw keep every owned +// name the store lists, whatever the next document reads of it and whatever +// the State records of the revision's reading: an absent name, a released +// one whose Secret is gone, and one nothing references. Which of them the +// nodes need is what nobody can read, since they run, and boot again, the +// revision before it. Values are still pushed, a foreign entry is left alone +// as always, and the caller's Desired keeps none of the listing, yet names +// the revision as what keeps each absent name the store retained (Holders), +// which is how the operator says why it keeps one. +func TestKeepListedDeletesNothingTheStoreLists(t *testing.T) { + ctx := context.Background() + keyD := secretref.Key{Namespace: "ns", Name: "d", Key: tokenKey} + nameD := StoreName(keyD) + eng := newFakeEngine() + eng.entries[foreignName] = fakeEntry{value: foreignValue, by: access} + // The State records revision2 letting nameB and nameC go: the engine + // stored it, as the operator activated it, without a seal. + state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a1", nameB: "b", nameC: "c", nameD: "d"}), + State{}.Activated(revision2, []secretref.Key{keyB, keyC})) + eng.takeCalls() + + next := desiredOf(map[string]string{nameA: "a2"}) + next.absent = map[string]*MissingError{nameB: {Key: keyB, Reason: SecretNotFound}} + next.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + next.Keep(revision2) + next.KeepListed(revision2) + synced, res, err := Sync(ctx, eng, next, state, self, discard) + if err != nil { + t.Fatalf("Sync: %v", err) + } + assertCalls(t, eng, "list", "put "+nameA) + assertStore(t, eng, map[string]fakeEntry{ + nameA: {value: "a2", by: self}, nameB: {value: "b", by: self}, nameC: {value: "c", by: self}, + nameD: {value: "d", by: self}, foreignName: {value: foreignValue, by: access}, + }) + if len(res.Deleted) != 0 || !slices.Equal(res.Retained, []string{nameB}) || len(synced.Withdrawn()) != 0 { + t.Errorf("Deleted = %v, Retained = %v and Withdrawn() = %v; want none, [%s] and none", + res.Deleted, res.Retained, synced.Withdrawn(), nameB) + } + if len(next.kept[revision2]) != 0 { + t.Errorf("the caller's Desired keeps %v for %s after Sync, want what it was given, nothing", + next.kept[revision2], revision2) + } + if got := next.Holders(nameB, synced); !slices.Equal(got, []string{revision2}) { + t.Errorf("Holders(%s) = %v after Sync, want [%s], which the State records letting it go", nameB, got, revision2) + } + + held := Desired{partial: true, absent: map[string]*MissingError{ + nameB: {Key: keyB, Reason: SecretNotFound}, nameD: {Key: keyD, Reason: KeyNotFound}, + }} + held.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + held.KeepListed(revision2) + withdrawn, res, err := Withdraw(ctx, eng, held, synced) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + assertCalls(t, eng, "list") + if len(res.Deleted) != 0 || !slices.Equal(res.Retained, []string{nameB, nameD}) || len(withdrawn.Withdrawn()) != 0 { + t.Errorf("Deleted = %v, Retained = %v and Withdrawn() = %v; want none, [%s %s] and none", + res.Deleted, res.Retained, withdrawn.Withdrawn(), nameB, nameD) + } + if got := held.Holders(nameD, withdrawn); !slices.Equal(got, []string{revision2}) { + t.Errorf("Holders(%s) = %v after Withdraw, want [%s]", nameD, got, revision2) + } + + // Kept as a revision whose document names nothing, the same Desired + // deletes all three: nameB and nameC as revision2 lets them go, and + // nameD as nothing references it. + plain := desiredOf(map[string]string{nameA: "a2"}) + plain.absent = next.absent + plain.Release(&MissingError{Key: keyC, Reason: SecretNotFound}) + plain.Keep(revision2) + _, res = mustSync(t, eng, plain, synced) + if !slices.Equal(res.Deleted, []string{nameB, nameC, nameD}) { + t.Errorf("Deleted = %v without KeepListed, want [%s %s %s]", res.Deleted, nameB, nameC, nameD) + } +} + // TestSyncWritesBeforeItDeletes: a document switch must find every name it // references written before anything it no longer references is removed. func TestSyncWritesBeforeItDeletes(t *testing.T) { @@ -609,7 +692,7 @@ func TestSyncReturnsWhatItAchieved(t *testing.T) { desired := desiredOf(map[string]string{nameA: "a1", nameB: "b2"}) boom := errors.New("engine unavailable") eng.failPut[nameB] = boom - partial, res, err := Sync(context.Background(), eng, desired, state, self) + partial, res, err := Sync(context.Background(), eng, desired, state, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -626,7 +709,7 @@ func TestSyncReturnsWhatItAchieved(t *testing.T) { delete(eng.failPut, nameB) eng.failDelete[nameC] = boom - partial, res, err = Sync(context.Background(), eng, desired, partial, self) + partial, res, err = Sync(context.Background(), eng, desired, partial, self, discard) if !errors.Is(err, boom) { t.Fatalf("Sync() error = %v, want the engine's", err) } @@ -656,7 +739,7 @@ func TestSyncChangesNothingWhenTheListingFails(t *testing.T) { eng.takeCalls() eng.failList = errors.New("connection refused") - got, res, err := Sync(context.Background(), eng, desiredOf(map[string]string{nameB: "b1"}), state, self) + got, res, err := Sync(context.Background(), eng, desiredOf(map[string]string{nameB: "b1"}), state, self, discard) if !errors.Is(err, eng.failList) { t.Fatalf("Sync() error = %v", err) } @@ -821,7 +904,7 @@ func TestSyncRefusesAPartialDesired(t *testing.T) { eng := newFakeEngine() state, _ := mustSync(t, eng, desiredOf(map[string]string{nameA: "a"}), State{}) eng.takeCalls() - next, _, err := Sync(context.Background(), eng, Desired{partial: true}, state, self) + next, _, err := Sync(context.Background(), eng, Desired{partial: true}, state, self, discard) if err == nil { t.Fatal("Sync took a partial Desired") } @@ -1177,3 +1260,203 @@ func TestSyncKeepsWhatEveryNodeRuns(t *testing.T) { } }) } + +// TestAWrittenValueIsOwedUntilARevisionActivatesAfterIt: every value Sync +// writes, rotated or written back over one the store lost or somebody else +// wrote, is owed an activation (State.Written) until a revision activates +// after it. A later sync with nothing to write keeps it owed, and so does +// settling the withdrawals alone (Reactivated): the write-back leaves the +// fingerprints as they were, so nothing else would say a revision may have +// read the other value. An activation the operator makes (Activated), or any +// revision activated after it (Reloaded), settles it, and a value Sync +// deletes, or one the store no longer holds and nothing writes again, owes +// nothing. +func TestAWrittenValueIsOwedUntilARevisionActivatesAfterIt(t *testing.T) { + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + if got := state.Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after the first sync = %v, want [%s %s]", got, nameA, nameB) + } + if got := state.Activated(revision1, nil).Written(); len(got) != 0 { + t.Fatalf("Written() after an activation = %v, want none", got) + } + state = state.Activated(revision1, nil) + + delete(eng.entries, nameA) + state, res := mustSync(t, eng, both, state) + if !slices.Equal(res.Put, []string{nameA}) || !slices.Equal(state.Written(), []string{nameA}) { + t.Fatalf("a value the store lost: Put = %v, Written() = %v, want [%s] for both", res.Put, state.Written(), nameA) + } + again, res := mustSync(t, eng, both, state) + if len(res.Put) != 0 || !slices.Equal(again.Written(), []string{nameA}) { + t.Fatalf("a sync with nothing to write: Put = %v, Written() = %v, want nothing put and [%s] still owed", + res.Put, again.Written(), nameA) + } + if got := again.Reactivated().Written(); !slices.Equal(got, []string{nameA}) { + t.Errorf("Written() once the withdrawals alone were settled = %v, want [%s] still owed", got, nameA) + } + if got := again.Reloaded().Written(); len(got) != 0 { + t.Errorf("Written() once a revision activated after the write = %v, want none", got) + } + + eng.entries[nameB] = fakeEntry{value: "theirs", by: access} + state, _ = mustSync(t, eng, both, again.Activated(revision2, nil)) + if got := state.Written(); !slices.Equal(got, []string{nameB}) { + t.Fatalf("Written() after a value somebody else wrote was written back = %v, want [%s]", got, nameB) + } + state, res = mustSync(t, eng, desiredOf(map[string]string{nameA: "a1"}), state) + if !slices.Equal(res.Deleted, []string{nameB}) || len(state.Written()) != 0 { + t.Fatalf("a written value Sync deleted: Deleted = %v, Written() = %v, want [%s] deleted and nothing owed", + res.Deleted, state.Written(), nameB) + } + + state, _ = mustSync(t, eng, desiredOf(map[string]string{nameA: "a2"}), state) + gone := Desired{absent: map[string]*MissingError{nameA: {Key: keyA}}} + withdrawn, _, err := Withdraw(context.Background(), eng, gone, state) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + if got := withdrawn.Written(); len(got) != 0 { + t.Errorf("Written() of a value Withdraw deleted = %v, want none: its withdrawal is what is owed", got) + } + + delete(eng.entries, nameA) + state, _ = mustSync(t, eng, desiredOf(nil), state) + if got := state.Written(); len(got) != 0 { + t.Errorf("Written() of a value the store lost and nothing writes again = %v, want none", got) + } +} + +// TestSyncSavesAWriteBeforeItWritesTheValue: Sync hands its caller the State +// with every name it is about to write recorded as written, and the +// fingerprints as they were, before it writes the first value, and writes +// nothing the caller did not save. A value written back leaves its +// fingerprint where it was, so a save lost after the write would otherwise +// leave nothing that said a revision has yet to read it: the next Sync finds +// the store holding the operator's own value under that fingerprint and has +// nothing left to write. And a rotated value saved under its new fingerprint +// before it reached the store would never be written, the next Sync taking +// the old value it finds there for the new one. +func TestSyncSavesAWriteBeforeItWritesTheValue(t *testing.T) { + ctx := context.Background() + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + state = state.Activated(revision1, nil) + + // The store lost a1 and somebody wrote over b1: both are written back. + delete(eng.entries, nameA) + eng.entries[nameB] = fakeEntry{value: "theirs", by: access} + var saved []State + _, res, err := Sync(ctx, eng, both, state, self, func(ahead State) error { + if eng.holds(nameA) || eng.entries[nameB].by != access { + t.Error("Sync saved the State after it wrote a value") + } + saved = append(saved, ahead) + return nil + }) + if err != nil || !slices.Equal(res.Put, []string{nameA, nameB}) { + t.Fatalf("Sync() = %+v, %v, want %s and %s written back", res, err, nameA, nameB) + } + if len(saved) != 1 || !slices.Equal(saved[0].Written(), []string{nameA, nameB}) { + t.Fatalf("saved ahead of the writes: %d States, want one recording [%s %s] written", len(saved), nameA, nameB) + } + // The save after the writes is lost: the next sync starts from the one + // ahead of them. b1, written back under the fingerprint it had, has + // nothing left to write, and a1, which the store had lost and so had no + // fingerprint, is written once more; both are still owed. + again, res := mustSync(t, eng, both, saved[0]) + if !slices.Equal(res.Put, []string{nameA}) || !slices.Equal(again.Written(), []string{nameA, nameB}) { + t.Fatalf("a sync from the State saved ahead of the writes: Put = %v, Written() = %v, want [%s] put and "+ + "[%s %s] still owed", res.Put, again.Written(), nameA, nameA, nameB) + } + + // A rotated value whose write fails after the save: the State saved + // ahead of it keeps the old fingerprint, so the next sync writes it. + state = again.Activated(revision2, nil) + rotated := desiredOf(map[string]string{nameA: "a2", nameB: "b1"}) + boom := errors.New("engine unavailable") + eng.failPut[nameA] = boom + saved = nil + if _, _, err := Sync(ctx, eng, rotated, state, self, func(ahead State) error { + saved = append(saved, ahead) + return nil + }); !errors.Is(err, boom) || len(saved) != 1 { + t.Fatalf("Sync() error = %v after %d saves, want the engine's after one", err, len(saved)) + } + delete(eng.failPut, nameA) + if _, res := mustSync(t, eng, rotated, saved[0]); !slices.Equal(res.Put, []string{nameA}) { + t.Fatalf("a sync from the State saved ahead of a rotation that never reached the store: Put = %v, want [%s]", + res.Put, nameA) + } + + // A save that fails writes nothing; a sync with nothing to write saves + // nothing ahead. + delete(eng.entries, nameA) + eng.takeCalls() + refused := errors.New("the API server did not answer") + if _, res, err := Sync(ctx, eng, rotated, state, self, func(State) error { return refused }); !errors.Is(err, refused) || + len(res.Put) != 0 { + t.Fatalf("Sync() = %+v, %v with a save that failed, want the save's error and nothing written", res, err) + } + assertCalls(t, eng, "list") + settled, _ := mustSync(t, eng, rotated, state) + if _, _, err := Sync(ctx, eng, rotated, settled.Activated(revision1, nil), self, func(State) error { + t.Error("Sync saved ahead of a sync that writes nothing") + return nil + }); err != nil { + t.Fatalf("Sync: %v", err) + } +} + +// TestAChangeIsSettledByAnActivationAboveTheEpochReadAfterIt: a write or a +// withdrawal waits for an activation above the epoch of the fleet's +// activation pointer read first after it (Observed), and any activation the +// engine is read running above it settles it (Ran), whoever made it. One no +// read has followed yet is settled by no epoch at all, since the activation a +// read names may predate the change; a later read keeps the earlier epoch; and +// a name changed again after a read waits for a read after that change. +func TestAChangeIsSettledByAnActivationAboveTheEpochReadAfterIt(t *testing.T) { + eng := newFakeEngine() + both := desiredOf(map[string]string{nameA: "a1", nameB: "b1"}) + state, _ := mustSync(t, eng, both, State{}) + if !state.Unobserved() { + t.Fatal("Unobserved() = false after a sync wrote two values") + } + if got := state.Ran(1 << 40).Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after an activation with no read after the writes = %v, want both still owed", got) + } + state = state.Observed(7) + if state.Unobserved() { + t.Fatal("Unobserved() = true after a read") + } + if got := state.Ran(7).Written(); !slices.Equal(got, []string{nameA, nameB}) { + t.Fatalf("Written() after the activation the read named = %v, want both still owed", got) + } + if got := state.Ran(8).Written(); len(got) != 0 { + t.Fatalf("Written() after an activation above the read = %v, want none", got) + } + if got := state.Observed(9).Ran(8).Written(); len(got) != 0 { + t.Fatalf("Written() after a later read and an activation above the first = %v, want none", got) + } + + delete(eng.entries, nameA) + state, _ = mustSync(t, eng, both, state) + if got := state.Ran(8).Written(); !slices.Equal(got, []string{nameA}) { + t.Fatalf("Written() of a value written again after the read = %v, want [%s] still owed", got, nameA) + } + + gone := Desired{absent: map[string]*MissingError{nameB: {Key: keyB}}} + withdrawn, _, err := Withdraw(context.Background(), eng, gone, state.Activated(revision1, []secretref.Key{keyB})) + if err != nil { + t.Fatalf("Withdraw: %v", err) + } + withdrawn = withdrawn.Observed(10) + if got := withdrawn.Ran(10).Withdrawn(); !slices.Equal(got, []string{nameB}) { + t.Fatalf("Withdrawn() after the activation the read named = %v, want [%s] still owed", got, nameB) + } + if got := withdrawn.Ran(11).Withdrawn(); len(got) != 0 { + t.Fatalf("Withdrawn() after an activation above the read = %v, want none", got) + } +} diff --git a/test/crd/cliagent_engine_test.go b/test/crd/cliagent_engine_test.go index 1cd6f919..f88c9846 100644 --- a/test/crd/cliagent_engine_test.go +++ b/test/crd/cliagent_engine_test.go @@ -1249,11 +1249,12 @@ func askEngine(t *testing.T, providers []v1.LLMProvider, list bool) engineReport } in.Seats = append(in.Seats, *s) } - // A person on a seat of their own, reached on a chat surface. Without - // one, Crewlet advises that nobody can be invited into the company, - // which is a verdict on the chart rather than on any provider; a - // tolerance for it here would hide that advisory on a provider's shape - // too, so the probe gives the engine nothing about the chart to say. + // A person on a seat of their own, on the company's own chat, where + // Crewlet asks no account of them. Without the seat, Crewlet advises + // that nobody can be invited into the company, which is a verdict on + // the chart rather than on any provider; a tolerance for it here would + // hide that advisory on a provider's shape too, so the probe gives the + // engine nothing about the chart to say. founder := seat("probe", "founder") founder.Spec.Kind = v1.SeatKindHuman founder.Spec.Contact = v1.SeatContact{GitHubLogin: "probe-founder"} diff --git a/test/crd/company_test.go b/test/crd/company_test.go index d577e38e..678b46b7 100644 --- a/test/crd/company_test.go +++ b/test/crd/company_test.go @@ -40,6 +40,10 @@ func TestCompanyDefaults(t *testing.T) { {"deletionPolicy", s.DeletionPolicy, v1.DeletionPolicyRetain}, {"tracker.backend", s.Tracker.Backend, v1.TrackerWorklet}, {"knowledge.backend", s.Knowledge.Backend, v1.KnowledgeDoclet}, + // No default: unset is a backend the engine derives from the + // company's Integrations, and a Chatlet written for it would be + // refused beside a Slack or Mattermost one. + {"chat.backend", s.Chat.Backend, v1.ChatBackend("")}, } { if !equal(d.got, d.want) { t.Errorf("%s defaulted to %v, want %v", d.field, d.got, d.want) @@ -63,6 +67,14 @@ func fullCompany(c *v1.Company) { RootSpace: "HQ", Vectors: new(true), } + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatChatlet, + Chatlet: v1.ChatletSettings{ + MessageRetentionDays: new(int32(90)), + DefaultChannelPrivate: new(true), + TypingStatus: v1.WorkingStatusAddressed, + }, + } c.Spec.TokenBudget = v1.TokenBudget{Day: 2_000_000} c.Spec.LLM.Order = []v1.LLMProviderReference{ {Name: providerName}, @@ -133,6 +145,67 @@ func TestCompanyValidation(t *testing.T) { }, want: "worklet applies to backend Worklet only", }, + { + name: "a chatlet block on backend Vendor is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatVendor, + Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}, + } + }, + want: msgChatlet, + }, + { + // As Crewlet takes it: with no backend the engine derives one, + // and a chatlet block applies wherever that is Chatlet. + name: "a chatlet block with no backend is accepted", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}} + }, + }, + { + name: "a chatlet block on backend None is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatNone, + Chatlet: v1.ChatletSettings{DefaultChannelPrivate: new(true)}, + } + }, + want: msgChatlet, + }, + { + name: "a company with no chat is accepted", + mutate: func(c *v1.Company) { c.Spec.Chat = v1.ChatSettings{Backend: v1.ChatNone} }, + }, + {name: "a message horizon of 0, for ever, is accepted", mutate: chatRetention(0)}, + {name: "a message horizon of 29 days is refused", mutate: chatRetention(29), want: msgChatRetention}, + {name: "a message horizon of 30 days is accepted", mutate: chatRetention(30)}, + {name: "a message horizon of 3650 days is accepted", mutate: chatRetention(3650)}, + { + name: "a message horizon of 3651 days is refused", + mutate: chatRetention(3651), + want: "should be less than or equal to 3650", + }, + { + name: "a message horizon below 0 is refused", + mutate: chatRetention(-1), + want: "should be greater than or equal to 0", + }, + { + name: "a typing status the engine does not have is refused", + mutate: func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{ + Backend: v1.ChatChatlet, + Chatlet: v1.ChatletSettings{TypingStatus: "Sometimes"}, + } + }, + want: "Unsupported value: \"Sometimes\"", + }, + { + name: "a chat backend the engine does not have is refused", + mutate: func(c *v1.Company) { c.Spec.Chat = v1.ChatSettings{Backend: "Slack"} }, + want: "Unsupported value: \"Slack\"", + }, { // The second reference names the same provider: kind defaults // to LLMProvider before the rule runs. @@ -807,7 +880,7 @@ func TestCompanyAssistantDeepWorkValidation(t *testing.T) { func TestCompanyEngineConfigReservedKeys(t *testing.T) { reserved := []string{ - "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", + "name", "mission", "vision", "policies", "timezone", "tracker", "knowledge", "chat", "token_budget", "assistant", "providers", "integrations", "mcp_servers", "roles", "units", } const msgReservedKey = "engineConfig may not set name, mission" @@ -843,6 +916,13 @@ func TestCompanyEngineConfigReservedKeys(t *testing.T) { runCreateCases(t, company, cases) } +// chatRetention sets Chatlet's message horizon to days. +func chatRetention(days int32) func(*v1.Company) { + return func(c *v1.Company) { + c.Spec.Chat = v1.ChatSettings{Backend: v1.ChatChatlet, Chatlet: v1.ChatletSettings{MessageRetentionDays: &days}} + } +} + func engineConfigOf(key string) []v1.EngineConfigEntry { return []v1.EngineConfigEntry{{Name: key, Value: &v1.EngineConfigValue{Raw: []byte(`{}`)}}} } diff --git a/test/crd/fixtures_test.go b/test/crd/fixtures_test.go index 638edb4b..d302237a 100644 --- a/test/crd/fixtures_test.go +++ b/test/crd/fixtures_test.go @@ -174,6 +174,8 @@ const ( msgS3EndpointHost = "endpoint's host must be a name, an IPv4 address or an IPv6 address in brackets" msgS3EndpointPort = "endpoint's port must be 1 to 65535" msgUnitID = "a Unit's name is its Crewlet unit id" + msgChatlet = "chatlet applies to backend Chatlet only, or to no backend" + msgChatRetention = "messageRetentionDays must be 0, which keeps messages for ever, or 30 to 3650" msgDuplicate = "Duplicate value" msgThreeReplicas = "Unsupported value: 3" msgStorageFloor = "size must be at least 11Gi" diff --git a/test/crd/orgchart_test.go b/test/crd/orgchart_test.go index c105f2be..bc8a8061 100644 --- a/test/crd/orgchart_test.go +++ b/test/crd/orgchart_test.go @@ -182,7 +182,8 @@ func TestSeatValidation(t *testing.T) { {name: "a human seat with contact is accepted", mutate: asHuman(nothing)}, { // A person who works only through the dashboard has no chat - // account to name; Crewlet advises rather than refuses. + // account to name; on Slack or Mattermost Crewlet advises + // rather than refuses, and on Chatlet says nothing. name: "a human seat without contact is accepted", mutate: func(s *v1.Seat) { s.Spec.Kind = v1.SeatKindHuman }, },