diff --git a/docs/changes/unreleased/2162-hosted-relay-authority-fastpath.md b/docs/changes/unreleased/2162-hosted-relay-authority-fastpath.md new file mode 100644 index 0000000000..307d190421 --- /dev/null +++ b/docs/changes/unreleased/2162-hosted-relay-authority-fastpath.md @@ -0,0 +1,23 @@ +--- +kind: fixed +title: Reuse confirmed hosted authority and prepare new identity peers asynchronously +pr: 2162 +surface: [remote, chat] +invalidates: + - Hosted reads previously repeated quorum work and synchronous presence updates; they now reuse confirmed authority while failing closed when it expires. + - New empty identities previously waited for three voting replicas; their first change now commits on one voter before background joint-consensus formation. +--- + +Directory mutations remain consensus commits. Committed revocations close device +watches and held store polls immediately. Authorization is checked once before +blob work and repeated after I/O only when the applied index changed. Advisory +presence is coalesced outside the request path. + +Internal quorum reads preserve an in-progress idle transition; ordinary reads +still wake the group. Link approvals validate certificates and grants before +reserving a pairing request. Exact retries of a committed decision remain valid +after its original pending deadline; expired undecided requests remain refused. + +Existing identities retain their voting configuration. New singleton births +persist immutable provenance that older relay code cannot serve; upgrades must +preserve compatible recovery rather than restore a local directory writer. diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 18b1389543..e6a11a6563 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -150,7 +150,7 @@ func Sign(c Command, key ed25519.PrivateKey) (Command, error) { // immutable records from unread records. func Dependencies(c Command) []RecordKey { keys := []RecordKey{QuiescenceKey(), IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} - if c.Change.InitMembership != nil { + if c.Change.InitMembership != nil || c.Change.PrepareQuiescence != nil { keys = append(keys, LegacyImportKey()) } m := c.Change @@ -360,11 +360,13 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo // Old imported groups retain their exact manifest and rows. Their FIRST // registry command may use only a manifest-provisioned anchor key; it logs // the authority installation instead of rewriting a frozen import snapshot. - if rows[AuthorityKey(c.Actor)] == nil && c.Change.InitMembership != nil { + // Before that installation, the initial singleton anchor may only prepare + // a quorum-confirmed idle marker for its actual one-voter configuration. + if rows[AuthorityKey(c.Actor)] == nil && (c.Change.InitMembership != nil || c.Change.PrepareQuiescence != nil && len(meta.Configuration.Voters) == 1 && !meta.Configuration.Joint()) { var manifest LegacyManifest if raw := rows[LegacyImportKey()]; raw != nil && Decode(raw, &manifest) == nil && manifest.Validate() == nil && manifest.Config.Cluster == c.Cluster && bytes.Equal(manifest.Config.SigningKey, identity.SigningKey) { for i, node := range manifest.Config.Anchors { - if c.Actor == AnchorActor(node) && i < len(manifest.Config.AnchorKeys) { + if c.Actor == AnchorActor(node) && i < len(manifest.Config.AnchorKeys) && (c.Change.InitMembership != nil || i == 0 && meta.Configuration.Voters[0] == node) { auth = Authority{Cluster: c.Cluster, Key: manifest.Config.AnchorKeys[i], Anchor: true} } } @@ -419,7 +421,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo return out, nil } } - service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && (c.Change.WorkerBeatBatch != nil || c.Change.DeviceSeen != nil) + service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && (c.Change.WorkerBeatBatch != nil || c.Change.DeviceSeen != nil || c.Change.AdvanceTime != nil) if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || (c.Change.EnrollMember == nil || c.Change.EnrollMember.Admission == nil) && (c.Change.RemoveMember == nil || c.Change.RemoveMember.Admission == nil) && c.Change.InitMembership == nil && c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } @@ -521,7 +523,10 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo } } } else if service { - code = applyEffect(c, out.Now, rows, put) + // A clock witness advances logical time without stamping presence. + if c.Change.AdvanceTime == nil { + code = applyEffect(c, out.Now, rows, put) + } } else { if assignmentMutation(c.Change) { code = applyAssignment(c, out.Now, identity, rows, put, &out.Result) diff --git a/internal/clusterstate/effects.go b/internal/clusterstate/effects.go index 7c038af468..324dc0bcfb 100644 --- a/internal/clusterstate/effects.go +++ b/internal/clusterstate/effects.go @@ -147,7 +147,7 @@ func applyEffect(c Command, now int64, rows map[RecordKey][]byte, put func(Recor return OK case c.Change.AdmitJoin != nil: v := c.Change.AdmitJoin - if v.Code == "" || len(v.PublicKey) != ed25519.PublicKeySize || v.Expires <= now || v.Expires > int64(MaxSafeInteger) { + if v.Code == "" || len(v.PublicKey) != ed25519.PublicKeySize || v.Expires > int64(MaxSafeInteger) { return BadRequest } k := JoinKey(v.Code) @@ -156,11 +156,19 @@ func applyEffect(c Command, now int64, rows map[RecordKey][]byte, put func(Recor if Decode(raw, &r) != nil { return BadRequest } - if bytes.Equal(r.PublicKey, v.PublicKey) && r.Expires == v.Expires && r.Platform == PlatformOf(v.Platform) { + sameAdmission := bytes.Equal(r.PublicKey, v.PublicKey) && r.Expires == v.Expires && r.Platform == PlatformOf(v.Platform) + // Pending expiry cannot invalidate an already committed decision replay. + if v.Expires <= now && (!sameAdmission || r.Decision == nil) { + return BadRequest + } + if sameAdmission { return OK } return Conflict } + if v.Expires <= now { + return BadRequest + } put(k, JoinRequest{Device: DeviceIDOfKey(v.PublicKey), PublicKey: v.PublicKey, Expires: v.Expires, Platform: PlatformOf(v.Platform)}) return OK case c.Change.CompleteJoin != nil: diff --git a/internal/clusterstate/effects_test.go b/internal/clusterstate/effects_test.go index ab89e08840..dc15939889 100644 --- a/internal/clusterstate/effects_test.go +++ b/internal/clusterstate/effects_test.go @@ -129,6 +129,63 @@ func TestR16R17R21EffectsAuthorizationCampaign(t *testing.T) { }) } } +func TestR16DecidedJoinAdmissionSurvivesPendingExpiry(t *testing.T) { + public := actorKey("joiner").Public().(ed25519.PublicKey) + admission := cs.AdmitJoin{Code: "request", PublicKey: public, Expires: 1002} + decision := cs.JoinDecision{Code: "request", Device: cs.DeviceIDOfKey(public), Cert: joinCert(t, "joiner"), Grant: "c2VhbGVk", Record: cs.Device{V: 1, Name: "c2VhbGVk"}} + t.Run("decided replay preserves the original decision", func(t *testing.T) { + a := newApplication(t) + if r := service(a, 1000, cs.Mutation{AdmitJoin: &admission}); r.Code != cs.OK { + t.Fatal(r) + } + if r := a.run("a", 1001, cs.Mutation{DecideJoin: &decision}); r.Code != cs.OK { + t.Fatal(r) + } + if r := service(a, 1003, cs.Mutation{AdmitJoin: &admission}); r.Code != cs.OK { + t.Fatal("pending expiry blocked an existing decision", r) + } + if r := a.run("a", 1004, cs.Mutation{DecideJoin: &decision}); r.Code != cs.OK { + t.Fatal("the original decision was not idempotent", r) + } + if r := a.run("a", 1005, cs.Mutation{DecideJoin: &cs.JoinDecision{Code: "request", Deny: true}}); r.Code != cs.AlreadyDecided { + t.Fatal("a contradictory decision changed the outcome", r) + } + extended := admission + extended.Expires = 2000 + if r := service(a, 1006, cs.Mutation{AdmitJoin: &extended}); r.Code != cs.Conflict { + t.Fatal("re-admission changed a decided request's lifetime", r) + } + mismatched := admission + mismatched.PublicKey = actorKey("different").Public().(ed25519.PublicKey) + if r := service(a, 1007, cs.Mutation{AdmitJoin: &mismatched}); r.Code != cs.BadRequest { + t.Fatal("expired re-admission changed the immutable requester", r) + } + }) + t.Run("expired pending admission cannot be revived", func(t *testing.T) { + a := newApplication(t) + if r := service(a, 1000, cs.Mutation{AdmitJoin: &admission}); r.Code != cs.OK { + t.Fatal(r) + } + if r := service(a, 1003, cs.Mutation{AdmitJoin: &admission}); r.Code != cs.BadRequest { + t.Fatal("an expired pending request was admitted again", r) + } + extended := admission + extended.Expires = 2000 + if r := service(a, 1004, cs.Mutation{AdmitJoin: &extended}); r.Code != cs.Conflict { + t.Fatal("re-admission extended the pending lifetime", r) + } + if r := a.run("a", 1005, cs.Mutation{DecideJoin: &decision}); r.Code != cs.RequestGone { + t.Fatal("an expired pending request acquired a decision", r) + } + }) + t.Run("unseen expired admission is refused", func(t *testing.T) { + a := newApplication(t) + if r := service(a, 1003, cs.Mutation{AdmitJoin: &admission}); r.Code != cs.BadRequest { + t.Fatal("an unseen expired request was admitted", r) + } + }) +} + func TestR16PairingCertificateAndStickyRevocation(t *testing.T) { a := newApplication(t) pub := actorKey("joiner").Public().(ed25519.PublicKey) diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index dcf7a9408e..64d7263df4 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -100,6 +100,17 @@ An open Home or compatibility events viewer pings every 10 seconds. The platform Connections naming held chats share a renewal cycle every 30 seconds on both relays while fresh hold evidence is available. Each cycle uses bounded batches; the 10-second presence pings between cycles do not commit another renewal. Presence alone never renews a chat. +## First connection and hosted read authority + +A new, empty identity can save its first change before the hosted relay finishes +preparing its other copies. Machine coordination can briefly ask you to retry +until that preparation finishes. Existing identities keep their voting group. + +Ordinary hosted reads reuse recently confirmed authority. Once that confirmation +expires, an unavailable majority makes reads fail rather than use stale device +permissions. Revocation and rotation remain committed changes; device activity +is advisory and can appear after a short delay. + ## Hosted relay memory capacity and retry After every voter has completed the group's idle barrier, the hosted relay frees its working memory for other households and keeps the saved directory in persistent storage. The next request reloads those copies and obtains a fresh majority response. Busy groups keep their memory. When busy groups fill the relay's memory allowance, another activation receives HTTP 503 with retry guidance; retrying after a group sleeps can succeed. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index a1e6bbd23c..6e7a286230 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -206,6 +206,8 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { return v1Commands(d) case 41, 42, 43, 44, 45: return r.pagedApplication(op, d) + case 49: + return r.formation(d) case 39: return hostedForward(d) case 36: diff --git a/internal/raftabi/formation.go b/internal/raftabi/formation.go new file mode 100644 index 0000000000..3943dc8277 --- /dev/null +++ b/internal/raftabi/formation.go @@ -0,0 +1,27 @@ +package raftabi + +import ( + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func (r *Runtime) formation(d *decoder) ([]byte, error) { + node := r.nodes[d.u32()] + if node == nil { + return nil, errFrame + } + var manifest cs.LegacyManifest + raw, birth, registry := d.blob(), d.blob(), d.blob() + if d.end() != nil || cs.Decode(raw, &manifest) != nil || !validSingleton(manifest, birth) || len(registry) > 0 { + return nil, errFrame + } + plan, err := node.PlanFormation(manifest.Config.Anchors) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(plan.Change) + e.flag(plan.Checkpoint) + e.flag(plan.Settle) + e.flag(plan.Configure) + return e.b, nil +} diff --git a/internal/raftabi/hosted.go b/internal/raftabi/hosted.go index 9902a5ee36..0f523b3da6 100644 --- a/internal/raftabi/hosted.go +++ b/internal/raftabi/hosted.go @@ -22,7 +22,7 @@ func hostedEffect(d *decoder) ([]byte, error) { if cs.Decode(raw, &change) != nil && (j.Decode(&change) != nil || j.Decode(new(any)) != io.EOF) { return nil, errFrame } - if change.DeviceSeen == nil && change.RenewHolds == nil && change.AdmitJoin == nil && change.CompleteJoin == nil && change.RecordUpload == nil { + if change.AdvanceTime == nil && change.DeviceSeen == nil && change.RenewHolds == nil && change.AdmitJoin == nil && change.CompleteJoin == nil && change.RecordUpload == nil { return nil, errFrame } key := ed25519.NewKeyFromSeed(seed) diff --git a/internal/raftabi/legacy.go b/internal/raftabi/legacy.go index 7e5c28e916..a8a141db99 100644 --- a/internal/raftabi/legacy.go +++ b/internal/raftabi/legacy.go @@ -4,6 +4,7 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" + "strconv" ) // The hosted source feeds original SQL JSON to this bounded conversion call. @@ -36,6 +37,10 @@ func legacyCodec(d *decoder) ([]byte, error) { func legacyGenesis(d *decoder) ([]byte, error) { var m clusterstate.LegacyManifest raw := d.blob() + var birth []byte + if len(d.b) > 0 { + birth = d.blob() + } if e := d.end(); e != nil { return nil, e } @@ -45,7 +50,14 @@ func legacyGenesis(d *decoder) ([]byte, error) { e := encoder{} e.message(&pb.Entry{Index: proto.Uint64(1), Term: proto.Uint64(1)}) e.message(&pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(1)}) - e.message(&pb.ConfState{Voters: m.Config.Anchors}) + voters := m.Config.Anchors + if len(birth) > 0 { + if !validSingleton(m, birth) { + return nil, clusterstate.ErrMigration + } + voters = voters[:1] + } + e.message(&pb.ConfState{Voters: voters}) return e.b, nil } @@ -73,3 +85,16 @@ func legacyAuthorities(d *decoder) ([]byte, error) { } return out.b, nil } + +func validSingleton(m clusterstate.LegacyManifest, raw []byte) bool { + var birth struct { + Cluster string `json:"cluster"` + Digest string `json:"digest"` + Voters []string `json:"voters"` + } + if clusterstate.Decode(raw, &birth) != nil || m.Validate() != nil || m.Version != 0 || len(m.Config.Anchors) != 3 || birth.Cluster != m.Config.Cluster || birth.Digest != m.Digest || len(birth.Voters) != 1 || birth.Voters[0] != strconv.FormatUint(m.Config.Anchors[0], 10) { + return false + } + rows, err := clusterstate.LegacyAuthorities(m.Config) + return err == nil && m.Rows == uint64(len(rows)+1) +} diff --git a/internal/raftabi/membership.go b/internal/raftabi/membership.go index 8a56e153d8..06dbab82d5 100644 --- a/internal/raftabi/membership.go +++ b/internal/raftabi/membership.go @@ -48,6 +48,16 @@ func learnerReady(d *decoder) ([]byte, error) { func anchorRegistry(d *decoder) ([]byte, error) { raw := d.blob() + var conf *pb.ConfState + if len(d.b) > 0 { + conf = new(pb.ConfState) + d.message(conf) + var manifest clusterstate.LegacyManifest + manifestRaw, birth := d.blob(), d.blob() + if clusterstate.Decode(manifestRaw, &manifest) != nil || !validSingleton(manifest, birth) { + return nil, errFrame + } + } if err := d.end(); err != nil { return nil, err } @@ -62,6 +72,23 @@ func anchorRegistry(d *decoder) ([]byte, error) { if err != nil { return nil, err } + if conf != nil { + for _, node := range append(append(append([]uint64{}, conf.GetVoters()...), conf.GetVotersOutgoing()...), conf.GetLearners()...) { + found := false + for _, peer := range peers { + found = found || peer.Node == node + } + if !found { + return nil, errFrame + } + } + for i := range peers { + peers[i].Voter = false + for _, node := range append(append([]uint64{}, conf.GetVoters()...), conf.GetVotersOutgoing()...) { + peers[i].Voter = peers[i].Voter || peers[i].Node == node + } + } + } b, err := clusterstate.Encode(peers) if err != nil { return nil, err diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 94f4e3c440..cc22031eef 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -258,14 +258,10 @@ func (c *Core) Step(input Input) (effects Effects, err error) { effects.SnapshotNeeded = c.SnapshotNeeded() } }() + idleRead := false if idle, ok := input.(IdleRead); ok { - active, sleeping := c.schedule.active, c.schedule.sleeping + idleRead = true input = Read{RequestID: idle.RequestID} - defer func() { - if err == nil && !sleeping { - c.schedule.active = active - } - }() } if c.stopped { return effects, ErrStopped @@ -315,7 +311,11 @@ func (c *Core) Step(input Input) (effects Effects, err error) { return effects, ErrBusy } switch input.(type) { - case Read, Propose, Configure, Transfer: + case Read: + if !idleRead || c.schedule.sleeping { + c.wakeScheduling() + } + case Propose, Configure, Transfer: c.wakeScheduling() case PeerMessage: if c.schedule.sleeping { diff --git a/internal/raftcore/formation.go b/internal/raftcore/formation.go new file mode 100644 index 0000000000..cf49ef01c0 --- /dev/null +++ b/internal/raftcore/formation.go @@ -0,0 +1,119 @@ +package raftcore + +import ( + "slices" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type FormationPlan struct { + Change []byte + Configure bool + Checkpoint bool + Settle bool +} + +// PlanFormation expands only an immutable singleton birth configuration. It +// cannot restore or shrink a formed cluster; the durable ConfState governs +// every retry, including recovery after the joint entry has committed. +func (c *Core) PlanFormation(anchors []uint64) (FormationPlan, error) { + var p FormationPlan + if c.stopped { + return p, ErrStopped + } + if !c.leader() { + return p, ErrLeader + } + if c.pending != nil { + return p, ErrBusy + } + if len(anchors) != 3 || !slices.Equal(c.cfg.Anchors, anchors) { + return p, ErrConfig + } + conf := c.store.conf + voters, learners, outgoing := conf.GetVoters(), conf.GetLearners(), conf.GetVotersOutgoing() + // A queued configuration must be decided before another can be planned. + last, err := c.store.LastIndex() + if err != nil { + return p, err + } + if last > c.applied { + entries, err := c.store.Entries(c.applied+1, last+1, ^uint64(0)) + if err != nil { + return p, err + } + for _, entry := range entries { + if entry.GetType() != pb.EntryNormal { + return p, ErrBusy + } + } + } + change := &pb.ConfChangeV2{} + if len(outgoing) > 0 { + if !sameIDs(voters, anchors) || !sameIDs(outgoing, anchors[:1]) || len(learners) > 0 || len(conf.GetLearnersNext()) > 0 || conf.GetAutoLeave() { + return p, ErrConfig + } + // An empty explicit change leaves joint consensus using both quorums. + } else if sameIDs(voters, anchors) { + if len(learners) > 0 || len(conf.GetLearnersNext()) > 0 || conf.GetAutoLeave() { + return p, ErrConfig + } + p.Settle = true + return p, nil + } else { + if !sameIDs(voters, anchors[:1]) || len(conf.GetLearnersNext()) > 0 || conf.GetAutoLeave() { + return p, ErrConfig + } + for _, node := range learners { + if !slices.Contains(anchors[1:], node) { + return p, ErrConfig + } + } + for _, node := range anchors[1:] { + if !slices.Contains(learners, node) { + change.Changes = []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(node)}} + p.Configure = true + p.Checkpoint = true + p.Change, err = proto.MarshalOptions{Deterministic: true}.Marshal(change) + return p, err + } + } + snapshot, err := c.store.Snapshot() + if err != nil { + return p, err + } + for _, node := range anchors[1:] { + if !slices.Contains(snapshot.GetMetadata().GetConfState().GetLearners(), node) { + p.Checkpoint = true + return p, nil + } + } + progress := c.node.Status().Progress + for _, node := range anchors[1:] { + peer, ok := progress[node] + if !ok || peer.Match < c.applied { + return p, nil + } + } + change.Transition = pb.ConfChangeTransitionJointExplicit.Enum() + for _, node := range anchors[1:] { + change.Changes = append(change.Changes, &pb.ConfChangeSingle{Type: pb.ConfChangeAddNode.Enum(), NodeId: proto.Uint64(node)}) + } + } + p.Configure = true + p.Change, err = proto.MarshalOptions{Deterministic: true}.Marshal(change) + return p, err +} + +func sameIDs(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for _, node := range a { + if !slices.Contains(b, node) { + return false + } + } + return true +} diff --git a/internal/raftcore/idle_read_quiescence_test.go b/internal/raftcore/idle_read_quiescence_test.go new file mode 100644 index 0000000000..6b8237a82d --- /dev/null +++ b/internal/raftcore/idle_read_quiescence_test.go @@ -0,0 +1,89 @@ +package raftcore + +import ( + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func TestIdleReadAfterCommittedMarkerPreservesQuiescence(t *testing.T) { + for _, public := range []bool{false, true} { + name := "maintenance" + if public { + name = "public" + } + t.Run(name, func(t *testing.T) { + g := newGroup(t) + l := g.elect() + c := g.nodes[l-1] + s := c.status() + marker := cs.Quiescence{Index: s.Applied, Term: s.Term, Barrier: s.Applied - 1, Voters: []uint64{1, 2, 3}} + acks := []SleepAck{{1, s.Applied, s.Term}, {2, s.Applied, s.Term}, {3, s.Applied, s.Term}} + if _, err := c.Schedule(ScheduleInput{Now: 0}); err != nil { + t.Fatal(err) + } + if _, err := c.Schedule(ScheduleInput{Now: 60000, Marker: marker, Acks: acks[:2]}); err != nil { + t.Fatal(err) + } + if public { + g.call(l, Read{RequestID: "public"}) + } else { + g.call(l, IdleRead{RequestID: "maintenance"}) + } + g.pump() + plan, err := c.Schedule(ScheduleInput{Now: 60000, Marker: marker, Acks: acks}) + if err != nil { + t.Fatal(err) + } + if plan.Sleeping == public { + t.Fatalf("completed %s read: sleeping=%t, want %t", name, plan.Sleeping, !public) + } + }) + } +} + +func TestIdleReadWakesSleepingQuiescence(t *testing.T) { + g := newGroup(t) + l := g.elect() + c := g.nodes[l-1] + s := c.status() + marker := cs.Quiescence{Index: s.Applied, Term: s.Term, Barrier: s.Applied - 1, Voters: []uint64{1, 2, 3}} + acks := []SleepAck{{1, s.Applied, s.Term}, {2, s.Applied, s.Term}, {3, s.Applied, s.Term}} + if _, err := c.Schedule(ScheduleInput{Now: 0}); err != nil { + t.Fatal(err) + } + plan, err := c.Schedule(ScheduleInput{Now: 60000, Marker: marker, Acks: acks}) + if err != nil || !plan.Sleeping { + t.Fatal(plan, err) + } + g.call(l, IdleRead{RequestID: "wake-internal"}) + g.pump() + plan, err = c.Schedule(ScheduleInput{Now: 60001, Marker: marker, Acks: acks}) + if err != nil || plan.Sleeping || len(plan.Wake) != 3 { + t.Fatal("internal read reused sleeping certificate", plan, err) + } +} + +func TestUnconfirmedIdleReadCannotReuseQuiescence(t *testing.T) { + g := newGroup(t) + l := g.elect() + c := g.nodes[l-1] + s := c.status() + marker := cs.Quiescence{Index: s.Applied, Term: s.Term, Barrier: s.Applied - 1, Voters: []uint64{1, 2, 3}} + acks := []SleepAck{{1, s.Applied, s.Term}, {2, s.Applied, s.Term}, {3, s.Applied, s.Term}} + if _, err := c.Schedule(ScheduleInput{Now: 0}); err != nil { + t.Fatal(err) + } + g.drop = func(from, to uint64) bool { return from == l || to == l } + g.call(l, IdleRead{RequestID: "no-quorum"}) + g.pump() + for _, read := range g.reads { + if read.RequestID == "no-quorum" { + t.Fatal("internal read completed without quorum") + } + } + plan, err := c.Schedule(ScheduleInput{Now: 60000, Marker: marker, Acks: acks}) + if err != nil || plan.Sleeping { + t.Fatal("unconfirmed read reused sleeping certificate", plan, err) + } +} diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index d9ae55bac0..38ccea43fd 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -85,7 +85,7 @@ export class IdentityDO extends DurableObject { async #noteLeft(ws) { const device = this.watchers.left(ws); const identity = device && (await this.ctx.storage.get('identity')); - if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.operation(async()=>{await this.host.wakeMaintenance();await this.host.commitEffect({device_seen:{device}},Date.now());}); + if (identity && !(await this.#isGone()) && this.host?.ready) this.host.noteSeen(device); } async #serve(request) { @@ -137,6 +137,7 @@ export class IdentityDO extends DurableObject { const route = matchRoute(request.method, url.pathname); const caller = await checkCert(signedHeaders(request)); const leave = route.frames ? this.#enterPut() : () => {}; + let putWork; try { const body = await readBody(request, route.limit, route.over); const who = await checkRequest(caller, { method: request.method, uri: signedUri(this.env, url) }, body, Date.now()); @@ -145,10 +146,12 @@ export class IdentityDO extends DurableObject { const tenant = this.#tenantOf(who.identity); this.#admit(tenant, who.device); this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); - const reply=await this.host.dispatch({tenant,device:who.device,body,request,env:this.env},route); + const reply=await this.host.dispatch({tenant,device:who.device,body,request,env:this.env,retainPut:work=>{if(route.frames)putWork=work;}},route); this.watchers=this.host.watchers;return reply; } finally { - leave(); + // A rejected response can precede the actual blob write. Has and erasure + // must still wait for that write, including its durable index updates. + if(putWork)this.ctx.waitUntil(putWork.then(leave,leave));else leave(); } } diff --git a/relay/hosted/src/link/decide.js b/relay/hosted/src/link/decide.js index 4a95b4dfaa..df2ff90f14 100644 --- a/relay/hosted/src/link/decide.js +++ b/relay/hosted/src/link/decide.js @@ -26,10 +26,15 @@ async function certDevice(c, cert) { return proof.device; } +/** checkApprovalDevice validates the certificate before a request can be reserved. */ +export async function checkApprovalDevice(c, body, request) { + if (!isObject(body) || !isObject(body.device)) throw badRequest(); + if ((await certDevice(c, body.cert)) !== request.device) throw badRequest(); +} + /** asApproval checks the approve body against the request it answers, and answers the device record to write. */ async function asApproval(c, body, request) { - if (!isObject(body.device)) throw badRequest(); - if ((await certDevice(c, body.cert)) !== request.device) throw badRequest(); + await checkApprovalDevice(c, body, request); if (c.tenant.dir.revoked(request.device)) throw badRequest(); // a revoked device re-joins only as a new request and a new key const grant = asGrant(body.grant, c.tenant.limits.linkMaxGrant); return { grant, device: { ...body.device, platform: platformOf(body.device.platform ?? request.platform) } }; diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 01aaec1106..4fdc79345d 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -84,7 +84,7 @@ source is refused, never enrolled as a fresh group. `RaftMemberDO.legacyImport(config, manifest, row, finish)` stages bounded rows and records `legacy-pending` before provisioning. Pending imports cannot initialize/load a core after eviction. Exact retries survive interruption; -finish checks the complete digest, installs a three-voter genesis and portable +finish checks the complete digest, installs its immutable genesis and portable checkpoint, then removes only redundant staging rows and clears the pending marker durably. The stored ceiling includes all grandfathered rows. One import request per member can run at once, preventing page buffers stacking in memory. @@ -169,7 +169,9 @@ use ABI 29 and the same deterministic evaluation. Empty legacy frame plans are normalized before original-proof comparison, preserving replacement semantics. Public source serialization spans coordination, while external blob I/O leaves -that queue and passes a new authority barrier afterward. Public watch sockets +that queue. Blob requests authorize once, then recheck confirmed authority and +its applied index after I/O; changed authority requires authorization again. +Public watch sockets carry presence without the legacy socket-vouch promise; v1 clients keep ordinary beats. The hosted relay admits personal learners and serves their status, exchange and hibernating sockets. Joined computers can explicitly request leadership; @@ -222,10 +224,85 @@ retirement deadlines are then rearmed independently. An alarm on a cold sleeping object reads socket attachments and that hint without constructing a tenant, loading member handles, acquiring a read barrier or reopening peer queues. Platform auto-response timestamps supply presence without data writes. Real -application input still activates coordination and takes a fresh quorum barrier; -a remembered leader never authorizes a reply. Watch hold renewals retain their +application input still activates coordination. Mutations commit through Raft; +reads use a fresh safe barrier or the router's confirmed read certificate below. +A remembered leader alone never authorizes a reply. Watch hold renewals retain their committed path and shared renewal period. +## Confirmed hosted reads + +The identity router can briefly reuse a safe ReadIndex result when at least two +hosted voters in a stable three-voter configuration have identical clean durable +tails: log tip, commit and applied index agree, as do term, leader and configuration. +Those voters intersect every quorum, including the personal-leader topology. +During singleton birth, the sole hosted voter supplies the same clean-tail fence; +joint configurations require an ordinary safe barrier. +No unseen append acknowledgment can commit a later record from these clean tails. +Every subsequent commit therefore needs anchor traffic through this singleton router. + +The router invalidates the certificate before every member mutation, peer turn, +snapshot operation and release of peer traffic. New member RPCs default to mutation; +only named read methods bypass that fence. In-flight work prevents certification. +Reads pin the selected member's activation and check the router generation before +and after asynchronous reads. A generation change, recovery or ten-second expiry +requires fresh confirmation. Failed confirmation never serves a stale certificate. +Recent read activity renews confirmation during maintenance; idle groups still sleep. +The shared Go core keeps ReadOnlySafe and its minority-read contract unchanged. + +When member traffic interrupts a cached read, the request restarts authorization +and row selection with a fresh SafeReadIndex rather than recertifying another +lease. This request pins the member activation and actual applied index across +identity, device and listing pages, including a final view after storage sync. +Changed applied state restarts the whole read; bounded retries still fail closed +without a coherent confirmed view or quorum. Blob I/O stays outside retries, and +its completion rechecks the applied index before deciding whether to authorize again. + +Directory reads do not propose AdvanceTime. Their observed wall-clock floor is +durable in the router's local metadata and never moves backward after recovery; +subsequent v1 mutation translation includes that floor. Directory GET signatures +are checked against it too, so clock rollback cannot revive stale request proofs. +Native API ingress still commits a restricted AdvanceTime clock witness before +checking signed commands or worker reads. It does not stamp device presence. +Without this witness, a backdated command could enroll expired worker scope. +Device presence from native ingress, blob +authorization and watch departure is advisory, coalesced for thirty seconds with +at most 64 pending devices, and flushed one device per maintenance turn after +critical work. Failed presence writes remain pending and do not skip other work. + + +## New identities and background formation + +Only a newly provisioned identity with no legacy directory rows may begin with +one hosted voter. Admission drains before this eligibility check and the freeze +records the decision once. Existing and imported identities keep their three +voters. The public legacy manifest and client snapshot schema do not change. + +A private birth record binds the cluster UUID, original manifest digest and +initial voter to each member's immutable configuration. The first member imports +and commits locally; the first public write does not create the other two actors. +An interrupted publication reuses the same provenance. A published member that +loses its store is never reconstructed from the frozen original. + +Maintenance adds the other anchors as learners before allocating their stores, +catches them up, promotes both through explicit joint consensus, then leaves +joint consensus. The existing membership registry is initialized only after the +stable three-voter configuration commits. Machine coordination that needs that +registry remains unavailable until then. Hold renewal, join delivery and advisory +presence continue during formation, even when a new anchor is unavailable. An idle +singleton can quiesce using its actual one-voter configuration while those +anchors are unavailable. Learner admission refreshes the leader snapshot before +catch-up, so a previously compacted snapshot cannot exclude the new learner. + +Recovery follows the actual durable ConfState. The birth record cannot reduce +the quorum after joint consensus or three-voter formation. Older hosted code +rejects the changed immutable member configuration instead of serving a different +genesis. Private ABI extensions do not change released client messages. + +Committed application changes notify watchers and held store polls from applied +state, including changes replicated from another leader. Revocation closes the +device watch and rejects held polls before the next public request or sweep. A +rejected upload retains its in-flight slot until its underlying storage work +settles, so other devices’ Has requests and retirement still wait for that work. ## Personal leader departure and anchor restoration diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index f31728ebbe..918f9dfa66 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -1,3 +1,4 @@ +import { encodeCanonical } from './codec.js'; import { wasiPreview1 } from './wasi.js'; export const ABI_VERSION = 1; @@ -135,6 +136,6 @@ export class HostCodec { const v = { state: r.blob(), digest: r.blob() }; r.end(); return v; } configuration(conf, entry) { const r = this.call(frame(20).blob(conf).blob(entry)); const next = r.blob(); r.end(); return next; } - genesis(data) {const r=this.call(frame(13).blob(data));const v={entry:r.blob(),hard:r.blob(),conf:r.blob()};r.end();return v;} + genesis(data,birth=null) {const w=frame(13).blob(data);if(birth)w.blob(new TextEncoder().encode(encodeCanonical(birth)));const r=this.call(w);const v={entry:r.blob(),hard:r.blob(),conf:r.blob()};r.end();return v;} snapshot(data) { const r = this.call(frame(9).blob(data)); const b = r.blob(); r.end(); return b; } } diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index 81d52ee441..823e5e220a 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -93,7 +93,7 @@ export class MemberActor { this.checkpointIndex = BigInt(app.applied); this.cache = this.store.one('SELECT coalesce(sum(length(data)),0) AS n FROM journal').n; continue; } if (this.cache > CACHE_BYTES) { this.stopped = true; throw new Error('raft member: log cache capacity; recover'); } - return { send: sends, reads, status: this.status }; + return { send: sends, reads, status: this.status, applicationVersion:app.version??'0' }; } } input(tag, value) { value = value instanceof Uint8Array ? value.slice() : value && typeof value === 'object' ? structuredClone(value) : value; return this.serial(() => this.drive(tag, value)); } @@ -117,6 +117,18 @@ export class MemberActor { }); } // Evidence is an internal authenticated target reply, derived under the // durable member queue. No public request may set its own applied index. + formation() {return this.serial(async()=>{ + const manifest=this.store.read('membership','legacy-import'),registry=this.store.read('membership','registry'); + const r=response(this.reactor.call(frame(49).u32(this.handle).blob(manifest).blob(new TextEncoder().encode(encodeCanonical(this.config.birth))).blob(registry??new Uint8Array()).finish())); + const change=r.blob(),checkpoint=r.u32()!==0,settle=r.u32()!==0,configure=r.u32()!==0;r.end(); + let effect=configure?await this.drive(8,change):{send:[],reads:[],status:this.status}; + if(checkpoint){ + const snapshot=await this.store.checkpoint(),next=await this.drive(11,snapshot); + this.checkpointIndex=this.status.applied; + effect={...next,send:[...effect.send,...next.send],reads:[...effect.reads,...next.reads]}; + } + return {...effect,settle:!configure&&settle,progressed:configure||checkpoint}; + });} leadershipEvidence(revision, barrier) { return this.serial(() => { if(this.stopped||this.store.stopped||!this.handle)throw new Error('raft member: stopped; recover from durable storage'); const header = this.store.read('membership', 'registry'); diff --git a/relay/hosted/src/raft/forward.js b/relay/hosted/src/raft/forward.js index 1212b9da0c..4df61304a8 100644 --- a/relay/hosted/src/raft/forward.js +++ b/relay/hosted/src/raft/forward.js @@ -22,7 +22,7 @@ export class PersonalForward { const flight=new Flight(),pending={ws,node,release:flight.enter(),reply:null}; const promise=flight.idle(host.peers.limits.ForwardTimeout).then(completed=>{if(!completed)this.finish(id,null);return pending.reply;}); this.pending.set(id,pending); - try{ws.send(raw);}catch{this.finish(id,null);} + try{host.invalidateRead();ws.send(raw);}catch{this.finish(id,null);} return {id,promise}; } finish(id,reply){const p=this.pending.get(id);if(!p)return;this.pending.delete(id);p.reply=reply;p.release();this.host.signal();} diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index e54c5d69a9..4898dd03e8 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -16,6 +16,9 @@ import { fromBase64Url } from '../codec.js'; import { presenceOf, parseBeat } from '../presence.js'; import { parseHolds } from '../vouch.js'; import { rotationView } from '../rotation.js'; +import { checkTime } from '../verify.js'; +import { checkApprovalDevice } from '../link/decide.js'; +import { asGrant } from '../link/code.js'; import { CommittedWatchers, CLOSE_REVOKED, CLOSE_ROTATED } from '../watch.js'; const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); @@ -23,7 +26,10 @@ const segment=s=>Array.from(enc.encode(s),b=>b.toString(16).padStart(2,'0')).joi const unsegment=s=>dec.decode(Uint8Array.from(s.match(/../g)??[],h=>parseInt(h,16))); const data=v=>enc.encode(encodeCanonical(v)); const MIGRATION_PAGE=64,COORDINATOR_QUEUE=64,MIN_MEMBERS=2; +const PRESENCE_INTERVAL=30_000,PRESENCE_BATCH=64,READ_ATTEMPTS=3; +const MEMBER_READS=new Set(['hostView','hostResult','hostSelected','hostPersonal','readEvidence','hostWatchState']); const unavailable=()=>new Wire('unreachable',503,1); +const interruptedRead=()=>Object.assign(unavailable(),{readInterrupted:true}); const status={not_found:404,unauthorized:401,revoked:401,rotated:410,bad_request:400,bad_grace:400,self_revoke:400,capacity:507,full:507}; function requireOK(code){if(code!=='ok')throw new Wire(code,status[code]??409);} async function keypair(){ @@ -41,6 +47,8 @@ export class HostedIdentity { this.policy=coordinationPolicy(sharedRuntime());this.started=Math.floor(performance.now());this.elapsed=0;this.asleep=false;this.wakeVersion=0;this.alarmTail=Promise.resolve(); this.ctx=ctx;this.env=env;this.tenant=tenant;this.runtime=sharedRuntime();this.codec=new HostCodec(this.runtime); this.source=tenant.migration(ctx.storage,this.runtime);this.tail=Promise.resolve();this.pending=0;this.operationTail=Promise.resolve();this.operations=0;this.waiters=new Set();this.progressVersion=0;this.forwarding=new PersonalForward(this);this.members=new Map();this.states=new Map();this.inactiveMembers=new Set();this.queue=[];this.reads=[]; + this.seen=new Map();this.pendingIO=new Map(); + this.readGeneration=0;this.memberWork=0;this.readLease=null; } serial(fn){ if(this.pending>=COORDINATOR_QUEUE)return Promise.reject(unavailable());this.pending++; @@ -91,10 +99,78 @@ export class HostedIdentity { await this.progress(deadline); } } - member(id){return this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(this.manifest.config.cluster+'/'+id));} + invalidateRead(){this.readGeneration++;this.readLease=null;} + member(id){ + const stub=this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(this.manifest.config.cluster+'/'+id)); + // Default to mutation: a new member RPC cannot silently bypass the fence. + return new Proxy(stub,{get:(target,name)=>{ + const call=Reflect.get(target,name,target);if(typeof call!=='function')return call; + return (...args)=>{ + if(MEMBER_READS.has(name))return Reflect.apply(call,target,args); + this.invalidateRead();this.memberWork++; + return Promise.resolve().then(()=>Reflect.apply(call,target,args)).finally(()=>{this.memberWork--;}); + }; + }}); + } + leaseCurrent(lease=this.readLease){return lease&&lease.generation===this.readGeneration&&!this.memberWork&&!this.asleep&&performance.now()this.members.has(id)); + if(voters.length<(singleton?1:2))return null; + const evidence=await Promise.all(voters.map(async node=>({node,...await this.members.get(node).readEvidence()}))); + if(generation!==this.readGeneration||this.memberWork||this.coordinationBusy())return null; + if(!selected.clean||BigInt(selected.index)e.clean&&e.index===selected.index&&e.term===selected.term&&e.conf===selected.conf&&e.leader===selected.leader))return null; + const leader=selected.leader;if(leader==='0'||!conf.voters.map(String).includes(leader))return null; + if(this.members.has(leader)&&!evidence.some(e=>e.node===leader&&e.role===2))return null; + // Every quorum intersects these hosted voters. With clean durable tails, + // another commit needs new anchor traffic, which invalidates this fence + // before delivery. The deadline additionally bounds idle reuse. + return this.readLease={node:b.node,index:BigInt(selected.index),term:selected.term,activation:selected.activation,generation,until:started+this.policy.tick_every*2}; + } + async readAuthority(fresh=false){ + this.lastReadAt=Math.max(this.elapsed,performance.now()-this.started); + if(!fresh&&this.leaseCurrent())return this.readLease; + const b=await this.barrier(),lease=!fresh&&await this.confirmRead(b); + if(lease)return lease; + const evidence=await this.members.get(b.node).readEvidence(); + return {...b,activation:evidence.activation}; + } + checkRead(b){if(b.generation!==undefined&&!this.leaseCurrent(b))throw interruptedRead();} + async retryRead(read){ + const deadline=Date.now()+(this.peers?.limits.ForwardTimeout??this.policy.tick_every); + for(let attempt=0;;attempt++){ + try{return await read(attempt>0);} + catch(error){ + if(!error.readInterrupted||attempt+1>=READ_ATTEMPTS||Date.now()>=deadline)throw error; + this.invalidateRead(); + } + } + } + async readView(b,namespace,key,after=null){ + this.checkRead(b); + let row;try{row=await this.members.get(b.node).hostView(b.index,namespace,key,after,b.activation??null);}catch(error){this.invalidateRead();throw unavailable();} + if(b.generation===undefined){ + b.applied??=String(row.app.applied); + if(b.applied!==String(row.app.applied))throw interruptedRead(); + } + this.checkRead(b);return row; + } + readNow(committed=0){ + const floor=this.tenant.meta.get('read-time')??0,now=Math.max(Date.now(),Number(committed),floor); + if(now>floor)this.tenant.meta.set('read-time',now);return now; + } + async finishRead(b,reply){ + await this.ctx.storage.sync(); + if(b.generation===undefined)await this.readView(b,'state','identity'); + this.checkRead(b);return reply; + } config(id){ const c=this.manifest.config; - return {cluster:c.cluster,id:String(id),seed:String(BigInt(id)+100n),bootstrap:c.anchors,ceiling:Number(this.manifest.ceiling),scope:{identity:c.identity,cluster:c.cluster,epoch:c.generation,protocol:1},anchors:c.anchors.map((node,i)=>({node,device:'',key:c.anchor_keys[i],incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'}))}; + return {cluster:c.cluster,id:String(id),seed:String(BigInt(id)+100n),bootstrap:this.source.gate()?.birth?.voters??c.anchors,...(this.source.gate()?.birth?{birth:this.source.gate().birth}:{}),ceiling:Number(this.manifest.ceiling),scope:{identity:c.identity,cluster:c.cluster,epoch:c.generation,protocol:1},anchors:c.anchors.map((node,i)=>({node,device:'',key:c.anchor_keys[i],incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'}))}; } async provision(identityKey){ let p=await this.ctx.storage.get('raft-provision'); @@ -103,44 +179,46 @@ export class HostedIdentity { if(this.source.gate())throw unavailable(); const anchors=[];for(let i=0;i<3;i++)anchors.push(await keypair()); const effects=await keypair(),limits=this.tenant.limits; - p={config:{cluster:crypto.randomUUID(),generation:'1',identity:this.tenant.identity,signing_key:b64(identityKey),anchors:['1','2','3'],anchor_keys:anchors.map(a=>a.public),effects_key:effects.public,now:Date.now(),grace:{min:limits.minGraceMs,max:limits.maxGraceMs,default:limits.defaultGraceMs}},anchors:anchors.map(a=>a.private),effects:effects.seed}; + p={singleton:true,config:{cluster:crypto.randomUUID(),generation:'1',identity:this.tenant.identity,signing_key:b64(identityKey),anchors:['1','2','3'],anchor_keys:anchors.map(a=>a.public),effects_key:effects.public,now:Date.now(),grace:{min:limits.minGraceMs,max:limits.maxGraceMs,default:limits.defaultGraceMs}},anchors:anchors.map(a=>a.private),effects:effects.seed}; await this.ctx.storage.put('raft-provision',p);await this.ctx.storage.sync();return p; } async ensure(identityKey){ // Recover inactive members without replacing live routing or sockets. - if(this.ready&&this.inactiveMembers.size){ + if((this.ready||this.initialized)&&this.inactiveMembers.size){ for(const id of [...this.inactiveMembers]){ try{ - const member=this.member(id);if(!await member.importReceipt(this.manifest))continue; + const member=this.member(id);if(!await member.importReceipt(this.manifest,this.birth??this.source.gate()?.birth??null))continue; const initialized=await member.initialize(this.config(id));if(initialized.capacity)continue; this.members.set(id,member);this.accept(id,initialized);this.accept(id,await member.drainRecovery()); }catch{this.members.delete(id);this.states.delete(id);} } - if([...this.members.keys()].filter(id=>!this.inactiveMembers.has(id)).length!this.inactiveMembers.has(id)).length<(this.birth?1:MIN_MEMBERS))throw unavailable(); } if(this.ready)return; if(this.initialized)return this.completeEnsure(); this.provisioning=await this.provision(identityKey); - if(!this.source.gate())await this.source.freeze(this.provisioning.config); + if(!this.source.gate())await this.source.freeze(this.provisioning.config,this.provisioning.singleton===true); this.manifest=await this.source.inventory(); if(encodeCanonical(this.manifest.config)!==encodeCanonical(this.provisioning.config))throw unavailable(); const owned=this.source.gate().stage==='raft-owned'; + this.birth=this.source.gate().birth??null; + const imports=this.birth?this.birth.voters:this.manifest.config.anchors; if(!owned){ // Each retry advances a bounded page; the cursor is durable only after // all three copies accepted identical bytes. A cut merely repeats a row. const imported=[]; - for(const id of this.manifest.config.anchors)if(await this.member(id).importReceipt(this.manifest))imported.push(id); - if(imported.length<3){ + for(const id of imports)if(await this.member(id).importReceipt(this.manifest,this.birth))imported.push(id); + if(imported.length? OR (namespace=? AND key>?) ORDER BY namespace,key LIMIT ${MIGRATION_PAGE+1}`,cursor.namespace,cursor.namespace,cursor.key)){ if(count++===MIGRATION_PAGE)throw unavailable(); const copied={namespace:row.namespace,key:row.key,data:bytes(row.data)}; - for(const id of this.manifest.config.anchors){ + for(const id of imports){ if(!imported.includes(id)){await this.member(id).legacyImport(this.config(id),this.manifest,copied);} } cursor={namespace:row.namespace,key:row.key};await this.ctx.storage.put('raft-import-cursor',cursor);await this.ctx.storage.sync(); } - for(const id of this.manifest.config.anchors){ + for(const id of imports){ if(!imported.includes(id)){await this.member(id).legacyImport(this.config(id),this.manifest,null,true);} } } @@ -148,9 +226,9 @@ export class HostedIdentity { // AFTER publication, absent or unclaimed stores are excluded, never // imported again under old voter IDs. Two actual survivors are required. const receipts=[]; - for(const id of this.manifest.config.anchors){ + for(const id of (owned?this.manifest.config.anchors:imports)){ try{ - const receipt=await this.member(id).importReceipt(this.manifest);if(!receipt)continue; + const receipt=await this.member(id).importReceipt(this.manifest,this.birth??this.source.gate()?.birth??null);if(!receipt)continue; const member=this.member(id);this.members.set(id,member);receipts.push(receipt); const initialized=await member.initialize(this.config(id)); // Durable import receipts cover all anchors; serving needs a live @@ -158,12 +236,12 @@ export class HostedIdentity { if(initialized.capacity){this.inactiveMembers.add(id);this.members.delete(id);this.states.delete(id);continue;} this.accept(id,initialized);this.elapsed=Math.max(this.elapsed,initialized.elapsed??0); this.accept(id,await member.drainRecovery()); - }catch(error){this.members.delete(id);this.states.delete(id);if(!owned)throw error;} + }catch(error){this.inactiveMembers.add(id);this.members.delete(id);this.states.delete(id);if(!owned)throw error;} } - if(this.members.sizetrue); this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.armAlarm(at))); @@ -178,6 +256,15 @@ export class HostedIdentity { } async completeEnsure(){ const b=await this.barrier(),header=await this.members.get(b.node).hostView(b.index,'membership','registry'); + this.forming=!!this.birth&&!header.raw; + if(this.forming){ + const conf=this.configuration(header.app.conf); + if(conf.joint||conf.outgoing?.length||conf.voters.length!==3){ + const root=await this.members.get(b.node).hostView(b.index,'state','identity');await this.rememberRetirement(this.project('identity',root.raw)); + if(!this.peers)await this.restorePeers(header.app.conf); + await this.wakeMaintenance();this.ready=true;return; + } + } if(!header.raw){ const anchors=this.config(this.manifest.config.anchors[0]).anchors; requireOK((await this.anchorCommand({init_membership:{anchors,limit:this.policy.max_member_records}},Date.now())).value.code); @@ -185,11 +272,12 @@ export class HostedIdentity { const root=await this.barrierRecord({namespace:'state',key:'identity'}); await this.rememberRetirement(this.project('identity',root)); if(!this.peers)await this.restorePeers();else await this.peers.recover(); + this.forming=false; this.membershipPending=!!(header.raw&&decodeCanonical(dec.decode(header.raw)).pending); await this.reconcileMembership(); await this.wakeMaintenance();this.ready=true; } - async restorePeers(){ + async restorePeers(initialConf=null){ const generation=await this.ctx.storage.get('raft-router-generation')??'0'; if(typeof generation!=='string'||!/^(0|[1-9][0-9]*)$/.test(generation)||BigInt(generation)>=(1n<<64n)-1n)throw unavailable(); const saved=await this.ctx.storage.get('raft-router-sleep'),active=await this.ctx.storage.get('raft-maintenance'); @@ -211,10 +299,12 @@ export class HostedIdentity { await this.ctx.storage.put('raft-router-generation',next);await this.ctx.storage.sync(); this.peers=new PersonalPeers(this,next); for(const ws of this.peers.sockets())this.peers.close(ws); - await this.peers.refresh(); - await this.peers.recover(); + this.peers.restoring=initialConf!==null; + try{await this.peers.refresh();}finally{this.peers.restoring=false;} + await this.peers.recover(initialConf); } - accept(id,e){if(!e)return;if(e.status){this.inactiveMembers?.delete(String(id));this.states.set(String(id),e.status);}this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));this.signal();} + accept(id,e){if(!e)return;if(e.status){this.inactiveMembers?.delete(String(id));this.states.set(String(id),e.status);} + if(e.applicationVersion!==undefined)this.appliedVersion=this.appliedVersion===undefined?BigInt(e.applicationVersion):this.appliedVersion>BigInt(e.applicationVersion)?this.appliedVersion:BigInt(e.applicationVersion);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));this.signal();} async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} // Reserve durable sequence ranges before emission. Concurrent public and // peer turns share one allocator, and recovery skips every unused sequence. @@ -245,13 +335,31 @@ export class HostedIdentity { if(this.peers)this.peers.blocked=new Set(); this.pumping=this.pumpQueue(); try{await this.pumping;}finally{this.pumping=null;} + await this.notifyApplied(); if(this.peers)await this.peers.push(); } + async notifyApplied(){ + if(!this.watchers||this.appliedVersion===undefined||this.notifiedVersion===this.appliedVersion)return; + const id=this.replica();if(!id)return; + const index=this.states.get(id).applied,devices=[...new Set([...this.ctx.getWebSockets().filter(ws=>!ws.deserializeAttachment()?.clusterPeer).map(ws=>this.ctx.getTags(ws)[0]),...this.pendingIO.keys()])]; + for(let offset=0;offset===0||offset=2048){this.queue=[];throw unavailable();} - const m=this.queue.shift(),to=this.members.get(m.to);if(!to){if(this.peers)await this.peers.send(m);continue;} - const snapshot=this.codec.inspect(4,m.bytes).snapshot; + const m=this.queue.shift(),to=this.members.get(m.to),snapshot=this.codec.inspect(4,m.bytes).snapshot; + if(!to){ + if(this.forming&&this.manifest.config.anchors.includes(m.to)){if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:false});} + else if(this.peers)await this.peers.send(m); + continue; + } try{ if(snapshot.length){ const meta=this.codec.inspect(3,snapshot),current=await to.hostInput(0);this.accept(m.to,current); @@ -272,7 +380,7 @@ export class HostedIdentity { if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:result.installed===true}); }catch(error){ if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:false}); - this.members.delete(m.to);this.states.delete(m.to); + this.inactiveMembers?.add(m.to);this.members.delete(m.to);this.states.delete(m.to); } } } @@ -285,13 +393,13 @@ export class HostedIdentity { for(const id of this.members.keys()){await this.input(id,16);await this.pump();if(this.leader())return;} throw unavailable(); } - async barrier(id=crypto.randomUUID(),deadline=Date.now()+(this.peers?.limits.ForwardTimeout??this.policy.tick_every)){ + async barrier(id=crypto.randomUUID(),deadline=Date.now()+(this.peers?.limits.ForwardTimeout??this.policy.tick_every),background=false){ await this.elect();const leader=this.leader(); - if(!leader){try{const reply=await this.remote({action:'read',id});return this.appliedReplica(reply.barrier,deadline);}catch(error){if(this.leader())return this.barrier(id,deadline);throw error;}} - try{await this.input(leader,4,id);await this.pump();} - catch(error){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline);throw error;} + if(!leader){try{const reply=await this.remote({action:background?'idle-read':'read',id});return this.appliedReplica(reply.barrier,deadline);}catch(error){if(this.leader())return this.barrier(id,deadline,background);throw error;}} + try{await this.input(leader,background?15:4,id);await this.pump();} + catch(error){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline,background);throw error;} try{for(;;){ - if(this.leader()!==leader){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline);} + if(this.leader()!==leader){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline,background);} const at=this.reads.findIndex(r=>r.node===leader&&r.id===id);if(at>=0)return this.reads.splice(at,1)[0]; if(!this.externalVoter())throw unavailable();await this.progress(deadline); }} @@ -342,15 +450,40 @@ export class HostedIdentity { async barrierOutbox(after,limit){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,'outbox','',after)).rows.slice(0,limit);} projectJSON(kind,raw){return projectRecord(this.runtime,kind,raw);} project(kind,raw){return JSON.parse(this.projectJSON(kind,raw));} - async authorize(device,certificate,write=false){ + async authorize(device,certificate,write=false,fresh=false){ await this.wakeMaintenance(); - requireOK((await this.commitEffect({device_seen:{device}},Date.now())).code); - const b=await this.barrier(),member=this.members.get(b.node),root=await member.hostView(b.index,'state','identity'),actor=await member.hostView(b.index,'state','device/'+segment(device)); - const r=response(this.runtime.call(frame(32).blob(root.raw).blob(actor.raw??new Uint8Array()).str(device).str(certificate).u32(write?1:0).finish()));requireOK(r.str());r.end();return {barrier:b,app:root.app,identity:this.project('identity',root.raw)}; + const b=await this.readAuthority(fresh),root=await this.readView(b,'state','identity'),actor=await this.readView(b,'state','device/'+segment(device)); + const r=response(this.runtime.call(frame(32).blob(root.raw).blob(actor.raw??new Uint8Array()).str(device).str(certificate).u32(write?1:0).finish()));requireOK(r.str());r.end();this.noteSeen(device);return {barrier:b,app:root.app,identity:this.project('identity',root.raw)}; + } + async recheckAuthority(previous,device,certificate,write,fresh){ + const current=await this.readAuthority(fresh); + const applied=current.generation===undefined?(await this.readView(current,'state','identity')).app.applied:current.index; + const changed=current.node!==previous.barrier.node||BigInt(applied)!==BigInt(previous.app.applied); + const authority=changed?await this.authorize(device,certificate,write,fresh):null; + await this.finishRead(authority?.barrier??current,null); + } + // Presence is advisory, like unflushed store statistics. It may be lost on + // eviction, but never delays authority checks or grows with request volume. + noteSeen(device,now=Date.now()){ + const pending=this.seen.get(device); + if(pending){pending.at=now;return;} + if(this.seen.size>=PRESENCE_BATCH)return; + this.seen.set(device,{at:now,due:now+PRESENCE_INTERVAL}); + this.ctx.waitUntil(this.wakeMaintenance()); + } + async flushSeen(now){ + for(const [device,pending]of this.seen){ + if(pending.due>now)continue; + const at=pending.at; + try{requireOK((await this.commitEffect({device_seen:{device}},at)).code);}catch{return;} + if(pending.at===at)this.seen.delete(device); + else pending.due=now+PRESENCE_INTERVAL; + return; + } } async listingJSON(b,app){ const devices=[],cells=[];let identity='null',after=''; - for(;;){const page=await this.members.get(b.node).hostView(b.index,'state','',after);if(!page.rows.length)break; + for(;;){const page=await this.readView(b,'state','',after);if(!page.rows.length)break; // The coordinator holds the only public mutation queue throughout pages. for(const row of page.rows){ after=row.key; @@ -423,7 +556,7 @@ export class HostedIdentity { } async maintenance(now=Date.now(),elapsed=Math.floor(performance.now())){ if(this.asleep)return; - const wakeVersion=this.wakeVersion,busy=this.coordinationBusy()||!!this.wakePending; + const wakeVersion=this.wakeVersion,busy=this.coordinationBusy()||!!this.wakePending||this.seen.size>0; this.wakePending=false; this.elapsed=Math.max(this.elapsed,elapsed-this.started); // Followers must advance their election clocks before any authority read: @@ -435,7 +568,7 @@ export class HostedIdentity { const batch=await member.scheduled(); if(batch){this.accept(id,{send:batch.send.map(m=>({to:m.to,bytes:unb64(m.bytes)}))});await this.pump();await member.acknowledgeScheduled(batch.id);} const effects=await member.schedule(this.elapsed,now,busy);this.accept(id,effects);plans.set(id,effects.plan); - }catch{this.members.delete(id);this.states.delete(id);} + }catch{this.inactiveMembers?.add(id);this.members.delete(id);this.states.delete(id);} } await this.pump(); try{ @@ -457,6 +590,11 @@ export class HostedIdentity { // upload receipts and the raw body pages of a large request that was cut // short, and those are not canonical records to decode. if(app.rows.some(row=>{if(!row.key.startsWith('join/'))return false;const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome;}))await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); + if(this.forming){ + this.formationSoon=false; + try{this.formationSoon=await this.formNext();}catch{} + if(this.formationSoon){await this.flushSeen(now);return;} + } let remoteAck,remoteStatus; const schedule=async(acks=[])=>{ if(remote){ @@ -512,11 +650,48 @@ export class HostedIdentity { } }catch{this.asleep=false;} } + await this.flushSeen(now); + if(!this.asleep&&this.elapsed-this.lastReadAt{entry.reject=reject;});entry.stopped.catch(()=>{}); + waits.add(entry);this.pendingIO.set(device,waits); + return {entry,stopped:entry.stopped,close:()=>{waits.delete(entry);if(!waits.size)this.pendingIO.delete(device);}}; + } + stopIO(device,code){ + for(const entry of this.pendingIO.get(device)??[]){ + if(code==='rotated'&&!entry.write)continue; + entry.error=new Wire(code,code==='revoked'?401:410);entry.reject(entry.error); + } + } async dispatch(c,route){ - if(!new URL(c.request.url).pathname.startsWith('/v1/store/'))return this.operation(()=>this.serve(c,route)); + if(!new URL(c.request.url).pathname.startsWith('/v1/store/'))return this.operation(()=>c.request.method==='GET'?this.retryRead(fresh=>this.serve(c,route,fresh)):this.serve(c,route)); const cert=c.request.headers.get('codeaf-cert'); - await this.operation(async()=>{ + const ticket=await this.operation(()=>this.retryRead(async fresh=>{ await this.ensure(fromBase64Url(c.request.headers.get('codeaf-identity'))); - await this.authorize(c.device,cert,route.write); - }); + const authority=await this.authorize(c.device,cert,route.write,fresh); + return {authority,io:this.waitIO(c.device,route.write)}; + })); // Blob waits cannot hold the coordination queue: revoke/rotation must be // able to commit during I/O, and Has must not block queued frame puts. - const result=await route.handler(c,route.args);await this.ctx.storage.sync(); - await this.operation(()=>this.authorize(c.device,cert,route.write));return result; - } - async serve(c,route){ + try{ + if(ticket.io.entry.error)throw ticket.io.entry.error; + const work=Promise.resolve().then(()=>route.handler(c,route.args)); + c.retainPut?.(work); + const result=await Promise.race([work,ticket.io.stopped]);await this.ctx.storage.sync(); + await this.operation(()=>this.retryRead(fresh=>this.recheckAuthority(ticket.authority,c.device,cert,route.write,fresh)));return result; + }finally{ticket.io.close();} + } + async serve(c,route,fresh=false){ const {request,device,body}=c,url=new URL(request.url),path=url.pathname,cert=request.headers.get('codeaf-cert'); await this.ensure(fromBase64Url(request.headers.get('codeaf-identity'))); await this.wakeMaintenance(); if(path==='/v1/dir/watch'){ if(request.headers.get('upgrade')?.toLowerCase()!=='websocket')throw new Wire('upgrade_required',426); const holds=parseHolds(url.searchParams.getAll('hold'),this.tenant.limits.maxHolds),beat=parseBeat(url.searchParams.getAll('beat')); - const a=await this.authorize(device,cert,true); + const a=await this.authorize(device,cert,true,fresh); + await this.finishRead(a.barrier,null); const reply=this.watchers.accept(device,Number(a.app.version),Date.now(),{holds,beat,events:url.searchParams.get('events')==='1',receipts:url.searchParams.get('receipts')==='1',certificate:cert}); - this.holds.opened(Date.now(),holds.length>0); - if(holds.length)await this.holds.renew(Date.now(),this.watchers.holdEvidence(Date.now())); - if(this.holds.next!==null)await this.armAlarm(this.holds.next);return reply; + try{ + this.holds.opened(Date.now(),holds.length>0); + if(holds.length)await this.holds.renew(Date.now(),this.watchers.holdEvidence(Date.now())); + if(this.holds.next!==null)await this.armAlarm(this.holds.next); + await this.retryRead(fresh=>this.recheckAuthority(a,device,cert,true,fresh));return reply; + }catch(error){ + const code=error.code==='revoked'?CLOSE_REVOKED:error.code==='rotated'?CLOSE_ROTATED:1012; + this.watchers.closeDevice(device,code,error.code??'reconnecting'); + throw error.readInterrupted?unavailable():error; + } } - const join=/^\/v1\/dir\/requests\/([^/]+)\/(approve|deny)$/.exec(path); - if(join){await this.authorize(device,cert,true);const code=decodeURIComponent(join[1]).trim().toLowerCase();const box=this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code));await this.effects.admit(box,code,Date.now());} - let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,Date.now(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} - const result=await this.commitV1(raw); - const b=await this.barrier(),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw),version=snapshot.app.version; - await this.rememberRetirement(identity); - this.watchers.broadcast(version); - if(join){const code=decodeURIComponent(join[1]).trim().toLowerCase();await this.effects.deliver(this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),code,Date.now());unwrap(await this.env.LINK_GATE.get(this.env.LINK_GATE.idFromName('gate')).decided(code));} if(request.method==='GET'){ - if(path==='/v1/identity/rotation')return json(rotationView(identity.rotation,Number(snapshot.app.now),this.tenant.limits)); + const authority=await this.authorize(device,cert,false,fresh),b=authority.barrier; + const now=this.readNow(authority.app.now),app={...authority.app,now:String(now)}; + checkTime(request.headers.get('codeaf-time'),now); + if(path==='/v1/identity/rotation')return this.finishRead(b,json(rotationView(authority.identity.rotation,now,this.tenant.limits))); if(path==='/v1/dir/list'||path==='/v1/dir/presence'){ - const raw=await this.listingJSON(b,snapshot.app),reply=path.endsWith('/presence')?json(presenceOf(JSON.parse(raw).devices,this.watchers.online(),Number(snapshot.app.now))):this.jsonBytes(raw);reply.headers.set('Codeaf-Dir-Version',version);return reply; + const raw=await this.listingJSON(b,app),reply=path.endsWith('/presence')?json(presenceOf(JSON.parse(raw).devices,this.watchers.online(),now)):this.jsonBytes(raw); + reply.headers.set('Codeaf-Dir-Version',app.version);return this.finishRead(b,reply); + } + const row=await this.readView(b,'state','cell/'+segment(decodeURIComponent(route.args[0])));if(!row.raw)throw new Wire('not_found',404); + const reply=this.jsonBytes('{"now":'+now+',"cell":'+this.projectJSON('cell',row.raw)+'}');reply.headers.set('codeaf-conditional-takes','1');return this.finishRead(b,reply); + } + const join=/^\/v1\/dir\/requests\/([^/]+)\/(approve|deny)$/.exec(path); + let joinCode,joinBox; + if(join){ + const authority=await this.authorize(device,cert,true,fresh);joinCode=decodeURIComponent(join[1]).trim().toLowerCase();joinBox=this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(joinCode)); + // Invalid approvals cannot reserve a request for this cluster. The Go + // command still checks authority and revocation at its committed index. + const pending=unwrap(await joinBox.read(0)); + if(join[2]==='approve'){ + const approval=JSON.parse(dec.decode(body));await checkApprovalDevice(c,approval,pending); + const target=await this.readView(authority.barrier,'state','device/'+segment(pending.device)); + if(target.raw&&this.project('device',target.raw).revoked)throw new Wire('bad_request',400); + asGrant(approval.grant,this.tenant.limits.linkMaxGrant); } - const row=await this.members.get(b.node).hostView(b.index,'state','cell/'+segment(decodeURIComponent(route.args[0])));if(!row.raw)throw new Wire('not_found',404); - const reply=this.jsonBytes('{"now":'+row.app.now+',"cell":'+this.projectJSON('cell',row.raw)+'}');reply.headers.set('codeaf-conditional-takes','1');return reply; } + let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,this.readNow(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} + if(join)await this.effects.admit(joinBox,joinCode,Date.now()); + const result=await this.commitV1(raw); + const b=await this.appliedReplica(result.index),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw); + await this.rememberRetirement(identity); + if(join){const code=decodeURIComponent(join[1]).trim().toLowerCase();await this.effects.deliver(this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),code,Date.now());unwrap(await this.env.LINK_GATE.get(this.env.LINK_GATE.idFromName('gate')).decided(code));} if(path==='/v1/identity/rotation'){ this.watchers.closeAll(CLOSE_ROTATED,'rotated');if(identity.rotation?.retire_at)await this.ctx.storage.setAlarm(identity.rotation.retire_at); return json(rotationView(identity.rotation,Number(snapshot.app.now),this.tenant.limits)); } - if(path.endsWith('/revoke'))this.watchers.closeDevice(decodeURIComponent(route.args[0]),CLOSE_REVOKED,'revoked'); + if(path.endsWith('/revoke')){ + const device=decodeURIComponent(route.args[0]); + this.watchers.announce({t:'revoked',device},device);this.watchers.closeDevice(device,CLOSE_REVOKED,'revoked'); + } if(path.endsWith('/release')||path.endsWith('/archive'))return empty(); return result.value.view?this.jsonBytes(this.projectJSON('view',result.raw)):empty(); } diff --git a/relay/hosted/src/raft/legacy.js b/relay/hosted/src/raft/legacy.js index 6e564f8a64..79d103cf4a 100644 --- a/relay/hosted/src/raft/legacy.js +++ b/relay/hosted/src/raft/legacy.js @@ -52,11 +52,11 @@ export class LegacySource { } gate() { const r=this.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0];return r?decodeCanonical(r.doc):null; } keep(g) { this.sql.exec('INSERT OR REPLACE INTO legacy_gate VALUES(1,?)',encodeCanonical(g)); } - async freeze(config) { - if(this.admission)return this.admission.drain(()=>this.freezeSource(config)); - return this.freezeSource(config); + async freeze(config,singleton=false) { + if(this.admission)return this.admission.drain(()=>this.freezeSource(config,singleton)); + return this.freezeSource(config,singleton); } - async freezeSource(config) { + async freezeSource(config,singleton=false) { if(await this.storage.get?.('gone'))throw new Wire('gone',410); // Go validates the binding before the source acquires an irreversible gate. convertLegacy(this.runtime,config,'identity','',JSON.stringify({V:1,identity:config.identity})); @@ -69,7 +69,8 @@ export class LegacySource { const until=c.lease.expires===0?0:this.dir.watchers.vouchedUntil(c.lease.device,id,c.lease.fence); if(until>c.lease.expires)this.sql.exec('INSERT INTO legacy_holds VALUES(?,?)',id,until); } - this.keep({stage:'frozen',config}); + const empty=!this.sql.exec('SELECT 1 FROM dir LIMIT 1').toArray().length; + this.keep({stage:'frozen',config,...(singleton&&empty?{birth:true}:{})}); }); await this.storage.sync(); this.dir.watchers.closeAll(1012,'reconnecting'); @@ -99,6 +100,7 @@ export class LegacySource { manifest={config:g.config,version:String(version),rows:String(count),bytes:String(total),digest:hex(digestRows),ceiling:"0"}; for(let i=0;i<8;i++){const next=String(Math.max(DEFAULT_ROWS_BYTES,total+size({namespace:'membership',key:importKey,data:data(manifest)})));if(next===manifest.ceiling)break;manifest.ceiling=next;} this.codec.genesis(data(manifest)); + if(g.birth===true)g.birth={cluster:manifest.config.cluster,digest:manifest.digest,voters:[manifest.config.anchors[0]]}; g.stage='inventoried';g.manifest=manifest;this.keep(g); });await this.storage.sync();return manifest; } @@ -107,7 +109,10 @@ export class LegacySource { this.storage.transactionSync(()=>{ const g=this.gate();check(g?.manifest);const nodes=new Set(); for(const r of receipts)if(g.config.anchors.includes(String(r.node))&&BigInt(r.applied)>=1n&&encodeCanonical(r.manifest)===encodeCanonical(g.manifest))nodes.add(String(r.node)); - check(nodes.size>=2);g.stage='raft-owned';this.keep(g); + if(g.birth){ + const genesis=this.codec.genesis(data(g.manifest),g.birth); + check(nodes.has(g.birth.voters[0])&&receipts.some(r=>r.node===g.birth.voters[0]&&r.conf===b64(genesis.conf)&&encodeCanonical(r.birth)===encodeCanonical(g.birth))); + }else check(nodes.size>=2);g.stage='raft-owned';this.keep(g); });await this.storage.sync(); } // A stale routing hint can never resurrect the old writer. The host assembles @@ -117,7 +122,7 @@ export class LegacySource { // Member storage imports a bounded row stream in one durable genesis. Existing // imported stores return their provenance without overwriting later cluster work. -export async function importLegacy(store, manifest, source) { +export async function importLegacy(store, manifest, source, birth=null) { const m=manifest, encoded=data(m), k={namespace:'membership',key:importKey}; check(m.config.cluster===store.identity.cluster&&m.config.anchors.includes(store.identity.id)&&encoded.length+128<=reserve); check(String(store.identity.ceiling)===m.ceiling); @@ -139,7 +144,7 @@ export async function importLegacy(store, manifest, source) { check(String(count)===m.rows&&String(total)===m.bytes&&hex(digest)===m.digest);add({...k,data:encoded}); // These protobuf messages contain only the common genesis term/index and // the source-authorized complete voter set, encoded by the Go ABI below. - const genesis=store.codec.genesis(data(m)); + const genesis=store.codec.genesis(data(m),birth); store.rows('INSERT INTO journal VALUES(?,?,?)','0000000000000001',genesis.entry,store.hash(genesis.entry)); store.put('tip','1');store.put('hard',b64(genesis.hard)); store.put('application',{applied:'1',conf:b64(genesis.conf),version:m.version,now:String(m.config.now),count:count+1,size:total+size({...k,data:encoded}),hash:hex(aggregate)}); @@ -150,8 +155,8 @@ export async function importLegacy(store, manifest, source) { // Cross-object import stages one bounded row per request. Incomplete stages // never start a core and never become application authority. Retries compare // exact bytes; final installation rechecks the source's complete stream digest. -export async function stageLegacyImport(store, manifest, row) { - store.codec.genesis(data(manifest)); +export async function stageLegacyImport(store, manifest, row, birth=null) { + store.codec.genesis(data(manifest),birth); check(manifest.config.cluster===store.identity.cluster&&manifest.config.anchors.includes(store.identity.id)&&String(store.identity.ceiling)===manifest.ceiling); await store.transaction(()=>{ store.rows('CREATE TABLE IF NOT EXISTS legacy_stage (one INTEGER PRIMARY KEY CHECK(one=1), doc TEXT NOT NULL)'); @@ -167,8 +172,8 @@ export async function stageLegacyImport(store, manifest, row) { store.rows('INSERT INTO legacy_stage_rows VALUES(?,?,?)',row.namespace,row.key,row.data); }); } -export async function finishLegacyImport(store, manifest) { - const snapshot=await importLegacy(store,manifest,function*(){for(const r of store.rows('SELECT namespace,key,data FROM legacy_stage_rows ORDER BY namespace,key'))yield {...r,data:bytes(r.data)};}); +export async function finishLegacyImport(store, manifest, birth=null) { + const snapshot=await importLegacy(store,manifest,function*(){for(const r of store.rows('SELECT namespace,key,data FROM legacy_stage_rows ORDER BY namespace,key'))yield {...r,data:bytes(r.data)};},birth); // These are redundant staging bytes. The immutable legacy ORIGINAL remains // at its source; imported provenance and rows remain in every snapshot. await store.transaction(()=>{store.rows('DROP TABLE IF EXISTS legacy_stage_rows');store.rows('DROP TABLE IF EXISTS legacy_stage');}); diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 133cbdf393..908b058027 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -17,7 +17,7 @@ export { installRaftModule, runtimeMetrics } from './actor.js'; // public Worker routes identity coordination through its authenticated source. // Enrollment and evaluation use the durable Go registry and shared Go policy. export class RaftMemberDO extends DurableObject { - constructor(ctx, env) { super(ctx, env); replaceMemberActivation(ctx.id.toString()); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } + constructor(ctx, env) { super(ctx, env); replaceMemberActivation(ctx.id.toString()); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; this.readActivation=crypto.randomUUID(); } evaluator(config) { const evaluate=directoryEvaluator(sharedRuntime(),config.cluster,config.ceiling??DEFAULT_ROWS_BYTES); this.results=new Map();this.resultBytes=0; @@ -34,23 +34,30 @@ export class RaftMemberDO extends DurableObject { } // These RPC methods have no public fetch route. The authenticated source // coordinator alone provisions stores and drives the same durable actor. - async importReceipt(manifest) { + async importReceipt(manifest,birth=null) { if(await this.ctx.storage.get('raft-erased'))throw new Error('raft member: identity retired'); const config=await this.ctx.storage.get('raft-config'); if(!config)return null; + if(encodeCanonical(config.birth??null)!==encodeCanonical(birth))throw new Error('raft member: genesis provenance differs'); const store=new MemberStorage(this.ctx.storage,new HostCodec(sharedRuntime())); if(!store.one("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'"))return null; await store.open({cluster:config.cluster,id:config.id,protocol:1},config.ceiling); const raw=store.read('membership','legacy-import'); if(!raw)return null; if(new TextDecoder().decode(raw)!==encodeCanonical(manifest))throw new Error('raft member: import provenance differs'); - return {node:config.id,applied:store.application().applied,manifest:decodeCanonical(new TextDecoder().decode(raw))}; + return {node:config.id,applied:store.application().applied,conf:store.application().conf,birth:config.birth??null,manifest:decodeCanonical(new TextDecoder().decode(raw))}; } async eraseRetired(manifest) { const gone=await this.ctx.storage.get('raft-erased'); if(gone && gone!==manifest.config.cluster)throw new Error('raft member: wrong erased cluster'); if(!gone){ - if(!await this.importReceipt(manifest))throw new Error('raft member: import provenance absent'); + const config=await this.ctx.storage.get('raft-config'); + if(!config){ + const store=new MemberStorage(this.ctx.storage,new HostCodec(sharedRuntime())); + if(store.one("SELECT name FROM sqlite_master WHERE type='table' AND name='records'"))throw new Error('raft member: missing configuration'); + await this.ctx.storage.put('raft-erased',manifest.config.cluster);await this.ctx.storage.sync();return; + } + if(!await this.importReceipt(manifest,config.birth??null))throw new Error('raft member: import provenance absent'); await this.load(); await this.actor.serial(async()=>{ const raw=this.actor.store.read('state','identity'),record=raw&&JSON.parse(projectRecord(sharedRuntime(),'identity',raw)); @@ -77,8 +84,9 @@ export class RaftMemberDO extends DurableObject { } return this.actor.input(tag,value); } - async hostView(index,namespace,key,after=null) { + async hostView(index,namespace,key,after=null,activation=null) { await this.load();return this.actor.serial(()=>{ + if(activation!==null&&activation!==this.readActivation)throw new Error('raft member: read activation changed'); if(this.actor.stopped||this.actor.store.stopped||this.actor.status.applied64)throw new Error('raft member: watch page capacity'); + await this.load();return this.actor.serial(()=>{ + if(this.actor.stopped||this.actor.store.stopped||this.actor.status.appliedb.toString(16).padStart(2,'0')).join(''),raw=store.read('state','device/'+key); + if(raw&&JSON.parse(projectRecord(sharedRuntime(),'device',raw)).revoked)revoked.push(device); + } + return {version:store.application().version,identity:JSON.parse(projectRecord(sharedRuntime(),'identity',store.read('state','identity'))),revoked}; + }); + } + // A clean tail excludes append acknowledgments already in flight that could + // let a personal leader commit a record absent from this durable view. + async readEvidence(){ + await this.load();return this.actor.serial(async()=>{ + const actor=this.actor,app=actor.store.application(),status=actor.status; + const scheduled=await this.ctx.storage.get('raft-scheduled'); + const staged=actor.store.one('SELECT count(*) AS n FROM snapshots WHERE complete=0').n; + return {activation:this.readActivation,index:app.applied,conf:app.conf,term:String(status.term),leader:String(status.leader),role:status.role, + clean:!actor.stopped&&!actor.store.stopped&&actor.pending===1&&!scheduled&&!staged&&!this.recoveredEffects?.send?.length&&actor.store.get('tip')===app.applied&&String(status.commit)===app.applied&&String(status.applied)===app.applied}; + }); + } // The coordinator's fresh ReadIndex must have applied before a scoped // read runs. Authority/time and selected rows share this member queue. async hostSelected(raw,index) { @@ -140,15 +171,15 @@ export class RaftMemberDO extends DurableObject { this.importPending=true; const run=Promise.resolve().then(async()=>{ const store=new MemberStorage(this.ctx.storage,new HostCodec(sharedRuntime())); - const canonical={...config,id:String(config.id),seed:String(config.seed),bootstrap:manifest.config.anchors.map(String),ceiling:Number(manifest.ceiling)}; + const canonical={...config,id:String(config.id),seed:String(config.seed),bootstrap:(config.birth?.voters??manifest.config.anchors).map(String),ceiling:Number(manifest.ceiling)}; const previous=await this.ctx.storage.get('raft-config'); if(previous&&JSON.stringify(previous)!==JSON.stringify(canonical))throw new Error('raft member: provisioned identity cannot change'); await this.ctx.storage.put('legacy-pending',true);await this.ctx.storage.sync(); if(!previous){await this.ctx.storage.put('raft-config',canonical);await this.ctx.storage.sync();} await store.open({cluster:canonical.cluster,id:canonical.id,protocol:1},canonical.ceiling); - await stageLegacyImport(store,manifest,row); + await stageLegacyImport(store,manifest,row,canonical.birth??null); if(!finish)return; - const snapshot=await finishLegacyImport(store,manifest); + const snapshot=await finishLegacyImport(store,manifest,canonical.birth??null); await this.ctx.storage.delete('legacy-pending');await this.ctx.storage.sync();this.importPending=false; return snapshot; }); @@ -225,7 +256,7 @@ export class RaftMemberDO extends DurableObject { const recovered=this.recoveredEffects; const sends=[...(recovered?.send??[]),...effects.send]; if(!sends.length&&!effects.plan.wake?.length)return []; - const view=this.actor.store.read('membership','registry')?committedRegistry(sharedRuntime(),this.actor.store):provisionedRegistry(sharedRuntime(),this.config); + const view=this.actor.store.read('membership','registry')?committedRegistry(sharedRuntime(),this.actor.store):provisionedRegistry(sharedRuntime(),this.config,this.actor.store); sends.sort((a,b)=>Number(!view[String(a.to)]?.voter)-Number(!view[String(b.to)]?.voter)); const id=String(BigInt(await this.ctx.storage.get('raft-scheduled-sequence')??'0')+1n); const batch={id,send:[],wake:effects.plan.wake},dropped=[]; @@ -271,6 +302,7 @@ export class RaftMemberDO extends DurableObject { async leadershipEvidence(revision,barrier) { await this.load(); return this.actor.leadershipEvidence(revision,barrier); } async leadership(evidence) { await this.load(); return this.actor.leadership(evidence); } async leadershipReceipt() { await this.load(); return this.actor.leadershipReceipt(); } + async formation() {await this.load();return this.actor.formation();} async membership() { await this.load(); return this.actor.membership(); } async joinStatus(advertised) { await this.load(); return this.actor.joinStatus(advertised); } async drainRecovery() { @@ -281,7 +313,7 @@ export class RaftMemberDO extends DurableObject { // view within that index, and invalidate it after every committed change. committedPeers(){ const index=this.actor.store.application().applied; - if(this.registryIndex!==index){this.registryCache=this.actor.store.read('membership','registry')?committedRegistry(sharedRuntime(),this.actor.store):provisionedRegistry(sharedRuntime(),this.config);this.registryIndex=index;} + if(this.registryIndex!==index){this.registryCache=this.actor.store.read('membership','registry')?committedRegistry(sharedRuntime(),this.actor.store):provisionedRegistry(sharedRuntime(),this.config,this.actor.store);this.registryIndex=index;} return this.registryCache; } async peer(raw) { diff --git a/relay/hosted/src/raft/membership.js b/relay/hosted/src/raft/membership.js index c3c1df361d..4cc3c63ff2 100644 --- a/relay/hosted/src/raft/membership.js +++ b/relay/hosted/src/raft/membership.js @@ -28,8 +28,10 @@ export function committedRegistry(runtime, store) { // Only authenticated internal provisioning supplies the three public anchor // records. Before registry commitment, personal keys cannot submit peer data. -export function provisionedRegistry(runtime, config) { - const raw = new TextEncoder().encode(encodeCanonical({ anchors: config.anchors, voters: config.bootstrap })); - const r = response(runtime.call(frame(19).blob(raw).finish())), peers = decode(r.blob()); r.end(); +export function provisionedRegistry(runtime, config, store) { + const raw = new TextEncoder().encode(encodeCanonical({ anchors: config.anchors, voters: config.anchors.map(a=>a.node) })); + const w=frame(19).blob(raw); + if(config.birth)w.blob(unb64(store.application().conf)).blob(store.read('membership','legacy-import')).blob(new TextEncoder().encode(encodeCanonical(config.birth))); + const r = response(runtime.call(w.finish())), peers = decode(r.blob()); r.end(); return Object.fromEntries(peers.map(p => [p.node, { incarnation: p.incarnation, key: p.key, voter: p.voter, revoked: p.revoked }])); } diff --git a/relay/hosted/src/raft/peers.js b/relay/hosted/src/raft/peers.js index fe4b54caa0..7babb1bb5c 100644 --- a/relay/hosted/src/raft/peers.js +++ b/relay/hosted/src/raft/peers.js @@ -27,7 +27,7 @@ export class PersonalPeers { this.host=host;this.generation=generation;this.scope=host.config('1').scope;this.view={};this.boxes=new Map();this.used=[0,0,0];this.progress=new Map();this.snapshots=new Map();this.acks=new Map();this.tail=Promise.resolve();this.pending=0;this.blocked=new Set();this.retryTargets=new Set();this.checkpoints=new Map(); this.limits=peerLimits(host.runtime); } - serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>{this.pending--;this.host.signal?.();});} + serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.host.invalidateRead();this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>{this.pending--;this.host.signal?.();});} members(){return Object.entries(this.view).map(([node,p])=>({node,...p}));} // Public and peer turns publish in order, so a delayed member read cannot // restore an older transport authority after another turn has advanced it. @@ -111,12 +111,15 @@ export class PersonalPeers { return messages; } wire(node,extra={}){ + this.host.invalidateRead(); const box=this.boxes.get(node),chunks=[...(box?.credits??[])].sort(([a],[b])=>BigInt(a)({node,sequence})); const r=response(this.host.runtime.call(frame(38).u32(1).blob(enc.encode(JSON.stringify({...extra,chunks,generation:this.generation,messages:this.deliveries(node)}))).finish()));const raw=r.blob();r.end();return raw; } - async recover(){ + async recover(configuration=null){ const from=this.host.leader();if(!from)return; - for(const [to,member]of Object.entries(this.view))if(!member.revoked&&!this.host.members.has(to)){ + const conf=this.host.configuration(configuration??(await this.host.members.get(from).readEvidence()).conf); + const configured=new Set(['voters','outgoing','learners','learners_next'].flatMap(key=>conf[key]??[]).map(String)); + for(const [to,member]of Object.entries(this.view))if(configured.has(to)&&!member.revoked&&!this.host.members.has(to)){ await this.host.input(from,12,{peer:to,snapshot:true,success:false}); await this.host.input(from,12,{peer:to,snapshot:false,success:false}); } @@ -214,7 +217,7 @@ export class PersonalPeers { this.validate(unb64(a.auth));const raw=this.wire(a.node),text=this.host.hash(raw); if(text===a.last)continue; const body=JSON.parse(dec.decode(raw));if(!body.messages.length&&!body.chunks?.length)continue; - ws.send(raw);const deadline=a.deadline??(body.messages.length?Date.now()+this.limits.Retention:null);ws.serializeAttachment({...a,last:text,deadline});if(deadline)await this.host.armAlarm(deadline); + this.host.invalidateRead();ws.send(raw);const deadline=a.deadline??(body.messages.length?Date.now()+this.limits.Retention:null);ws.serializeAttachment({...a,last:text,deadline});if(deadline)await this.host.armAlarm(deadline); }catch{this.close(ws);} } } @@ -227,7 +230,7 @@ export class PersonalPeers { const deadline=initial&&JSON.parse(dec.decode(initial)).messages.length?Date.now()+this.limits.Retention:null; const {client,server}=acceptHibernatingSocket(this.host.ctx,['cluster-peer'],{clusterPeer:true,kind,node:req.node,generation:this.generation,auth:b64(raw),last:initial?this.host.hash(initial):null,deadline}); if(deadline)await this.host.armAlarm(deadline); - if(initial)server.send(initial); + if(initial){this.host.invalidateRead();server.send(initial);} return new Response(null,{status:101,webSocket:client}); } async frame(ws,raw){ @@ -235,7 +238,7 @@ export class PersonalPeers { try{ this.validate(unb64(att.auth));if(att.kind==='forward'){this.host.forwarding.frame(ws,raw);return;}if(att.kind!=='peer'||!(raw instanceof ArrayBuffer))throw refused('invalid'); const bytes=new Uint8Array(raw),next=JSON.parse(dec.decode(bytes));if(next.node!==att.node||next.generation!==att.generation)throw refused('unauthorized'); - const result=await this.exchange(bytes);ws.send(result.raw);const deadline=JSON.parse(dec.decode(result.raw)).messages.length?Date.now()+this.limits.Retention:null;ws.serializeAttachment({...att,last:this.host.hash(result.raw),deadline});if(deadline)await this.host.armAlarm(deadline); + const result=await this.exchange(bytes);this.host.invalidateRead();ws.send(result.raw);const deadline=JSON.parse(dec.decode(result.raw)).messages.length?Date.now()+this.limits.Retention:null;ws.serializeAttachment({...att,last:this.host.hash(result.raw),deadline});if(deadline)await this.host.armAlarm(deadline); }catch{this.close(ws);} } async departed(ws){ diff --git a/relay/hosted/src/verify.js b/relay/hosted/src/verify.js index b58bfce28c..220bdce02f 100644 --- a/relay/hosted/src/verify.js +++ b/relay/hosted/src/verify.js @@ -46,7 +46,9 @@ async function requestMessage({ method, uri, time }, body) { return enc.encode(`${REQ_LABEL}${method}\n${uri}\n${time}\n${hex(await sha256(body))}`); } -const withinSkew = (time, now) => /^-?\d+$/.test(time) && Math.abs(now - Number(time)) <= SKEW_MS; +export function checkTime(time, now) { + if (!/^-?\d+$/.test(time) || Math.abs(now - Number(time)) > SKEW_MS) throw new Refusal('skew', 'clock'); +} /** * checkCert is the half of verification that needs no body: the headers are present and the @@ -74,7 +76,7 @@ export async function checkCert(headers) { export async function checkRequest(caller, req, body, nowMs) { const msg = await requestMessage({ method: req.method, uri: req.uri, time: caller.time }, body); if (!(await verifySig(await importKey(caller.devicePub), fromB64Url(caller.sig), msg))) throw unauthorized('bad signature'); - if (!withinSkew(caller.time, nowMs)) throw new Refusal('skew', 'clock'); + checkTime(caller.time, nowMs); return { identity: caller.identity, device: caller.device }; } diff --git a/relay/hosted/test/api.e2e.mjs b/relay/hosted/test/api.e2e.mjs index ecf0b33ee5..29eeb12115 100644 --- a/relay/hosted/test/api.e2e.mjs +++ b/relay/hosted/test/api.e2e.mjs @@ -38,7 +38,10 @@ assert.equal(streamed.status, 401, 'an unsigned stream is refused as unsigned, h assert.equal((await call(a, 'POST', '/v1/dir/cells/c1', init())).json.cell.lease.fence, 1); assert.equal((await call(a, 'POST', '/v1/dir/cells/c1', init())).status, 409); assert.equal(answerOf(await call(b, 'POST', '/v1/dir/cells/c1/acquire', {})), 'lease_held'); -assert.equal((await call(b, 'POST', '/v1/dir/cells/c1/acquire', { force: true })).json.cell.lease.fence, 2); +const blind = await call(b, 'POST', '/v1/dir/cells/c1/acquire', { force: true }); +assert.deepEqual([blind.status, blind.json.err], [409, 'lease_held']); +const observed = (await call(b, 'GET', '/v1/dir/cells/c1')).json.cell.lease; +assert.equal((await call(b, 'POST', '/v1/dir/cells/c1/acquire', { force: true, expect: { device: observed.device, fence: observed.fence } })).json.cell.lease.fence, 2); const pub = (fence) => ({ fence, old_head: 'h0', head: 'h1', size: 0, class: 'work', pending: 0 }); const stale = await call(a, 'POST', '/v1/dir/cells/c1/publish', pub(1)); assert.deepEqual([stale.status, stale.json.err], [409, 'fence_stale']); diff --git a/relay/hosted/test/caps.e2e.mjs b/relay/hosted/test/caps.e2e.mjs index f41f531743..5c08d45303 100644 --- a/relay/hosted/test/caps.e2e.mjs +++ b/relay/hosted/test/caps.e2e.mjs @@ -73,8 +73,15 @@ const tests = { }, }; -for (const [name, run] of Object.entries(tests)) { - await run(); - console.log('caps ok', name); +const selected = process.argv[2]; +if (selected === '--list') { + console.log(Object.keys(tests).join('\n')); +} else { + const cases = Object.entries(tests).filter(([name]) => !selected || name === selected); + if (!cases.length) throw new Error(`unknown caps case: ${selected}`); + for (const [name, run] of cases) { + await run(); + console.log('caps ok', name); + } + console.log('caps: all pass'); } -console.log('caps: all pass'); diff --git a/relay/hosted/test/e2e.sh b/relay/hosted/test/e2e.sh index e0262df37c..a8645e8616 100644 --- a/relay/hosted/test/e2e.sh +++ b/relay/hosted/test/e2e.sh @@ -9,16 +9,41 @@ pids="" stop() { for p in $pids; do kill "$p" 2>/dev/null || true; done; rm -rf "$work"; } trap stop EXIT +# All local workers consume the same reactor. Multiple custom-build watchers +# otherwise keep rewriting it and restarting each other during requests. +sh ../../scripts/raft-hosted-build.sh +node --input-type=module - "$work/wrangler.toml" <<'JS' +import { readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +const config = readFileSync('wrangler.toml', 'utf8') + .replace(/^\[build\][\s\S]*?(?=^\[|$(?![\s\S]))/m, '') + .replace(/^main = "([^"]+)"/m, (_, path) => `main = ${JSON.stringify(resolve(path))}`); +writeFileSync(process.argv[2], config); +JS + start() { # port, extra wrangler args port=$1; shift + persist="$work/$port" + if [ "$port" = "${scenario_port:-}" ]; then persist=$scenario_state; fi : >"$work/$port.log" - node_modules/.bin/wrangler dev --local --ip 127.0.0.1 --port "$port" --persist-to "$work/$port" "$@" >"$work/$port.log" 2>&1 & + # Reuse the opt-in local proxy transport policy; retries remain off by default. + sh test/local.sh node_modules/.bin/wrangler dev --config "$work/wrangler.toml" --local --ip 127.0.0.1 --port "$port" --persist-to "$persist" "$@" >"$work/$port.log" 2>&1 & last=$! pids="$pids $last" for _ in $(seq 60); do grep -q 'Ready on' "$work/$port.log" && return 0; sleep 1; done cat "$work/$port.log"; return 1 } +scenario_count=0 +restartScenario() { # port, previous pid, limits, label, extra wrangler args + scenario_port=$1; previous_pid=$2; scenario_limits=$3; scenario_label=$4; shift 4 + kill "$previous_pid"; wait "$previous_pid" 2>/dev/null || true + # Prior scenarios' durable alarms must not reactivate groups in this isolate. + scenario_count=$((scenario_count + 1)) + scenario_state="$work/scenario-$scenario_count-$scenario_port-$scenario_label" + start "$scenario_port" --var "CAF_LIMITS:$scenario_limits" "$@" +} + TIGHT='{"newIdentitiesPerIpPerDay":3,"requestsPerMinute":40,"requestsPerMinutePerDevice":25,"framesPerDay":6,"storeBytes":3000,"storeObjects":12,"concurrentPuts":1,"pairTtlMs":3000,"pairCreatePerHour":4,"pairWritePerMinute":10,"pairConcurrentPolls":2,"pairMaxBoxes":4}' # The default relay takes any number of new identities from one address, as the conformance suite # makes one per case; the tight one keeps the short limits, and the third is for the restart check. @@ -29,28 +54,65 @@ TIGHT='{"newIdentitiesPerIpPerDay":3,"requestsPerMinute":40,"requestsPerMinutePe # It also has the shortest grace and a two-object sweep page, so the deletion case (and the page loop) run in seconds. OPEN='{"newIdentitiesPerIpPerDay":1000000,"pairTtlMs":4000,"minGraceMs":2000,"sweepPageSize":2,"requestsPerMinute":100000,"requestsPerMinutePerDevice":100000}' start 18791 --var "CAF_LIMITS:$OPEN" --var TRUST_PROXY:1 +default_pid=$last +restartDefault() { + restartScenario 18791 "$default_pid" "$OPEN" "$1" --var TRUST_PROXY:1 + default_pid=$last + default_state=$scenario_state +} +scenario() { + script=$1; shift + restartDefault "$script" + IDENTITY_DO_DIR="$default_state/v3/do/codeaf-hosted-relay-IdentityDO" R2_DIR="$default_state/v3/r2/miniflare-R2BucketObject" node "test/$script.e2e.mjs" "$@" +} start 18792 --var "CAF_LIMITS:$TIGHT" +tight_pid=$last +restartTight() { + restartScenario 18792 "$tight_pid" "$TIGHT" "$1" + tight_pid=$last + tight_state=$scenario_state +} start 18794 --var 'CAF_LIMITS:{"newIdentitiesPerIpPerDay":1000000,"pairMaxBoxes":2}' --var TRUST_PROXY:1 # The watch cap case needs a relay that lets one identity hold five sockets, not a thousand. start 18796 --var 'CAF_LIMITS:{"newIdentitiesPerIpPerDay":1000000,"maxWatchers":5}' --var TRUST_PROXY:1 # Link pairing: a 3 s request life, decided requests kept 1.5 s, 40 live requests, 6 creates an hour, 3 misses a minute. LINK='{"newIdentitiesPerIpPerDay":1000000,"pairTtlMs":3000,"linkDecidedKeepMs":1500,"linkMaxLive":40,"linkCreatePerHour":6,"linkMissPerMinute":3,"linkReadPerMinute":500,"requestsPerMinute":100000,"requestsPerMinutePerDevice":100000}' +# Capacity needs forty simultaneously live requests. Use the production TTL +# only there; expiry and every other link case retain the short test TTL. +LINK_CAPACITY=$(node -e 'const limits=JSON.parse(process.argv[1]);delete limits.pairTtlMs;console.log(JSON.stringify(limits));' "$LINK") start 18797 --var "CAF_LIMITS:$LINK" --var TRUST_PROXY:1 +link_pid=$last +restartLink() { + link_limits=$LINK + if [ "$1" = RelayFull ]; then link_limits=$LINK_CAPACITY; fi + restartScenario 18797 "$link_pid" "$link_limits" "$1" --var TRUST_PROXY:1 + link_pid=$last +} # A relay served under /fabric, as the hosted deployment is: the same wires, beneath one prefix. start 18798 --var "CAF_LIMITS:$OPEN" --var TRUST_PROXY:1 --var BASE_PATH:/fabric node test/base.e2e.mjs RELAY=http://127.0.0.1:18798/fabric node test/api.e2e.mjs -for script in api race flight dedup; do node "test/$script.e2e.mjs"; done -node test/caps.e2e.mjs +for script in api race flight dedup; do scenario "$script"; done +for cap in $(node test/caps.e2e.mjs --list); do + restartTight "$cap" + node test/caps.e2e.mjs "$cap" +done +restartTight pair node test/pair.e2e.mjs -node test/link.e2e.mjs -node test/watch.e2e.mjs -node test/watch.e2e.mjs lease -WATCH_LOG="$work/18791.log" node test/watch.e2e.mjs idle -IDENTITY_DO_DIR="$work/18791/v3/do/codeaf-hosted-relay-IdentityDO" R2_DIR="$work/18791/v3/r2/miniflare-R2BucketObject" node test/rotation.e2e.mjs -IDENTITY_DO_DIR="$work/18792/v3/do/codeaf-hosted-relay-IdentityDO" node test/newcomers.e2e.mjs +for link_case in $(node test/link.e2e.mjs --list); do + restartLink "$link_case" + node test/link.e2e.mjs "$link_case" +done +for watch_case in $(node test/watch.e2e.mjs --list); do scenario watch "$watch_case"; done +scenario watch lease +WATCH_LOG="$work/18791.log" scenario watch idle +scenario rotation +restartTight newcomers +# The runtime creates this namespace lazily; the initial stored count is zero. +mkdir -p "$tight_state/v3/do/codeaf-hosted-relay-IdentityDO" +IDENTITY_DO_DIR="$tight_state/v3/do/codeaf-hosted-relay-IdentityDO" node test/newcomers.e2e.mjs # Counts survive the relay process: count, wait for the flush, stop the relay, start it again over the same port (so the same storage). PERSIST='{"newIdentitiesPerIpPerDay":1000000,"statsFlushMs":300}' diff --git a/relay/hosted/test/flight.e2e.mjs b/relay/hosted/test/flight.e2e.mjs index 75622cf1e7..58eba4ef6f 100644 --- a/relay/hosted/test/flight.e2e.mjs +++ b/relay/hosted/test/flight.e2e.mjs @@ -20,6 +20,9 @@ const myOther = await newDevice(mine.identity); const theirs = await newDevice(await newIdentity()); const f = frame([rid(1)]); +// Measure isolation between active identities, separately from cold preparation. +for (const device of [mine, theirs]) assert.equal((await call(device, 'GET', '/v1/store/stats')).status, 200); + const putting = trickle(await signed(mine, 'POST', '/v1/store/frames', f.bytes), '/v1/store/frames', f.bytes, PAUSE_MS); await sleep(300); diff --git a/relay/hosted/test/link.e2e.mjs b/relay/hosted/test/link.e2e.mjs index 1730681a7c..1dbdbc4500 100644 --- a/relay/hosted/test/link.e2e.mjs +++ b/relay/hosted/test/link.e2e.mjs @@ -1,6 +1,7 @@ // Link pairing end to end (docs/ux-pairing-contract.md): create, read, approve, deny, presence, // and every limit the contract names. Runs against a relay started with a 3 s request time to live, // decided requests kept 1.5 s, 40 live requests, 6 creates an hour, 3 misses a minute (see e2e.sh). +// The isolated capacity case uses production pending expiry to keep all forty requests live. // Each case that needs its own allowances comes from its own IP (X-Forwarded-For). import assert from 'node:assert/strict'; import { randomBytes } from 'node:crypto'; @@ -23,6 +24,7 @@ async function open(method, path, { ip = freshIp(), body } = {}) { async function member() { const identity = await newIdentity(); const approver = await newDevice(identity); + assert.equal((await call(approver, 'GET', '/v1/dir/list', undefined, { base: BASE })).status, 200); const newcomer = async (platform) => { const device = await newDevice(identity); const pubkey = Buffer.from(device.cert.device, 'hex'); @@ -48,7 +50,8 @@ const asked = async (m, platform) => { return { n, ...made.json }; }; -const limits = (await open('GET', '/v1/link/limits')).json; +const selected = process.argv[2]; +const limits = selected === '--list' ? null : (await open('GET', '/v1/link/limits')).json; const tests = { async Limits() { @@ -92,13 +95,15 @@ const tests = { const listed = (await call(m.approver, 'GET', '/v1/dir/list', undefined, { base: BASE })).json.devices[device]; assert.deepEqual([listed.platform, listed.revoked, typeof listed.created], ['linux', false, 'number'], 'the key is linked to the identity'); assert.equal((await approve(m, code, body)).status, 204, 'the same approve again is idempotent'); + const changed = await approve(m, code, { ...body, grant: b64u(randomBytes(64)) }); + assert.deepEqual([changed.status, answerOf(changed)], [409, 'already_decided'], 'a different grant cannot replace the committed decision'); assert.equal(answerOf(await deny(m, code)), 'already_decided'); }, async ClientCannotSetCreatedOrLastSeen() { const m = await member(); const { n, code, device } = await asked(m); - await approve(m, code, approveBody(n, { device: { device: { created: 5, last_seen: 5 } } })); + assert.equal((await approve(m, code, approveBody(n, { device: { created: 5, last_seen: 5 } }))).status, 204); const rec = (await call(m.approver, 'GET', '/v1/dir/list', undefined, { base: BASE })).json.devices[device]; assert.ok(rec.created > 1_700_000_000_000 && rec.last_seen === undefined, JSON.stringify(rec)); }, @@ -116,7 +121,8 @@ const tests = { async RaceHasOneWinner() { const m = await member(); const { n, code } = await asked(m); - const [a1, d1, a2, d2] = await Promise.all([approve(m, code, approveBody(n)), deny(m, code), approve(m, code, approveBody(n)), deny(m, code)]); + const body = approveBody(n); + const [a1, d1, a2, d2] = await Promise.all([approve(m, code, body), deny(m, code), approve(m, code, body), deny(m, code)]); const state = (await read(code)).json.state; const [wins, loses] = state === 'approved' ? [[a1, a2], [d1, d2]] : [[d1, d2], [a1, a2]]; assert.deepEqual([...wins, ...loses].map((a) => a.status), [204, 204, 409, 409], `the first decision (${state}) wins and the other is refused`); @@ -151,6 +157,26 @@ const tests = { assert.equal(answerOf(await approve(m, again.json.code, approveBody(first.n))), 'bad_request'); }, + async InvalidApprovalDoesNotReserveTheRequest() { + const owner = await member(), other = await member(); + for (const invalid of ['certificate', 'identity', 'grant', 'device']) { + const { n, code, device } = await asked(owner); + const cert = { ...n.device.cert, sig: '' }; + cert.sig = Buffer.from(await crypto.subtle.sign('Ed25519', other.identity.pair.privateKey, new TextEncoder().encode('codeaf/device-cert/v1\n' + JSON.stringify(cert)))).toString('hex'); + const body = approveBody({ ...n, cert: b64u(JSON.stringify(cert)) }); + if (invalid === 'certificate') body.cert = 'invalid'; + if (invalid === 'identity') body.cert = n.cert; + if (invalid === 'grant') body.grant = b64u(randomBytes(4097)); + if (invalid === 'device') body.device = []; + const refused = await approve(other, code, body); + assert.deepEqual([refused.status, answerOf(refused)], invalid === 'grant' ? [413, 'too_big'] : [400, 'bad_request'], invalid); + assert.equal((await read(code)).json.state, 'pending', invalid); + const listed = (await call(other.approver, 'GET', '/v1/dir/list', undefined, { base: BASE })).json; + assert.equal(listed.devices[device], undefined, invalid); + assert.equal((await approve(owner, code, approveBody(n))).status, 204, `${invalid} did not reserve the request for another identity`); + } + }, + async ExpiryIsGone() { const m = await member(); const { n, code } = await asked(m); @@ -220,14 +246,19 @@ const tests = { }, async RelayFull() { - await sleep(limits.ttl_ms + 300); + // The selected capacity case starts in a fresh namespace in e2e.sh. + if (!selected) await sleep(limits.ttl_ms + 300); const m = await member(); - let ip = freshIp(); - for (let i = 0; i < limits.max_live; i++) { - if (i % limits.pending_per_ip === 0) ip = freshIp(); - assert.equal((await create((await m.newcomer()).body, ip)).status, 201, `request ${i}`); - } - const full = await create((await m.newcomer()).body, freshIp()); + // Prepare keys first so the capacity check measures live request slots. + const requests = await Promise.all(Array.from({ length: limits.max_live + 1 }, () => m.newcomer())); + const ips = Array.from({ length: Math.ceil(limits.max_live / limits.pending_per_ip) }, () => freshIp()); + const started = Date.now(); + const made = await Promise.all(requests.slice(0, limits.max_live).map((n, i) => create(n.body, ips[Math.floor(i / limits.pending_per_ip)]))); + for (const [i, answer] of made.entries()) assert.equal(answer.status, 201, `request ${i}`); + const full = await create(requests.at(-1).body, freshIp()); + const finished = Date.now(), expires = made.map((answer) => answer.json.expires_at); + console.log('link live capacity', JSON.stringify({ live: expires.filter((at) => at > finished).length, span_ms: finished - started, earliest_expiry_ms: Math.min(...expires) - finished, status: full.status })); + assert.ok(expires.every((at) => at > finished), 'all requests remain live when capacity is checked'); assert.deepEqual([full.status, full.json.err], [503, 'full']); }, @@ -253,8 +284,14 @@ const tests = { }, }; -for (const [name, run] of Object.entries(tests)) { - await run(); - console.log('link ok', name); +if (selected === '--list') { + console.log(Object.keys(tests).join('\n')); +} else { + const cases = Object.entries(tests).filter(([name]) => !selected || name === selected); + if (!cases.length) throw new Error(`unknown link case: ${selected}`); + for (const [name, run] of cases) { + await run(); + console.log('link ok', name); + } + console.log('link: all pass'); } -console.log('link: all pass'); diff --git a/relay/hosted/test/race.e2e.mjs b/relay/hosted/test/race.e2e.mjs index 6c895ad0b5..0b39f0f436 100644 --- a/relay/hosted/test/race.e2e.mjs +++ b/relay/hosted/test/race.e2e.mjs @@ -22,8 +22,11 @@ const pubs = tally(await Promise.all(Array.from({ length: 50 }, (_, i) => call(o assert.deepEqual(pubs, { 200: 1, head_moved: 49 }); console.log('head race', JSON.stringify(pubs)); +const blind = await call(rivals[0], 'POST', '/v1/dir/cells/heads/acquire', { force: true }); +assert.deepEqual([blind.status, blind.json.err], [409, 'lease_held']); +const observed = (await call(rivals[0], 'GET', '/v1/dir/cells/heads')).json.cell.lease; const t0 = performance.now(); -const took = await call(rivals[0], 'POST', '/v1/dir/cells/heads/acquire', { force: true }); +const took = await call(rivals[0], 'POST', '/v1/dir/cells/heads/acquire', { force: true, expect: { device: observed.device, fence: observed.fence } }); const ms = Math.round(performance.now() - t0); assert.equal(took.json.cell.lease.fence, fence + 1); const stale = await call(owner, 'POST', '/v1/dir/cells/heads/publish', { fence, old_head: 'h0', head: 'hx', size: 1, class: 'work', pending: 0 }); diff --git a/relay/hosted/test/raft/hosted-bootstrap.test.js b/relay/hosted/test/raft/hosted-bootstrap.test.js new file mode 100644 index 0000000000..1633d9fcb9 --- /dev/null +++ b/relay/hosted/test/raft/hosted-bootstrap.test.js @@ -0,0 +1,105 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { hostedFixture } from './hosted-fixture.js'; + +test('the first write does not recover peers outside the committed singleton configuration',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start(); + assert.equal((await h.call(a,'PUT','/v1/dir/devices/'+a.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + const metrics=await h.control({op:'metrics'}); + assert.equal(metrics.idleReads,0,'the initial transport view has no prior delivery to refresh'); + assert.equal(metrics.peerFailures,0,'unconfigured anchors have no failed Raft delivery to recover'); + }finally{await h.close();} +}); + +test('a new identity commits its first write without either additional anchor',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start({needle:' member(id){',after:"if(String(id)!=='1')throw new Error('fixture: additional anchors unavailable');"}); + const made=await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}}); + assert.equal(made.status,200,JSON.stringify(made)); + const source=JSON.parse((await h.control({op:'source'})).gate),cluster=source.config.cluster; + assert.deepEqual(source.birth.voters,['1']); + for(const node of ['2','3'])assert.equal(await h.control({op:'has-member',cluster,node}),false); + const read=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(read.status,200);assert.equal(read.body.cell.head,'first'); + }finally{await h.close();} +}); + + +test('background formation preserves the first write through joint consensus and restart',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}})).status,200); + let state; + for(let turn=0;turn<16;turn++){ + state=await h.control({op:'formation-turn'}); + if(state.registry)break; + } + assert.equal(state.registry,true,JSON.stringify(state));assert.deepEqual(state.conf.voters,['1','2','3']);assert.equal(state.conf.outgoing?.length??0,0); + await h.start();const read=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(read.status,200);assert.equal(read.body.cell.head,'first'); + }finally{await h.close();} +}); + +test('recovery after singleton publication retains immutable birth provenance',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start({needle:'if(!owned){await this.source.own(receipts);',after:"throw new Error('fixture: crash after singleton publication');"}); + assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}})).status,500); + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}})).status,200); + let state;for(let turn=0;turn<16;turn++){state=await h.control({op:'formation-turn'});if(state.registry)break;} + assert.equal(state.registry,true,JSON.stringify(state)); + assert.deepEqual(JSON.parse((await h.control({op:'source'})).gate).birth.voters,['1']); + }finally{await h.close();} +}); + +test('singleton formation keeps renewing live holds while additional anchors remain unavailable',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true,now:Date.now()}),a=h.client(),sockets=[]; + const watch=async()=>{const response=await h.request(a,'GET','/v1/dir/watch?hold=chat:1&receipts=1',undefined,{headers:{upgrade:'websocket'}});assert.equal(response.status,101);response.webSocket.accept();sockets.push(response.webSocket);}; + try{ + await h.start({needle:' member(id){',after:"if(String(id)!=='1')throw new Error('fixture: additional anchors unavailable');"}); + const created=await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}});assert.equal(created.status,200); + await watch();let expires=(await h.call(a,'GET','/v1/dir/cells/chat')).body.cell.lease.expires; + for(let turn=0;turn<4;turn++){ + await h.control({op:'shift-clock',records:20_000});await watch(); + await h.control({op:'shift-clock',records:10_000});await h.control({op:'formation-turn'}); + const cell=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(cell.status,200);assert(cell.body.cell.lease.expires>expires,'maintenance must renew fresh watch evidence during formation');expires=cell.body.cell.lease.expires; + } + }finally{for(const socket of sockets)socket.close();await h.close();} +}); + + +test('a lost published singleton is not recreated from the frozen source',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),cut={needle:' member(id){',after:"if(String(id)!=='1')throw new Error('fixture: additional anchors unavailable');"}; + try{ + await h.start(cut);assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}})).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + await h.control({op:'discard-unformed-member',cluster,node:'1'});await h.start(cut); + assert.equal((await h.call(a,'GET','/v1/dir/cells/chat')).status,503); + assert.equal(await h.control({op:'has-member',cluster,node:'1'}),false); + }finally{await h.close();} +}); + +test('formation resumes from durable configuration after every background turn',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'first',class:'work',size:0,keys:{}})).status,200); + let state; + for(let turn=0;turn<16;turn++){ + state=await h.control({op:'formation-turn'});await h.start(); + const read=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(read.status,200,JSON.stringify(state));assert.equal(read.body.cell.head,'first'); + if(state.registry)break; + } + assert.equal(state.registry,true,JSON.stringify(state)); + }finally{await h.close();} +}); + +test('an idle singleton releases memory when additional anchors remain unavailable',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),cut={needle:' member(id){',after:"if(String(id)!=='1')throw new Error('fixture: additional anchors unavailable');"}; + try{ + await h.start(cut);assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const idle=await h.control({op:'idle-schedule'});assert.equal(idle.sleeping,true);assert.equal(idle.recurringMemberCalls,0); + assert.equal((await h.control({op:'metrics'})).active,0); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200,'the durable singleton can wake through its actual quorum'); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-capacity.test.js b/relay/hosted/test/raft/hosted-capacity.test.js index d0545809f3..a065e01b94 100644 --- a/relay/hosted/test/raft/hosted-capacity.test.js +++ b/relay/hosted/test/raft/hosted-capacity.test.js @@ -56,10 +56,10 @@ test('r4.3 incomplete sleep retains busy members and full admission retries afte try{ await h.start();const sleeping=household(h);assert.equal((await sleeping.request()).status,200);await idle(sleeping); let next,refusal; - for(let n=0;n<5;n++){ + for(let n=0;n<16;n++){ const a=household(h),reply=await a.request(); if(reply.status!==200){next=a;refusal=reply;break;} - const states=await a.control('retention-state','remember');a.busyNodes=Object.keys(states).filter(node=>states[node].active);assert(a.busyNodes.length>=2); + const states=await a.control('retention-state','remember');a.busyNodes=Object.keys(states).filter(node=>states[node].active);assert(a.busyNodes.length>=1); busy.push(a);await a.control('pause-public'); } assert(next,'busy activation exceeded the admission budget');assert(busy.length>0); @@ -76,7 +76,8 @@ test('r4.3 incomplete sleep retains busy members and full admission retries afte const compact=m=>({active:m.active,reservedBytes:m.reservedBytes,memory:m.memory}); console.log('busy admission',JSON.stringify({before:compact(before),released:compact(released),retry:retry.status,final:compact(await h.control({op:'metrics'}))})); assert.equal(retry.status,200,'released reservations did not admit the retry'); - assert(busy.length>=2);await busy[1].control('resume-public');await idle(busy[1]); + assert(busy.length>=2); + for(const a of busy.slice(1)){await a.control('resume-public');await idle(a);} assert.equal((await sleeping.request()).status,200,'refused wake did not recover after capacity became available'); }finally{for(const a of busy)await a.control('resume-public').catch(()=>{});await h.close();} }); @@ -84,7 +85,7 @@ test('r4.3 incomplete sleep retains busy members and full admission retries afte test('r4.4 released members stay silent, fence stale references and wake on signed input',{timeout:60000},async()=>{ const h=await hostedFixture({fixture:true}),a=household(h); try{ - await h.start();assert.equal((await a.request()).status,200);await a.control('retention-state','remember');await idle(a); + await h.start();assert.equal((await a.request()).status,200);await a.control('form-cluster');await a.control('retention-state','remember');await idle(a); const stale=await a.control('retention-state','stale'); for(const state of Object.values(stale)){assert.deepEqual(state.refused,[true,true,true]);assert.equal(state.handle,0);assert.equal(state.stopped,true);} const before=await a.control('quiet-state');assert.equal(before.alarm,null);assert.equal(before.maintenance,false);assert.deepEqual(before.memberAlarms,{'1':null,'2':null,'3':null}); diff --git a/relay/hosted/test/raft/hosted-fallback.test.js b/relay/hosted/test/raft/hosted-fallback.test.js index e17c3620c4..3bcf1a0666 100644 --- a/relay/hosted/test/raft/hosted-fallback.test.js +++ b/relay/hosted/test/raft/hosted-fallback.test.js @@ -12,7 +12,7 @@ test('R7 pending membership cannot stop surviving follower election clocks',asyn }])); members.set('3',{drainRecovery:async()=>{throw Error('standby unavailable');}}); const host=Object.assign(Object.create(HostedIdentity.prototype),{ - asleep:false,wakeVersion:0,started:0,elapsed:0,operations:1,membershipPending:true, + asleep:false,wakeVersion:0,started:0,elapsed:0,operations:1,membershipPending:true,seen:new Map(), manifest:{config:{anchors:['1','2','3']}},members,states:new Map(),queue:[],reads:[],waiters:new Set(),progressVersion:0, holds:{next:null},policy:{tick_every:5000},forwarding:{pending:new Map(),socket:()=>null}, peers:{view:{'4':{voter:true}},boxes:new Map(),snapshots:new Map(),retryTargets:new Set()}, diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index aef087ae70..34d22ad1c6 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -51,10 +51,10 @@ export async function hostedFixture({fixture=false,base='',go=false,rootLabel,no mf=new Miniflare(convertV4MiniflareOptions({log:diagnostics,handleStructuredLogs:entry=>diagnostics.capture(entry.level,entry.message),host:'127.0.0.1',inspectorPort:0,resourcePersistencePath:join(dir,'persist'),telemetry:{enabled:false},cf:false,workers:[worker]})); await mf.ready;return mf; }; - const request=async(c,method,path,body)=>{ + const request=async(c,method,path,body,options={})=>{ const raw=body===undefined?'':body instanceof Uint8Array?body:JSON.stringify(body),at=String(Date.now()); const headers={'Codeaf-Identity':rootKey.toString('base64url'),'Codeaf-Cert':Buffer.from(JSON.stringify(c.cert)).toString('base64url'),'Codeaf-Time':at,'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\n'+method+'\n'+prefix+path+'\n'+at+'\n'+sha(raw)),c.key.privateKey).toString('base64url')}; - return mf.dispatchFetch('http://local'+prefix+path,{method,headers,...(method!=='GET'&&raw?{body:raw}:{})}); + return mf.dispatchFetch('http://local'+prefix+path,{method,headers:{...headers,...options.headers},...(method!=='GET'&&raw?{body:raw}:{})}); }; const call=async(...args)=>{const response=await request(...args),text=await response.text();return {status:response.status,body:text?JSON.parse(text):null};}; const control=async value=>{assert(fixture);const r=await mf.dispatchFetch('http://local/fixture',{method:'POST',body:JSON.stringify({identity,...value})});assert.equal(r.status,200,await r.clone().text());return r.json();}; @@ -69,5 +69,5 @@ export async function hostedFixture({fixture=false,base='',go=false,rootLabel,no certificate={v:1,identity,...certificate,signature:null}; certificate.signature=sign(null,Buffer.from('codeaf/worker-cert/v1\0'+encodeCanonical(certificate)),root.privateKey).toString('base64');return certificate; }; - return {identity,client,dir,start,request,call,control,api,delegate,diagnostics,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; + return {identity,rootPublicKey:rootKey,client,dir,start,request,call,control,api,delegate,diagnostics,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; } diff --git a/relay/hosted/test/raft/hosted-forward.test.js b/relay/hosted/test/raft/hosted-forward.test.js index 455665a2b9..f55fc76b60 100644 --- a/relay/hosted/test/raft/hosted-forward.test.js +++ b/relay/hosted/test/raft/hosted-forward.test.js @@ -89,7 +89,7 @@ test('R6 a stale disconnected hint cannot allocate a second forwarding ID',async test('R6 R9 HTTP catch-up lets peer progress finish before returning an empty response',{timeout:peerLimits(runtime).ForwardTimeout},async()=>{ let delivered=false; const peer=Object.assign(Object.create((await import('../../src/raft/peers.js')).PersonalPeers.prototype),{tail:Promise.resolve(),pending:0,view:{'5':{voter:false}},refresh:async()=>{},validate:()=>{},wire:()=>new TextEncoder().encode(JSON.stringify({messages:delivered?['append']:[]})),exchange:async()=>({request:{node:'5',applied:'14',messages:[]},raw:new TextEncoder().encode('{"messages":[]}'),rejected:false})}); - const host={peers:peer,asleep:false,leader:()=>undefined,leaderID:()=> '4',replica:()=> '1',states:new Map([['1',{applied:15n}]]),forwarding:{socket:()=>({})},forward:async(node,work)=>{assert.equal(node,'4');assert.equal(work.action,'poke');await peer.serial(()=>{delivered=true;});return {barrier:'15'};}}; + const host={invalidateRead(){},peers:peer,asleep:false,leader:()=>undefined,leaderID:()=> '4',replica:()=> '1',states:new Map([['1',{applied:15n}]]),forwarding:{socket:()=>({})},forward:async(node,work)=>{assert.equal(node,'4');assert.equal(work.action,'poke');await peer.serial(()=>{delivered=true;});return {barrier:'15'};}}; peer.host=host; const result=await peer.httpExchange(new Uint8Array());assert.deepEqual(JSON.parse(new TextDecoder().decode(result.raw)).messages,['append']); }); @@ -99,7 +99,7 @@ test('R6 R9 HTTP catch-up lets peer progress finish before returning an empty re test('R6 R9 enrollment retains admitted traffic between unchanged members',async()=>{ const r=rig(),members={'1':{incarnation:'1',key:r.anchor.raw,voter:true,revoked:false},'4':{incarnation:'1',key:r.personal.raw,voter:true,revoked:false}}; let view=members; - const host={runtime,config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>null},input:async()=>{},hash:()=>'',asleep:false}; + const host={invalidateRead(){},runtime,config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>null},input:async()=>{},hash:()=>'',asleep:false}; const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[];await peers.refresh(); peers.enqueue([{from:'4',to:'1',sequence:'7',payload:'AQ=='}]); const before=structuredClone(peers.deliveries('1')),used=[...peers.used]; @@ -114,7 +114,7 @@ test('R6 R9 enrollment retains admitted traffic between unchanged members',async test('R6 R9 overlapping membership refreshes leave the latest committed view',async()=>{ const first=Promise.withResolvers(),second=Promise.withResolvers();let calls=0; const old={'1':{incarnation:'1',key:'anchor',voter:true,revoked:false}},newer={...old,'5':{incarnation:'1',key:'personal',voter:false,revoked:false}}; - const host={runtime,config:()=>({scope:rig().host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:()=>++calls===1?first.promise:second.promise}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1'],forwarding:{socket:()=>null},input:async()=>{},asleep:false}; + const host={invalidateRead(){},runtime,config:()=>({scope:rig().host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:()=>++calls===1?first.promise:second.promise}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1'],forwarding:{socket:()=>null},input:async()=>{},asleep:false}; const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[]; const earlier=peers.refresh();host.states.get('1').applied=14n;const later=peers.refresh(); second.resolve(newer);await Promise.resolve();await Promise.resolve();first.resolve(old); @@ -125,7 +125,7 @@ test('R6 R9 overlapping membership refreshes leave the latest committed view',as // queue must obtain a new quorum round without holding the peer queue. test('R6 R27 a drained voter queue retries a dropped quorum round',async()=>{ const r=rig(),view={'1':{incarnation:'1',key:r.anchor.raw,voter:true,revoked:false},'4':{incarnation:'1',key:r.personal.raw,voter:true,revoked:false}},reply=Promise.withResolvers(),actions=[]; - const host={runtime,codec:new HostCodec(runtime),config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>({}),begin:async(node,work)=>{actions.push([node,work.action]);return {id:'retry',promise:reply.promise};}},serial:fn=>fn(),input:async()=>{},pump:async()=>{},wakeMaintenance:async()=>{},asleep:false}; + const host={invalidateRead(){},runtime,codec:new HostCodec(runtime),config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>({}),begin:async(node,work)=>{actions.push([node,work.action]);return {id:'retry',promise:reply.promise};}},serial:fn=>fn(),input:async()=>{},pump:async()=>{},wakeMaintenance:async()=>{},asleep:false}; const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[];await peers.refresh();actions.length=0; peers.anchors=async()=>{};peers.chunks=async()=>{};peers.push=async()=>{};peers.validate=raw=>({request:JSON.parse(new TextDecoder().decode(raw)),obsolete:new Set()}); for(let n=0;n{ const cuts=[ - {name:'freeze',needle:'if(!this.source.gate())await this.source.freeze(this.provisioning.config);'}, + {name:'freeze',needle:'if(!this.source.gate())await this.source.freeze(this.provisioning.config,this.provisioning.singleton===true);'}, {name:'inventory',needle:'this.manifest=await this.source.inventory();'}, ...['1','2','3'].map(id=>({name:'stage-'+id,needle:'await this.member(id).legacyImport(this.config(id),this.manifest,copied);',after:`if(id==='${id}')throw new Error('H2 injected process cut');`})), ...['1','2','3'].map(id=>({name:'install-'+id,needle:'await this.member(id).legacyImport(this.config(id),this.manifest,null,true);',after:`if(id==='${id}')throw new Error('H2 injected process cut');`})), {name:'readback',needle:'await this.pump();await this.elect();await this.barrier();'}, - {name:'ownership',needle:'if(!owned)await this.source.own(receipts);'}, + {name:'ownership',needle:'if(!owned){await this.source.own(receipts);'}, ]; for(const cut of cuts){ const h=await hostedFixture({fixture:true}),a=h.client(); diff --git a/relay/hosted/test/raft/hosted-presence.test.js b/relay/hosted/test/raft/hosted-presence.test.js new file mode 100644 index 0000000000..8c2271e0d1 --- /dev/null +++ b/relay/hosted/test/raft/hosted-presence.test.js @@ -0,0 +1,93 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { hostedFixture } from './hosted-fixture.js'; +import { encode } from '../../src/frame.js'; + +test('blob requests do not synchronously commit advisory device presence',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start(); + assert.equal((await h.call(a,'PUT','/v1/dir/devices/'+a.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + await h.control({op:'reset-metrics'}); + const rid='a'.repeat(64),frame=encode('0'.repeat(32),[{rid,bytes:new TextEncoder().encode('AGEO\x01sealed')}]); + assert.equal((await h.call(a,'POST','/v1/store/frames',frame)).status,200); + const get=await h.request(a,'GET','/v1/store/objects/'+rid);assert.equal(get.status,200);await get.arrayBuffer(); + assert.equal((await h.call(a,'POST','/v1/store/has',{rids:[rid]})).status,200); + assert.equal((await h.control({op:'metrics'})).presenceCommits,0); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + assert.equal((await h.control({op:'metrics'})).presenceCommits,1,'one maintenance commit covers all three requests'); + const list=await h.call(a,'GET','/v1/dir/list');assert.equal(list.status,200);assert(list.body.devices[a.device].last_seen>0); + }finally{await h.close();} +}); + +test('watch departure coalesces presence without a synchronous proposal',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'PUT','/v1/dir/devices/'+a.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + const watch=await h.request(a,'GET','/v1/dir/watch',undefined,{headers:{upgrade:'websocket'}});assert.equal(watch.status,101); + const socket=watch.webSocket;socket.accept(); + await h.control({op:'reset-metrics'}); + const closed=new Promise(resolve=>socket.addEventListener('close',resolve,{once:true}));socket.close(1000);await closed; + assert.equal((await h.control({op:'metrics'})).presenceCommits,0); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + assert.equal((await h.control({op:'metrics'})).presenceCommits,1); + }finally{await h.close();} +}); + +test('queued presence cannot restore a device revoked during a streamed take',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client(); + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + const sealed=new Uint8Array(2<<20);sealed.set(new TextEncoder().encode('AGEO\x01sealed')); + const frame=encode('0'.repeat(32),[{rid:'b'.repeat(64),bytes:sealed}]); + const put=await h.call(a,'POST','/v1/store/frames',frame);assert.equal(put.status,200); + const take=await h.request(b,'GET','/v1/store/frames/'+put.body.frame);assert.equal(take.status,200); + const reader=take.body.getReader(),first=await reader.read();assert(first.value.length>0); + assert.equal((await h.call(a,'POST','/v1/dir/devices/'+b.device+'/revoke')).status,204); + assert.deepEqual(await h.call(b,'GET','/v1/store/stats'),{status:401,body:{err:'revoked'}}); + let bytes=first.value.length;for(;;){const chunk=await reader.read();if(chunk.done)break;bytes+=chunk.value.length;} + assert.equal(bytes,frame.length); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + assert.deepEqual(await h.call(b,'GET','/v1/dir/list'),{status:401,body:{err:'revoked'}}); + const retained=await h.request(a,'GET','/v1/store/frames/'+put.body.frame);assert.equal(retained.status,200); + assert.deepEqual(new Uint8Array(await retained.arrayBuffer()),frame); + }finally{await h.close();} +}); + + +test('a committed revoke interrupts a held store poll before its upload finishes',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client();let upload,poll; + const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + await h.control({op:'hold-upload'}); + upload=h.call(a,'POST','/v1/store/frames',encode('0'.repeat(32),[{rid:'c'.repeat(64),bytes:new TextEncoder().encode('AGEO\x01sealed')}])); + poll=h.call(b,'POST','/v1/store/has',{rids:['c'.repeat(64)]}); + let waiting=false;for(let n=0;n<100&&!waiting;n++){await delay(10);waiting=await h.control({op:'has-waiting'});} + assert(waiting,'the authenticated store poll reached its real in-flight upload wait'); + assert.equal((await h.call(a,'POST','/v1/dir/devices/'+b.device+'/revoke')).status,204); + assert.equal(await Promise.race([poll.then(r=>r.status),delay(750).then(()=>null)]),401,'revocation releases a held poll on commit'); + }finally{await h.control({op:'release-upload'}).catch(()=>{});await Promise.allSettled([upload,poll].filter(Boolean));await h.close();} +}); + +test('revoking an upload keeps other devices waiting until its actual write finishes',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client();let upload,poll; + const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + await h.control({op:'hold-upload'}); + const rid='d'.repeat(64); + upload=h.call(b,'POST','/v1/store/frames',encode('0'.repeat(32),[{rid,bytes:new TextEncoder().encode('AGEO\x01sealed')}])); + poll=h.call(a,'POST','/v1/store/has',{rids:[rid]}); + let waiting=false;for(let n=0;n<100&&!waiting;n++){await delay(10);waiting=await h.control({op:'has-waiting'});} + assert(waiting,'the other device is waiting for the actual upload'); + assert.equal((await h.call(a,'POST','/v1/dir/devices/'+b.device+'/revoke')).status,204); + assert.equal(await Promise.race([upload.then(r=>r.status),delay(750).then(()=>null)]),401,'the revoked upload returns without waiting for storage'); + assert.equal(await Promise.race([poll.then(()=>true),delay(100).then(()=>false)]),false,'Has cannot overtake an upload that still writes'); + await h.control({op:'release-upload'}); + assert.deepEqual(await poll,{status:200,body:{have:[true]}}); + }finally{await h.control({op:'release-upload'}).catch(()=>{});await Promise.allSettled([upload,poll].filter(Boolean));await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-readpath.test.js b/relay/hosted/test/raft/hosted-readpath.test.js new file mode 100644 index 0000000000..169fbe10d5 --- /dev/null +++ b/relay/hosted/test/raft/hosted-readpath.test.js @@ -0,0 +1,117 @@ +import WebSocket from 'ws'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { hostedFixture } from './hosted-fixture.js'; +import { encode } from '../../src/frame.js'; +import { createHash, sign } from 'node:crypto'; +import { encodeCanonical } from '../../src/raft/codec.js'; + +test('warm blob and directory reads reuse confirmed authority without proposals or barriers',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + await h.control({op:'reset-metrics'}); + const raw=encode('0'.repeat(32),[{rid:'a'.repeat(64),bytes:new TextEncoder().encode('AGEO\x01sealed')}]); + assert.equal((await h.call(a,'POST','/v1/store/frames',raw)).status,200); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const metrics=await h.control({op:'metrics'}); + assert.equal(metrics.proposals,0); + assert.equal(metrics.barriers,0); + assert.equal(metrics.authorizations,2,'each request authorizes once'); + }finally{await h.close();} +}); + +test('native enrollment cannot revive a certificate expired at an observed read time',{timeout:30000},async()=>{ + const base=Date.now(),h=await hostedFixture({fixture:true,now:base}),a=h.client(),worker=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const certificate=h.delegate({cluster,key:worker.key.publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('base64'),made:base-1000,expires:base+15_000,cells:['chat'],learner:false}); + await h.control({op:'shift-clock',records:30_000}); + const observed=await h.call(a,'GET','/v1/dir/list');assert(observed.body.now>certificate.expires); + await h.control({op:'shift-clock',records:-20_000}); + const fields={cluster,actor:a.device,session:'clock-floor',sequence:'1',change:{enroll_worker:{certificate,grants:{chat:'gQ=='}}}}; + const command={schema:1,...fields,certificate:Buffer.from(JSON.stringify(a.cert)).toString('base64url'),request_digest:createHash('sha256').update(encodeCanonical(fields)).digest('hex'),now:Date.now(),signature:null}; + command.signature=sign(null,Buffer.from('codeaf.directory.command.v1\0'+encodeCanonical(command)),a.key.privateKey).toString('base64'); + await h.control({op:'reset-metrics'}); + const result=await h.api(a,{cluster,action:'submit',incarnation:'0',command}); + assert.equal((await h.control({op:'metrics'})).presenceCommits,0,'native ingress time enforcement does not synchronously commit presence'); + assert.equal(result.status,200,JSON.stringify(result));assert.equal(result.body.result.code,'bad_request'); + }finally{await h.close();} +}); + +test('proposal-free reads retain their observed time across clock regression and recovery',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true,now:Date.now()}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + await h.control({op:'shift-clock',records:30_000}); + const first=await h.call(a,'GET','/v1/dir/list');assert.equal(first.status,200); + await h.control({op:'shift-clock',records:-20_000}); + const second=await h.call(a,'GET','/v1/dir/list');assert.equal(second.status,200);assert(second.body.now>=first.body.now); + await h.start();const recovered=await h.call(a,'GET','/v1/dir/list');assert.equal(recovered.status,200);assert(recovered.body.now>=first.body.now); + }finally{await h.close();} +}); + +test('expired cached authority cannot answer from an isolated leader',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + await h.control({op:'expire-read-quorum'}); + const read=await h.call(a,'GET','/v1/dir/list');assert.equal(read.status,503);assert.equal(read.body.err,'unreachable'); + }finally{await h.close();} +}); + +test('a selected member recovering cannot reuse its previous activation certificate',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + await h.control({op:'replace-read-member'}); + const read=await h.call(a,'GET','/v1/dir/list');assert.equal(read.status,503);assert.equal(read.body.err,'unreachable'); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + }finally{await h.close();} +}); + +test('snapshot traffic invalidates warm authority before the following read',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}})).status,200); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + await h.control({op:'reset-metrics'}); + assert.equal(await h.control({op:'stale-snapshot'}),3); + const read=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(read.status,200);assert.equal(read.body.cell.head,'saved'); + assert((await h.control({op:'metrics'})).barriers>0); + }finally{await h.close();} +}); + + +test('directory reads reject signatures stale against previously observed time after clock rollback',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true,now:Date.now()}),a=h.client(),path='/v1/dir/list'; + try{ + await h.start();assert.equal((await h.call(a,'GET',path)).status,200); + const advance=6*60*1000,at=String(Date.now()+advance); + const signature=sign(null,Buffer.from('codeaf-req-v1\nGET\n'+path+'\n'+at+'\n'+createHash('sha256').update('').digest('hex')),a.key.privateKey).toString('base64url'); + await h.control({op:'shift-clock',records:advance}); + assert.equal((await h.call(a,'GET',path,undefined,{headers:{'Codeaf-Time':at,'Codeaf-Sig':signature}})).status,200); + await h.control({op:'shift-clock',records:-advance}); + assert.equal((await h.call(a,'GET',path)).status,401,'a regressed ingress clock cannot revive an old signed request'); + }finally{await h.close();} +}); + + +test('a replicated revoke closes the device watch before another public request',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client();let socket; + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + const path='/v1/dir/watch',at=String(Date.now()),url=new URL(path,await h.mf.ready);url.protocol='ws:'; + const headers={'Codeaf-Identity':h.rootPublicKey.toString('base64url'),'Codeaf-Cert':Buffer.from(JSON.stringify(b.cert)).toString('base64url'),'Codeaf-Time':at,'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\nGET\n'+path+'\n'+at+'\n'+createHash('sha256').update('').digest('hex')),b.key.privateKey).toString('base64url')}; + socket=new WebSocket(url,{headers,closeTimeout:200});await new Promise((resolve,reject)=>{socket.once('open',resolve);socket.once('error',reject);}); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const fields={cluster,actor:a.device,session:'external-revoke',sequence:'1',change:{revoke:{device:b.device}}}; + const command={schema:1,...fields,certificate:Buffer.from(JSON.stringify(a.cert)).toString('base64url'),request_digest:createHash('sha256').update(encodeCanonical(fields)).digest('hex'),now:Date.now(),signature:null}; + command.signature=sign(null,Buffer.from('codeaf.directory.command.v1\0'+encodeCanonical(command)),a.key.privateKey).toString('base64'); + const closed=new Promise(resolve=>{const timeout=setTimeout(()=>resolve(null),1500);socket.once('close',code=>{clearTimeout(timeout);resolve(code);});}); + assert.equal(await h.control({op:'direct-commit',records:Buffer.from(encodeCanonical(command)).toString('base64')}),'ok'); + assert.equal(await closed,4401,'committed revocation closes the watch without a sweep or subsequent request'); + }finally{socket?.close();await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-recovery.test.js b/relay/hosted/test/raft/hosted-recovery.test.js new file mode 100644 index 0000000000..fe67c06dea --- /dev/null +++ b/relay/hosted/test/raft/hosted-recovery.test.js @@ -0,0 +1,127 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {encode} from '../../src/frame.js'; +import {createHash,sign} from 'node:crypto'; +import {encodeCanonical} from '../../src/raft/codec.js'; +import {hostedFixture} from './hosted-fixture.js'; + +test('formed identity retries cold anchors after temporary activation capacity clears',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'POST','/v1/dir/cells/retained',{head:'retained-head',class:'work',size:7,keys:{}})).status,200); + assert.equal((await h.call(c,'POST','/v1/dir/cells/retained/release',{fence:1})).status,204); + await h.control({op:'form-cluster'}); + const formed=await h.call(c,'GET','/v2/cluster/control');assert.equal(formed.status,200); + assert.deepEqual(formed.body.configuration.voters.map(String),['1','2','3']); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + await h.control({op:'evict-idle'});await h.control({op:'transient-capacity',records:true}); + assert.equal((await h.call(c,'GET','/v1/dir/cells/retained')).status,503,'one formed voter cannot authorize a read'); + await h.control({op:'transient-capacity',records:false}); + const recovered=await h.call(c,'GET','/v1/dir/cells/retained'); + assert.equal(recovered.status,200,'healthy anchors must be retried without replacing the coordinator'); + assert.equal(recovered.body.cell.head,'retained-head'); + }finally{await h.close();} +}); + +test('ordinary member maintenance during a confirmed directory read does not refuse the read',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'interrupt-next-read'}); + const raced=await h.call(c,'GET','/v1/dir/list'); + assert.equal(raced.status,200,'safe maintenance during the read must refresh authority and finish'); + }finally{await h.close();} +}); + +for(const mode of ['no-quorum','advance'])test('interrupted read fails closed with '+mode,{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'interrupt-next-read',records:mode}); + const refused=await h.call(c,'GET','/v1/dir/list'); + assert.equal(refused.status,503);assert.equal(refused.body.err,'unreachable'); + }finally{await h.close();} +}); + +test('continuous harmless maintenance permits a fresh quorum-confirmed directory read',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'interrupt-next-read',records:'always'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200,'healthy quorum must remain readable during ordinary member maintenance'); + }finally{await h.close();} +}); + +test('interrupted upload authorization retries without repeating blob I/O',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'interrupt-next-read'}); + const frame=encode('0'.repeat(32),[{rid:'a'.repeat(64),bytes:new TextEncoder().encode('AGEO\x01sealed')}]); + assert.equal((await h.call(c,'POST','/v1/store/frames',frame)).status,200); + const stats=await h.call(c,'GET','/v1/store/stats');assert.equal(stats.status,200);assert.equal(stats.body.puts,1); + }finally{await h.close();} +}); + +test('serving identity retries anchors removed after a temporary peer RPC failure',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const c=h.client(); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200); + await h.control({op:'transient-peer-failure',records:true}); + assert.equal((await h.call(c,'POST','/v1/dir/cells/pending',{head:'pending-head',class:'work',size:7,keys:{}})).status,503); + await h.control({op:'transient-peer-failure',records:false}); + assert.equal((await h.call(c,'GET','/v1/dir/list')).status,200,'healthy anchors must recover without evicting a serving coordinator'); + }finally{await h.close();} +}); + +test('a revoke after selecting a row is enforced before a fresh-authority response finishes',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client(); + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + assert.equal((await h.call(a,'POST','/v1/dir/cells/retained',{head:'retained-head',class:'work',size:7,keys:{}})).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(b,'GET','/v1/dir/cells/retained')).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const fields={cluster,actor:a.device,session:'fresh-read-revoke',sequence:'1',change:{revoke:{device:b.device}}}; + const command={schema:1,...fields,certificate:Buffer.from(JSON.stringify(a.cert)).toString('base64url'),request_digest:createHash('sha256').update(encodeCanonical(fields)).digest('hex'),now:Date.now(),signature:null}; + command.signature=sign(null,Buffer.from('codeaf.directory.command.v1\0'+encodeCanonical(command)),a.key.privateKey).toString('base64'); + await h.control({op:'interrupt-next-read',records:{command:Buffer.from(encodeCanonical(command)).toString('base64'),after:4}}); + assert.deepEqual(await h.call(b,'GET','/v1/dir/cells/retained'),{status:401,body:{err:'revoked'}}); + }finally{await h.close();} +}); + +test('a revoke during fresh authorization cannot admit a new watch',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),b=h.client(); + try{ + await h.start(); + for(const c of [a,b])assert.equal((await h.call(c,'PUT','/v1/dir/devices/'+c.device,{V:1,name:'c2VhbGVk',platform:'linux'})).status,204); + assert.equal((await h.call(a,'POST','/v1/dir/cells/retained',{head:'retained-head',class:'work',size:7,keys:{}})).status,200); + await h.control({op:'form-cluster'}); + assert.equal((await h.call(b,'GET','/v1/dir/cells/retained')).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const fields={cluster,actor:a.device,session:'fresh-read-revoke',sequence:'1',change:{revoke:{device:b.device}}}; + const command={schema:1,...fields,certificate:Buffer.from(JSON.stringify(a.cert)).toString('base64url'),request_digest:createHash('sha256').update(encodeCanonical(fields)).digest('hex'),now:Date.now(),signature:null}; + command.signature=sign(null,Buffer.from('codeaf.directory.command.v1\0'+encodeCanonical(command)),a.key.privateKey).toString('base64'); + await h.control({op:'interrupt-next-read',records:{command:Buffer.from(encodeCanonical(command)).toString('base64'),after:3}}); + const response=await h.request(b,'GET','/v1/dir/watch',undefined,{headers:{Upgrade:'websocket'}}); + assert.equal(response.status,401); + assert.deepEqual(await response.json(),{err:'revoked'}); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-resources.test.js b/relay/hosted/test/raft/hosted-resources.test.js index 90de29a838..a0739a14a0 100644 --- a/relay/hosted/test/raft/hosted-resources.test.js +++ b/relay/hosted/test/raft/hosted-resources.test.js @@ -26,6 +26,7 @@ test('H4 R27 public Worker stays within isolate and ordinary-turn CPU budgets',{ try{ await h.start();inspector=await inspect(h.mf); const begin=performance.now();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}})).status,200);report.coldMs=performance.now()-begin; + await h.control({op:'form-cluster'}); await h.control({op:'reset-metrics'});const before=await sample();report.samples.push(before); for(let i=0;i<24;i++){ const start=performance.now(),r=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(r.status,200);report.calls.push(performance.now()-start); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index ce6ca32bc4..22cc9a56a4 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -11,7 +11,7 @@ import {CORE_MEMORY_BUDGET} from '../../src/raft/abi.js'; import { runtimeMetrics,sharedRuntime } from '../../src/raft/actor.js'; import {coordinationPolicy} from '../../src/raft/liveness.js'; const recoveryHistory=new Map(); -const measurements={turns:0,wall:[],membership:0,campaigns:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0}; +const measurements={turns:0,wall:[],membership:0,campaigns:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0,presenceCommits:0,proposals:0,barriers:0,authorizations:0,idleReads:0,peerFailures:0}; async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} import { policyOf } from '../../src/rules.js'; @@ -19,6 +19,14 @@ export class IdentityDO extends Source { constructor(ctx,env){ Date.now();if(globalThis.fixtureClockOffset){const setAlarm=ctx.storage.setAlarm.bind(ctx.storage);ctx.storage.setAlarm=at=>setAlarm(Number(at)-globalThis.fixtureClockOffset);} super(ctx,env); + let host; + Object.defineProperty(this,'host',{get:()=>host,set:value=>{ + host=value;if(!host)return; + const commit=host.commit.bind(host);host.commit=raw=>{this.lastCommand=raw.slice();return commit(raw);}; + const effect=host.commitEffect.bind(host);host.commitEffect=(change,...args)=>{if(change.device_seen)measurements.presenceCommits++;return effect(change,...args);}; + const input=host.input.bind(host);host.input=(node,tag,...args)=>{if(tag===15)measurements.idleReads++;if(tag===12)measurements.peerFailures++;return input(node,tag,...args);}; + for(const [method,counter]of [['commit','proposals'],['barrier','barriers'],['authorize','authorizations']]){const original=host[method].bind(host);host[method]=(...args)=>{measurements[counter]++;return original(...args);};} + }}); const accept=ctx.acceptWebSocket.bind(ctx); ctx.acceptWebSocket=(ws,tags)=>{accept(ws,tags);const send=ws.send.bind(ws);ws.send=raw=>{if(ws.deserializeAttachment()?.clusterPeer)measurements.peerOut++;return send(raw);};}; for(const key of ['put','delete']){const call=ctx.storage[key].bind(ctx.storage);ctx.storage[key]=(...args)=>{measurements.writes++;return call(...args);};} @@ -26,7 +34,78 @@ export class IdentityDO extends Source { const exec=ctx.storage.sql.exec.bind(ctx.storage.sql); ctx.storage.sql.exec=(query,...args)=>{if(!/^\s*(SELECT|PRAGMA)/i.test(query))measurements.writes++;return exec(query,...args);}; } + interruptNextRead(mode){ + const host=this.current.host;let pending=true,views=0; + for(const [id,member] of host.members){ + host.members.set(id,new Proxy(member,{get(target,name){ + const call=Reflect.get(target,name,target); + if(name==='hostView')return async(...args)=>{ + const result=await Reflect.apply(call,target,args); + if(pending){ + pending=false;views++;const leader=host.leader()??host.replica(); + try{ + await host.members.get(leader).scheduled(); + if(mode==='advance')await host.commitEffect({device_seen:{device:host.seen.keys().next().value}}); + if(mode?.command&&views>=mode.after)await host.commit(unb64(mode.command)); + if(mode==='no-quorum'){for(const node of [...host.members.keys()])if(node!==leader){host.members.delete(node);host.states.delete(node);}host.queue=[];} + }finally{pending=mode==='always'||mode==='advance'||!!mode?.command&&viewsReflect.apply(call,target,args):call; + }})); + } + return true; + } + transientPeerFailure(blocked){ + this.blockedPeerCalls=blocked; + if(!blocked)return true; + const host=this.current.host,self=this,leader=host.leader(); + for(const [id,member]of host.members){ + if(id===leader)continue; + host.members.set(id,new Proxy(member,{get(target,name){ + const call=Reflect.get(target,name,target); + if(name==='peer')return async(...args)=>{ + if(self.blockedPeerCalls)throw new Error('fixture: temporary peer RPC failure'); + return Reflect.apply(call,target,args); + }; + return typeof call==='function'?(...args)=>Reflect.apply(call,target,args):call; + }})); + } + return true; + } + async transientCapacity(blocked){ + this.blockedAnchors=blocked; + if(!this.transientEnvironment){ + const cfg=await this.ctx.storage.get('raft-provision'),ns=this.env.RAFT_MEMBERS,self=this; + const missing=new Set(cfg.config.anchors.slice(1).map(node=>ns.idFromName(cfg.config.cluster+'/'+node).toString())); + this.transientEnvironment={...this.env,RAFT_MEMBERS:{idFromName:name=>ns.idFromName(name),get:id=>{ + const stub=ns.get(id);return new Proxy(stub,{get(target,name){ + if(name==='initialize'&&missing.has(id.toString())&&self.blockedAnchors)return async()=>({capacity:true}); + const value=Reflect.get(target,name,target);return typeof value==='function'?async(...args)=>Reflect.apply(value,target,args):value; + }}); + }}}; + this.rebuilt=new Source(this.ctx,this.transientEnvironment); + } + return true; + } get current(){return this.rebuilt??this;} + async directCommit(raw){return this.host.operation(async()=>(await this.host.commit(unb64(raw))).value.code);} + async formed(){ + if(!this.host?.forming)return; + await this.host.operation(async()=>{ + for(let turn=0;turn<16&&this.host.forming;turn++)await this.host.maintenance(Date.now(),Math.floor(performance.now())); + if(this.host.forming){const id=this.host.replica(),view=await this.host.members.get(id).hostView(this.host.states.get(id).applied,'state','identity'),conf=this.host.configuration(view.app.conf);if(conf.voters.length!==3||conf.outgoing?.length)throw new Error('fixture: three-voter formation did not complete');} + await this.host.readAuthority(); + }); + } + async formationTurn(){ + return this.host.operation(async()=>{ + let error;try{await this.host.maintenance(Date.now(),Math.floor(performance.now()));}catch(e){error=e.message;} + const id=this.host.replica(),view=await this.host.members.get(id).hostView(this.host.states.get(id).applied,'membership','registry'); + return {conf:this.host.configuration(view.app.conf),registry:!!view.raw,forming:this.host.forming,error:error??null}; + }); + } async evictIdle(){ const old=this.current.host; if(old){old.ready=false;old.members.clear();old.states.clear();old.queue=[];old.reads=[];old.peers=null;old.holds=null;} @@ -69,11 +148,7 @@ export class IdentityDO extends Source { webSocketClose(ws,code){return this.rebuilt?this.rebuilt.webSocketClose(ws,code):super.webSocketClose(ws,code);} webSocketError(ws){return this.rebuilt?this.rebuilt.webSocketError(ws):super.webSocketError(ws);} async fetch(request){ - const result=await measured(()=>this.rebuilt?this.rebuilt.fetch(request):super.fetch(request)); - if(this.host&&!this.capturing){ - const commit=this.host.commit.bind(this.host);this.host.commit=async raw=>{this.lastCommand=raw.slice();return commit(raw);};this.capturing=true; - } - return result; + return measured(()=>this.rebuilt?this.rebuilt.fetch(request):super.fetch(request)); } async sequenceProof(){return this.host.operation(async()=>{ const from=this.host.manifest.config.anchors[0],to=this.host.manifest.config.anchors[1],term=Number(this.host.states.get(this.host.leader()).term); @@ -93,18 +168,35 @@ export class IdentityDO extends Source { return this.host.members.get(id).transportCheckpoint(); } holdUpload(){ - const put=this.host.tenant.store.put.bind(this.host.tenant.store); + const put=this.host.tenant.store.put.bind(this.host.tenant.store),has=this.host.tenant.has.bind(this.host.tenant); + this.host.tenant.has=async(...args)=>{this.hasWaiting=true;try{return await has(...args);}finally{this.hasWaiting=false;}}; this.pauseUpload=new Promise(resolve=>{this.releaseUpload=resolve;}); this.host.tenant.store.put=async(...args)=>{this.uploadStarted=true;await this.pauseUpload;return put(...args);}; } + waitingHas(){return !!this.hasWaiting;} releaseHeldUpload(){this.releaseUpload?.();} async retryWithoutQuorum(){ + if(!(this.lastCommand instanceof Uint8Array))throw new Error('fixture: no committed command to replay'); + const original=this.lastCommand.slice();await this.formed(); return this.host.operation(async()=>{ const leader=this.host.leader();for(const id of [...this.host.members.keys()])if(id!==leader)this.host.members.delete(id); this.host.queue=[]; - try{await this.host.commit(this.lastCommand);return 'acknowledged';}catch{return 'unavailable';} + try{await this.host.commit(original);return 'acknowledged';}catch{return 'unavailable';} }); } + async expireReadQuorum(){ + await this.formed(); + const host=this.host,leader=host.leader(); + if(!host.readLease)throw new Error('fixture: expected cached authority'); + host.readLease.until=performance.now()-1; + for(const id of [...host.members.keys()])if(id!==leader)host.members.delete(id); + host.queue=[];return true; + } + async replaceReadMember(){ + const lease=this.host.readLease;if(!lease)throw new Error('fixture: expected cached authority'); + return this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(this.host.manifest.config.cluster+'/'+lease.node)).replaceActivation(); + } + invalidateReadAuthority(){this.host?.invalidateRead();} async seed(identity,records) { const dir=new Directory(this.ctx.storage.sql,identity,Date.now,policyOf()); for(const r of records){ @@ -114,6 +206,7 @@ export class IdentityDO extends Source { await this.ctx.storage.put({identity,admitted:true});await this.ctx.storage.sync(); } async replayStale() { + await this.formed(); return this.host.operation(async()=>{ const from=this.host.leader(),to=this.host.manifest.config.anchors.find(id=>id!==from); const member=this.host.member(to),snapshot=await member.oldSnapshotAndCheckpoint(); @@ -187,7 +280,7 @@ export class RaftMemberDO extends Member { await this.load();const old=this.actor,before=old.store.application().applied; const restored=new Member(this.ctx,this.env);await restored.load(); let refused=false;try{await old.input(1);}catch{refused=true;} - for(const key of ['actor','config','registry','results','resultBytes','recoveredEffects'])this[key]=restored[key]; + for(const key of ['actor','config','registry','results','resultBytes','recoveredEffects','readActivation'])this[key]=restored[key]; return {before,after:this.actor.store.application().applied,refused,oldHandle:old.handle,active:runtimeMetrics().active}; } async retentionState(op){ @@ -245,10 +338,17 @@ export default { if(op==='leadership-ticks')return new Response(encodeCanonical(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).leadershipTicks(ticks,missing))); if(op==='stale-retirement')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).staleRetirement()); if(op==='diagnostic'){console.warn('local runtime warning capture');console.error('local runtime error capture');return Response.json(true);} + if(op==='shift-clock'){globalThis.fixtureClockOffset+=records;return Response.json(true);} if(op==='evict-minority')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictMinority()); if(op==='presence-schedule')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceSchedule()); if(op==='retention-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).retentionState(records)); + if(op==='direct-commit')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).directCommit(records)); + if(op==='form-cluster'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).formed();return Response.json(true);} + if(op==='formation-turn')return new Response(encodeCanonical(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).formationTurn())); if(op==='quiet-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).quietState()); + if(op==='interrupt-next-read')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).interruptNextRead(records)); + if(op==='transient-peer-failure')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).transientPeerFailure(records)); + if(op==='transient-capacity')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).transientCapacity(records)); if(op==='evict-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictIdle()); if(op==='presence-turn')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceTurn()??true); if(op==='member-alarm')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).alarmState()); @@ -264,16 +364,23 @@ export default { if(op==='reset-metrics'){for(const key of Object.keys(measurements))measurements[key]=key==='wall'?[]:0;return Response.json(true);} if(op==='seed'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).seed(identity,records);return Response.json(true);} if(op==='hold-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).holdUpload();return Response.json(true);} + if(op==='has-waiting')return Response.json(!!await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).waitingHas()); if(op==='release-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).releaseHeldUpload();return Response.json(true);} if(op==='retry-without-quorum')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).retryWithoutQuorum()); + if(op==='expire-read-quorum')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).expireReadQuorum()); + if(op==='replace-read-member')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).replaceReadMember()); if(op==='stale-snapshot')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).replayStale()); if(op==='idle-schedule')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).idleSchedule()); if(op==='source')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).inspect()); - if(op==='seed-transport-tail'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).seedTransportTail();return Response.json(true);} + if(op==='seed-transport-tail'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).formed();await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).seedTransportTail();return Response.json(true);} if(op==='retained-control')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).retainedControl()); - if(op==='discard-member'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).discardStore();return Response.json(true);} + if(op==='discard-unformed-member'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).discardStore();return Response.json(true);} + if(op==='discard-member'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).formed();await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).discardStore();return Response.json(true);} if(op==='has-member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).hasStore()); - if(op==='replace-activation')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).replaceActivation()); + if(op==='replace-activation'){ + await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).invalidateReadAuthority(); + return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).replaceActivation()); + } if(op==='staging')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).staging()); if(op==='member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).inspect()); return new Response(null,{status:404}); diff --git a/relay/hosted/test/raft/v1-large.test.js b/relay/hosted/test/raft/v1-large.test.js index 93c8d9f49f..03ef226737 100644 --- a/relay/hosted/test/raft/v1-large.test.js +++ b/relay/hosted/test/raft/v1-large.test.js @@ -22,6 +22,7 @@ test('R25 archived requests above 64 KiB and just under 1 MiB commit in workerd' try{ await h.start(); for(const c of fixture.cases)await exact(h,c); + await h.control({op:'form-cluster'}); const before=await h.control({op:'metrics'});console.log(JSON.stringify({peakCoreBytes:before.peakMemory,activeHandles:before.active,reservedBytes:before.reservedBytes})); assert.equal(before.active,3,'cold object recreation leaked live handles'); const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; @@ -45,6 +46,7 @@ test('R25 interrupted workerd staging stays atomic and bounded after restart',{t const failed=await send(h,c);assert(failed.status>=500,`interruption returned ${failed.status}`);await failed.text(); await h.start(); const absent=await send(h,c,true);assert.equal(absent.status,404);await absent.text(); + await h.control({op:'form-cluster'}); const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; for(const node of ['1','2','3']){const state=await h.control({op:'staging',cluster,node});assert(state.rows.n>0);assert(state.rows.n<=33);assert(state.rows.bytes<=1048576+65536);assert.equal(state.app.version,'0');} if(attempt===0)await h.start(cut); @@ -95,6 +97,7 @@ test('R25 maximum legacy title escaping remains lossless in workerd',{timeout:60 } await h.start();const headers={...c.headers};headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\nGET\n${c.uri}\n${headers['Codeaf-Time']}\n${sha('')}`),key).toString('base64url'); const read=await h.mf.dispatchFetch('http://local'+c.uri,{headers});assert.equal(read.status,200,(await read.clone().text()).slice(0,200));assert.equal((await read.json()).cell.title,title); + await h.control({op:'form-cluster'}); const metrics=await h.control({op:'metrics'});assert.equal(metrics.active,3);assert(metrics.peakMemory<=metrics.coreBudget,JSON.stringify(metrics));console.log(JSON.stringify({escapedTitleCoreBytes:metrics.peakMemory,activeHandles:metrics.active})); }finally{await h.close();} }); @@ -113,6 +116,7 @@ test('R25 small legacy resumes preserve large retained frame plans in workerd',{ const response=await send(h,request);assert.equal(response.status,200,(await response.clone().text()).slice(0,200)); const view=await response.json();assert.equal(view.cell.lease.fence,fence);assert.deepEqual(view.cell.frames,JSON.parse(c.body).frames); } + await h.control({op:'form-cluster'}); const metrics=await h.control({op:'metrics'});console.log(JSON.stringify({resumedFrameCoreBytes:metrics.peakMemory,activeHandles:metrics.active}));assert.equal(metrics.active,3);assert(metrics.peakMemory<=metrics.coreBudget); }finally{await h.close();} }); diff --git a/relay/hosted/test/watch.e2e.mjs b/relay/hosted/test/watch.e2e.mjs index 699ac39eed..4ff7249869 100644 --- a/relay/hosted/test/watch.e2e.mjs +++ b/relay/hosted/test/watch.e2e.mjs @@ -6,6 +6,7 @@ // maxWatchers 5, named by RELAY_WATCH_CAP; the restart case talks to RELAY_PERSIST. import assert from 'node:assert/strict'; import http from 'node:http'; +import WebSocket from 'ws'; import { readFileSync, writeFileSync } from 'node:fs'; import { newIdentity, newDevice, signed, deviceId } from './party.js'; import { init } from './helpers.js'; @@ -20,7 +21,7 @@ const record = { V: 1, name: '', added_by: '', revoked: false, caps: { os: 'linu /** openWatch dials the watch route signed by dev and answers a socket that remembers every text frame. */ async function openWatch(dev, base = BASE, query = '') { const headers = await signed(dev, 'GET', `/v1/dir/watch${query}`); - const ws = new WebSocket(base.replace(/^http/, 'ws') + `/v1/dir/watch${query}`, { headers }); + const ws = new WebSocket(base.replace(/^http/, 'ws') + `/v1/dir/watch${query}`, { headers, closeTimeout: 200 }); const w = { frames: [], cursor: 0, ws, send: (text) => ws.send(text), close: () => ws.close(), wake: () => {} }; w.closed = new Promise((done) => ws.addEventListener('close', (e) => done({ code: e.code, reason: e.reason }))); ws.addEventListener('message', (e) => (w.frames.push(e.data), w.wake())); @@ -137,7 +138,7 @@ const cases = { assert.match(await next(wc), WIRE); await next(wc); assert.deepEqual(await event(wa), { t: 'presence', device: idc, online: true }); - assert.equal((await call(a, 'POST', `/v1/dir/devices/${idc}/revoke`, {})).status, 204); + assert.equal((await call(a, 'POST', `/v1/dir/devices/${idc}/revoke`)).status, 204); assert.match(await next(wa), WIRE); assert.deepEqual(await event(wa), { t: 'revoked', device: idc }); assert.deepEqual((await wc.closed).code, 4401); @@ -207,7 +208,7 @@ const cases = { ['acquire', () => call(a, 'POST', '/v1/dir/cells/c1/acquire', {})], ['publish', () => call(a, 'POST', '/v1/dir/cells/c1/publish', pub(2, 'h1', 'h0'))], ['release', () => call(a, 'POST', '/v1/dir/cells/c1/release', { fence: 2 })], - ['archive', () => call(a, 'POST', '/v1/dir/cells/c1/archive', {})], + ['archive', () => call(a, 'POST', '/v1/dir/cells/c1/archive')], ['device put', async () => call(a, 'PUT', `/v1/dir/devices/${await deviceId(a)}`, { ...record, name: 'renamed' })], ['vault', () => call(a, 'POST', '/v1/dir/vault', { old: '', new: 'x' })], ]; @@ -299,9 +300,9 @@ const cases = { async holdQuery() { const [a] = await party(); const plain = await dialled(a, ''); - assert.deepEqual([plain.status, plain.headers['codeaf-vouch']], [101, '1'], 'a relay that vouches says so on every upgrade'); + assert.deepEqual([plain.status, plain.headers['codeaf-presence'], plain.headers['codeaf-vouch']], [101, '1', undefined], 'presence is advertised while lease authority stays with committed holds'); const held = await dialled(a, '?hold=c1:1&hold=nowhere:9&hold=c1:1'); - assert.deepEqual([held.status, held.headers['codeaf-vouch']], [101, '1'], 'holds of cells that do not exist are accepted'); + assert.deepEqual([held.status, held.headers['codeaf-presence'], held.headers['codeaf-vouch']], [101, '1', undefined], 'holds of cells that do not exist are accepted without claiming socket lease authority'); const sixteen = Array.from({ length: 16 }, (_, i) => `hold=c${i}:1`).join('&'); assert.equal((await dialled(a, `?${sixteen}`)).status, 101); const refusals = ['?hold=c1', '?hold=:1', '?hold=c1:', '?hold=c1:x', '?hold=c1:-1', `?hold=${'c'.repeat(65)}:1`, `?${sixteen}&hold=c16:1`]; @@ -406,6 +407,10 @@ async function lease() { } const [mode, file] = process.argv.slice(2); +if (mode === '--list') { + console.log(Object.keys(cases).join('\n')); + process.exit(0); +} if (mode === 'idle') await idle(); else if (mode === 'lease') await lease(); else if (mode === 'put') await putState(file);