diff --git a/PERF.md b/PERF.md index 050d8b1844..e781d3d61a 100644 --- a/PERF.md +++ b/PERF.md @@ -17,6 +17,13 @@ is diagnostic; regression gates count deterministic work, bytes, allocations, request fanout, retained data and cache reuse. Do not gate on wall-clock thresholds. Use scripted providers and explicit synchronization rather than real sleeps. +Keyed vault restores read the sealed vault at most twice when no selection +policy is needed, and at most three times when incoming model choices need +provider and revision checks. These caps are independent of the number of keys; +`TestARestoreOpensTheVaultAFixedNumberOfTimes` and +`TestSelectiveRestoreM1BoundedVaultReads` exercise 16 and 256 keys. A model's +optional provider metadata is backfilled without advancing its choice clock. + Activity layout lines share immutable task records instead of copying both task and conversation payloads into every line. Conversation overlays keep their own copy, so live titles do not mutate the held reading. Paint scratch still expires diff --git a/cmd/codeaf/app.go b/cmd/codeaf/app.go index 61a9524c4d..f4f230b3e5 100644 --- a/cmd/codeaf/app.go +++ b/cmd/codeaf/app.go @@ -24,8 +24,8 @@ package main // AND IT IS LAZY BY LAW. Startup opens agents with work or recovery to settle; // idle conversations stay as indexed records, crew members included. Viewing, // sending or waking opens one through the same per-agent gate. Listing agents, -// tasks and crews reads the disk, and a conversation left alone for -// [appAgentIdle] is let go and rebuilt transparently on demand. +// tasks and crews reads the disk. A conversation left alone for [appAgentIdle] +// without an open view is let go and rebuilt transparently on demand. import ( "cmp" @@ -67,9 +67,9 @@ import ( ) // appAgentIdle is how long a conversation the app holds may sit with no turn -// running, no question waiting and nothing sent before it is let go. Letting go -// closes the conversation in this process and nothing else: its journal is on -// disk, its state still reads idle, and the next send opens it again. +// running, no question waiting and no open view before it is let go. Its +// journal stays on disk, its state still reads idle, and the next send opens +// it again. An explicitly open view keeps the conversation and its hold. const appAgentIdle = 10 * time.Minute // appSweepEvery is how often the idle rule above is asked. @@ -455,12 +455,15 @@ func appConnOf(client *remote.Client, release func(), assembled bool) *appConn { // appBackend is the [appwire.Engine] over this process's conversations. type appBackend struct { + chats appChats + pair appPair drives *driveBook sync appSync syncPending map[string][]appSyncFact // devices observes existing sync traffic without starting a roster watch. - devices *appDevices + devices *appDevices + stopSyncedSettings func() // organizeProposer supplies a naming door when the backend is embedded without held conversations. organizeProposer session.TeamProposer @@ -618,6 +621,9 @@ type appAgent struct { syncGate sync.Mutex localNotices map[string]bool windowMoved bool + // viewOpen preserves the person's open view through idle sweeps. Retiring + // it would let its next read immediately take back the released hold. + viewOpen bool // crewHeld keeps an established crew hold due after its engine closes. crewHeld bool @@ -873,10 +879,12 @@ func newAppBackend(emit appwire.Emitter, open appOpener) *appBackend { mint: appMint, } b.emit = b.newAppInbox(emit) + b.stopSyncedSettings = b.listenSyncedSettings() b.devices = b.listenDevices() b.stopPhases = b.listenPhases() b.stopProgress = b.listenProgress() guard.Go("app/sweep", b.sweep) + b.startLocalChatPublication() return b } @@ -959,7 +967,21 @@ func (b *appBackend) remember(row session.SessionRow) (view appwire.Agent) { // Read freshness against this backend's clock, including in deterministic // tests, rather than trusting a row another reader dated earlier. row.Presence, row.Live = session.ReadSessionPresence(row.Dir, b.now()) - b.mu.Lock() + // Publish cannot miss an agent first remembered from a saved row. + // Holding both locks makes its initial location agree with the inventory; + // a later publication will see the new agent and send its location change. + lockRememberedLocation := func() *appwire.Location { + b.chats.mu.Lock() + defer b.chats.mu.Unlock() + // The backend lock stays held after the inventory lock is released. + // The remainder of remember owns its existing deferred release. + b.mu.Lock() + if cached, ok := b.chats.byAgent[row.ID]; ok { + return chatLocation(cached) + } + return nil + } + location := lockRememberedLocation() a := b.agentLocked(row.ID) defer func() { b.mu.Unlock() @@ -968,6 +990,9 @@ func (b *appBackend) remember(row session.SessionRow) (view appwire.Agent) { }() if a.transcript == "" { a.transcript = row.Transcript + if a.view.Location == nil { + a.view.Location = location + } } disk := appRowView(row) disk.Location = a.view.Location @@ -1054,6 +1079,7 @@ func appRowOnDisk(id string) (session.SessionRow, bool) { // ListAgents reads the disk and never opens a conversation. func (b *appBackend) ListAgents(workspace string) ([]appwire.Agent, error) { + b.startChatInventory() want := "" if strings.TrimSpace(workspace) != "" { want = filepath.Clean(workspace) @@ -1080,7 +1106,7 @@ func (b *appBackend) ListAgents(workspace string) ([]appwire.Agent, error) { seen := map[string]bool{} var agents []appwire.Agent for _, row := range session.ReadWorld(session.PlacesRoot()).Sessions() { - if row.DeletionPending || factoryOwnedConversation(row.Transcript) || !matches(row.Workspace, row.ProjectDir) { + if appChatStaging(row.Dir) || row.DeletionPending || factoryOwnedConversation(row.Transcript) || !matches(row.Workspace, row.ProjectDir) { continue } seen[row.ID] = true @@ -1163,6 +1189,9 @@ type appCreatePrep struct { // prepareCreate is everything agents.create can refuse, checked with nothing // made: the workspace, shutdown, and every attachment read whole. func (b *appBackend) prepareCreate(ask appwire.CreateAsk) (appCreatePrep, error) { + if err := b.pairingWorkRefusal(); err != nil { + return appCreatePrep{}, err + } if err := (session.Submission{ClientID: ask.ClientID}).Check(); err != nil { return appCreatePrep{}, appEngineError(err) } @@ -1489,17 +1518,27 @@ func (b *appBackend) OpenAgent(id string) (appwire.Agent, error) { // workspace is many seconds, and its own agent.changed says the rest. var view appwire.Agent busy := false - b.locked(func() { view, busy = a.view, a.live == nil && (a.opening || a.starting) }) + b.locked(func() { + view, busy = a.view, a.live == nil && (a.opening || a.starting) + if busy { + a.viewOpen = true + } + }) if busy { return view, nil } - view, opened, err := b.ensure(a) + a.gate.Lock() + defer a.gate.Unlock() + b.locked(func() { a.viewOpen = true }) + view, opened, err := b.ensureLocked(a) if err != nil { return appwire.Agent{}, err } if opened { b.emit.Emit(appwire.EventAgentChanged, appwire.AgentChanged{Agent: view}) } + b.startChatInventory() + b.dress(&view) return view, nil } @@ -2077,7 +2116,7 @@ func (b *appBackend) send(id, text, clientID string, shown *appSending) (uint64, // sendLocked opens a's conversation if it is not held and starts one turn on // it. The caller holds a.gate. func (b *appBackend) sendLocked(a *appAgent, text, clientID string, shown *appSending) (uint64, error) { - if err := b.knownWorkRefusal(a); err != nil { + if err := b.refreshWorkRefusal(a); err != nil { return 0, err } // Opening can wake saved crew work, so reject an invalid identity before @@ -2383,6 +2422,7 @@ func (b *appBackend) startTurnLocked(a *appAgent, text, clientID string, shown * func (b *appBackend) pump(a *appAgent, id string, live *appLive, turn uint64, n int, text string, shown *appSending, events <-chan session.Event) { defer close(live.pumpDone) ok, failure := true, "" + retryable := true kind, status := "", 0 entry := func(e appwire.Entry) { e.Agent, e.Turn = id, turn @@ -2623,6 +2663,7 @@ func (b *appBackend) pump(a *appAgent, id string, live *appLive, turn uint64, n case session.EventError: ok = false kind, status, failure = appTurnFailure(event) + retryable = !provider.ReasoningUnsupported(status, failure) case session.EventTurnDone: // A turn the person steered ends once per stretch; the last ending // on the stream is the turn's. @@ -2654,7 +2695,7 @@ func (b *appBackend) pump(a *appAgent, id string, live *appLive, turn uint64, n } a.failed, a.failedKind, a.failedShown, a.failedClient = "", "", nil, "" a.view.Retryable = false - if !ok { + if !ok && retryable { a.failed, a.failedKind, a.failedShown = text, kind, shown a.view.Retryable = text != "" } @@ -2965,7 +3006,7 @@ func (b *appBackend) closeAgent(a *appAgent) (view appwire.Agent) { // hold a.gate. func (b *appBackend) release(a *appAgent) { var live *appLive - b.locked(func() { live, a.live = a.live, nil }) + b.locked(func() { live, a.live, a.viewOpen = a.live, nil, false }) if live == nil { return } @@ -4092,6 +4133,11 @@ func (b *appBackend) Close() error { if already { return nil } + b.closePair() + if b.stopSyncedSettings != nil { + b.stopSyncedSettings() + } + b.closeChats() if b.devices != nil { b.devices.close() } @@ -4191,7 +4237,7 @@ func (b *appBackend) sweep() { } // sweepOnce is one look at the idle rule. A conversation is kept while a turn -// runs or a question waits for the person, whatever the clock says. +// runs, a question waits for the person or a view stays open, whatever the clock says. func (b *appBackend) sweepOnce() { var syncing bool b.locked(func() { syncing = b.drives != nil }) @@ -4202,7 +4248,7 @@ func (b *appBackend) sweepOnce() { var idle []*appAgent b.locked(func() { for _, a := range b.agents { - if live := a.live; live != nil && !live.running && now.Sub(live.lastUse) >= appAgentIdle { + if live := a.live; live != nil && !a.viewOpen && !live.running && now.Sub(live.lastUse) >= appAgentIdle { idle = append(idle, a) } } @@ -4223,7 +4269,7 @@ func (b *appBackend) sweepAgent(a *appAgent, now time.Time) { keep := true b.locked(func() { live = a.live - keep = live == nil || live.running || now.Sub(live.lastUse) < appAgentIdle + keep = live == nil || a.viewOpen || live.running || now.Sub(live.lastUse) < appAgentIdle }) // A member of an open crew stays held while the crew is open. if !keep && !b.liveNeedsYou(live) && !b.crewKeeps(a.view.ID) && !b.serviceKeeps(a.view.ID) { diff --git a/cmd/codeaf/app_chat_copy.go b/cmd/codeaf/app_chat_copy.go new file mode 100644 index 0000000000..46d387a634 --- /dev/null +++ b/cmd/codeaf/app_chat_copy.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/Agent-Field/codeaf/internal/chatlist" +) + +const appTakeFailed = "This chat could not be moved to this computer." + +// A live lease means this chat is open there; it says nothing about a turn. +func appChatStatusLine(r chatlist.Row, online *bool) string { + if online != nil && !*online && r.Device != "" { + return fmt.Sprintf("%s offline", r.Device) + } + if r.Status == chatlist.Running { + return fmt.Sprintf("open on %s", r.Device) + } + if r.Status == chatlist.Off { + return "" + } + return chatlist.StatusLine(r) +} + +// The desktop formats SavedAt locally. Only holder and pending-work facts add +// anything to the takeover sentence, and neither changes as the clock advances. +func appChatTakeoverLine(r chatlist.Row, online *bool) string { + parts := []string{} + if status := appChatStatusLine(r, online); status != "" { + parts = append(parts, status) + } + if r.Pending > 0 { + parts = append(parts, fmt.Sprintf("Some work may still be on %s.", r.Device)) + } + return strings.Join(parts, "; ") +} diff --git a/cmd/codeaf/app_chat_events_test.go b/cmd/codeaf/app_chat_events_test.go new file mode 100644 index 0000000000..5b80ce8bb6 --- /dev/null +++ b/cmd/codeaf/app_chat_events_test.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// R4 holds the watch between its version and presence frames. The relay list +// remains real; its mutations can be read before the watch announces them. +func TestAppChatEventsR4F4UnknownPresenceAndSilentLists(t *testing.T) { + celltest.Available(t) + frames := make(chan string, 8) + startWatch := make(chan struct{}) + reconnect := make(chan struct{}, 1) + connected := make(chan struct{}, 8) + var initial atomic.Uint64 + var peerID string + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/dir/presence" { + _ = json.NewEncoder(w).Encode(directory.PresenceView{Now: 3, Devices: map[string]directory.DevicePresence{peerID: {Online: true}}}) + return + } + if r.URL.Path != "/v1/dir/watch" { + next.ServeHTTP(w, r) + return + } + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer conn.CloseNow() + connected <- struct{}{} + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + go func() { + defer cancel() + for { + _, text, err := conn.Read(ctx) + if err != nil { + return + } + if string(text) == "ping" { + _ = conn.Write(ctx, websocket.MessageText, []byte("pong")) + } + } + }() + select { + case <-startWatch: + case <-ctx.Done(): + return + } + if err := conn.Write(ctx, websocket.MessageText, []byte(fmt.Sprintf(`{"v":%d}`, initial.Load()))); err != nil { + return + } + for { + select { + case text := <-frames: + if conn.Write(ctx, websocket.MessageText, []byte(text)) != nil { + return + } + case <-ctx.Done(): + return + case <-reconnect: + return + } + } + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + aHome, bHome := t.TempDir(), t.TempDir() + id, err := identity.Ensure(aHome) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(bHome, id, false); err != nil { + t.Fatal(err) + } + s := appDeviceSync(t, aHome) + peerID = s.Device.ID() + if err := syncsetup.PutOwnDevice(t.Context(), s.Dir, s.Identity, s.Device, "Mac"); err != nil { + t.Fatal(err) + } + // Both paired computers already have records, so startup publication does + // not add a directory mutation to this controlled chat-version fixture. + other := appDeviceSync(t, bHome) + if err := syncsetup.PutOwnDevice(t.Context(), other.Dir, other.Identity, other.Device, devname.Name(bHome)); err != nil { + t.Fatal(err) + } + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Dir.Create(t.Context(), c.ID, directory.CellInit{Head: "first-head"}); err != nil { + t.Fatal(err) + } + l, err := s.Dir.List(t.Context()) + if err != nil { + t.Fatal(err) + } + initial.Store(l.Version) + backend, rec, wire := appDeviceBackend(t, bHome) + // Represent an already admitted local copy without opening a turn engine. + a := backend.agent(c.ID) + backend.locked(func() { a.transcript = filepath.Join(c.Root, cell.TranscriptPath) }) + machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + rec.mu.Lock() + for _, name := range rec.events { + if name == appwire.EventAgentChanged || name == appwire.EventChatChanged || name == appwire.EventChatRemoved { + t.Errorf("R4 list emitted before watch spoke: %s", name) + } + } + rec.mu.Unlock() + close(startWatch) + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { return name == appwire.EventChatChanged }) + row := machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats[0] + if row.Online != nil || strings.Contains(row.Line, "offline") { + t.Fatalf("R4 unset presence must be omitted: %+v", row) + } + frames <- fmt.Sprintf(`{"t":"presence","device":%q,"online":true,"at":1}`, s.Device.ID()) + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { + if name != appwire.EventChatChanged { + return false + } + row := data.(appwire.ChatChanged).Chat + return row.Online != nil && *row.Online + }) + cursor := func() int { + rec.mu.Lock() + defer rec.mu.Unlock() + n := 0 + for i, name := range rec.events { + if name == appwire.EventChatChanged && rec.data[i].(appwire.ChatChanged).Chat.Cell == c.ID { + n++ + } + } + return n + } + before := cursor() + for range 3 { + machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + } + title, err := directory.SealName(directory.MetadataKey(id.CellKey()), "Changed R4") + if err != nil { + t.Fatal(err) + } + if _, err := s.Dir.Publish(t.Context(), c.ID, directory.Publish{Fence: 1, OldHead: "first-head", Head: "second-head", Title: title}); err != nil { + t.Fatal(err) + } + listed := machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + if listed.Chats[0].Title != "Changed R4" { + t.Fatalf("R4 list did not read fresh content: %+v", listed) + } + if after := cursor(); after != before { + t.Fatalf("R4 chats.list emitted: before=%d after=%d", before, after) + } + frames <- fmt.Sprintf(`{"v":%d}`, listed.Version) + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { + return name == appwire.EventChatChanged && data.(appwire.ChatChanged).Chat.Title == "Changed R4" + }) + // A real presence change may update the same inventory version. + frames <- fmt.Sprintf(`{"t":"presence","device":%q,"online":false,"at":2}`, s.Device.ID()) + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { + return name == appwire.EventChatChanged && data.(appwire.ChatChanged).Chat.Line == "Mac offline" + }) + frames <- fmt.Sprintf(`{"t":"presence","device":%q,"online":true,"at":3}`, s.Device.ID()) + appSyncWait(t, "R4 presence returns online", func() bool { return cursor() >= 5 }) + machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + // F4: a watch reconnect repeats the current version, without changing a row. + <-connected + initial.Store(listed.Version) + beforeReconnect := cursor() + reconnect <- struct{}{} + select { + case <-connected: + case <-time.After(10 * time.Second): + t.Fatal("F4 watch did not reconnect") + } + marker, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if _, err = s.Dir.Create(t.Context(), marker.ID, directory.CellInit{Head: "marker-head"}); err != nil { + t.Fatal(err) + } + latest, err := s.Dir.List(t.Context()) + if err != nil { + t.Fatal(err) + } + frames <- fmt.Sprintf(`{"v":%d}`, latest.Version) + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { + return name == appwire.EventChatChanged && data.(appwire.ChatChanged).Chat.Cell == marker.ID + }) + if got := cursor(); got != beforeReconnect { + t.Fatalf("F4 reconnect emitted %d extra chat changes", got-beforeReconnect) + } + rec.mu.Lock() + defer rec.mu.Unlock() + var changed []appwire.ChatChanged + for i, name := range rec.events { + if name == appwire.EventChatChanged && rec.data[i].(appwire.ChatChanged).Chat.Cell == c.ID { + changed = append(changed, rec.data[i].(appwire.ChatChanged)) + } + } + if len(changed) != 5 { + t.Fatalf("R4 changes=%+v; want unknown, online, title, offline, online", changed) + } + for i, event := range changed { + if i == 0 && event.Chat.Online != nil { + t.Errorf("R4 initial event invented presence: %+v", event) + } + if i < 2 && event.Version != l.Version || i >= 2 && event.Version != listed.Version { + t.Errorf("R4 version: %+v", event) + } + } +} diff --git a/cmd/codeaf/app_chat_rows_test.go b/cmd/codeaf/app_chat_rows_test.go new file mode 100644 index 0000000000..d3d0020789 --- /dev/null +++ b/cmd/codeaf/app_chat_rows_test.go @@ -0,0 +1,270 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +func TestAppChatRowsR1P2PresenceIsIndependentOfClosedChat(t *testing.T) { + celltest.Available(t) + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/dir/list" { + next.ServeHTTP(w, r) + return + } + reply := httptest.NewRecorder() + next.ServeHTTP(reply, r) + var listing directory.Listing + if err := json.Unmarshal(reply.Body.Bytes(), &listing); err != nil { + t.Error(err) + return + } + for id, c := range listing.Cells { + c.Lease.Expires = listing.Now - 1 + listing.Cells[id] = c + } + for key, values := range reply.Header() { + w.Header()[key] = values + } + w.WriteHeader(reply.Code) + _ = json.NewEncoder(w).Encode(listing) + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + homes := []string{t.TempDir(), t.TempDir(), t.TempDir()} + id, err := identity.Ensure(homes[0]) + if err != nil { + t.Fatal(err) + } + for _, home := range homes[1:] { + if _, err := identity.Adopt(home, id, false); err != nil { + t.Fatal(err) + } + } + var ids []string + var holder, sleeping *syncsetup.Sync + for i, name := range []string{"Mac", "Windows", "Sleeping"} { + s := appDeviceSync(t, homes[i]) + if i == 0 { + holder = s + } + if i == 2 { + sleeping = s + } + if err := syncsetup.PutOwnDevice(t.Context(), s.Dir, s.Identity, s.Device, name); err != nil { + t.Fatal(err) + } + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Dir.Create(t.Context(), c.ID, directory.CellInit{Head: "saved-head"}); err != nil { + t.Fatal(err) + } + ids = append(ids, c.ID) + } + feed := holder.Follow() + defer feed.Close() + _, rec, wire := appDeviceBackend(t, homes[1]) + var rows map[string]appwire.ChatRow + appSyncWait(t, "R1 holder presence", func() bool { + rows = map[string]appwire.ChatRow{} + for _, row := range machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats { + rows[row.Cell] = row + } + row := rows[ids[0]] + return row.Online != nil && *row.Online + }) + // P2: an explicit snapshot also leaves an unobserved computer unknown. + machineCall[appwire.DevicesResult](t, wire, "devices.list", struct{}{}) + for _, row := range machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats { + rows[row.Cell] = row + } + for _, id := range ids[:2] { + row := rows[id] + if row.Status != "off" || row.Online == nil || !*row.Online || row.Line != "" || row.TakeoverLine != "" { + t.Errorf("R1 online holder with closed chat: %+v", row) + } + } + row := rows[ids[2]] + if row.Online != nil || row.Line != "" || row.TakeoverLine != "" { + t.Fatalf("P2 unobserved holder: %+v", row) + } + // Establish an actual connection and disconnect so the offline assertion + // still checks real presence independently of the closed chat's lease. + watch, err := sleeping.Dir.(directory.Watcher).Watch(t.Context()) + if err != nil { + t.Fatal(err) + } + watch.Close() + rec.waitFor(t, directory.OfflineDebounce+5*time.Second, func(event string, data any) bool { + if event != appwire.EventDevicesChanged { + return false + } + for _, dev := range data.(appwire.DevicesResult).Devices.Devices { + if dev.ID == sleeping.Device.ID() && dev.Online != nil && !*dev.Online { + return true + } + } + return false + }) + for _, fresh := range machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats { + rows[fresh.Cell] = fresh + } + row = rows[ids[2]] + if row.Online == nil || *row.Online || row.Line != "Sleeping offline" || row.TakeoverLine != "Sleeping offline" { + t.Errorf("R1 offline holder: %+v", row) + } +} + +func TestAppChatRowsR2SavedTurnTime(t *testing.T) { + celltest.RequireEngine(t) + relay := e2WireRelay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + home, peer := t.TempDir(), t.TempDir() + t.Setenv("CODEAF_HOME", home) + id, err := identity.Ensure(home) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(peer, id, false); err != nil { + t.Fatal(err) + } + c := appFinishedCell(t, home, "Saved R2") + meta, err := session.LoadMeta(c.Root) + if err != nil { + t.Fatal(err) + } + saved := time.Date(2026, 10, 28, 3, 54, 0, 123000000, time.UTC) + e := cellstore.Engine{DataRoot: filepath.Join(home, "v3", "stores"), Workspace: meta.Workspace, Now: func() time.Time { return saved }} + if _, err := e.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + b, _, _ := appDeviceBackend(t, home) + if _, err := b.ListChats(); err != nil { + t.Fatal(err) + } + _, _, wire := appDeviceBackend(t, peer) + check := func(want time.Time) { + t.Helper() + rows := machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats + if len(rows) != 1 || !rows[0].SavedAt.Equal(want) { + t.Errorf("R2 saved_at: %+v, want %s", rows, want) + } + for _, row := range rows { + if strings.Contains(row.TakeoverLine, "Last saved") || strings.Contains(row.TakeoverLine, "UTC") || strings.Contains(row.TakeoverLine, "ago") { + t.Errorf("R2 timestamp in takeover line: %+v", row) + } + } + } + check(saved) + s := appDeviceSync(t, home) + v, err := s.Dir.Acquire(t.Context(), c.ID, directory.AcquireOpts{}) + if err != nil { + t.Fatal(err) + } + saved = time.Date(2026, 10, 28, 4, 0, 0, 456000000, time.UTC) + if err := os.WriteFile(filepath.Join(meta.Workspace, "new-turn.txt"), []byte("new saved work"), 0600); err != nil { + t.Fatal(err) + } + head, err := e.Seal(t.Context(), c, cellstore.TurnInfo{}) + if err != nil { + t.Fatal(err) + } + // A later seal must not lend its timestamp to the head being uploaded. + newer := saved.Add(time.Minute) + e.Now = func() time.Time { return newer } + if _, err := e.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + publisher := cellsync.Publisher{Engine: e.Sync(cellstore.SyncKeys{CellKey: id.CellKey(), Dedup: id.DedupSecret()}, s.Ledger), Dir: s.Dir, Store: s.Store} + driving := cellsync.Driving{Cell: c, Fence: v.Cell.Lease.Fence, Head: v.Cell.Head} + if err := publisher.Publish(t.Context(), &driving, head.Turn.ID, cellsync.PublishInfo{Class: string(cell.FilesOnly)}); err != nil { + t.Fatal(err) + } + check(saved) + if err := s.Dir.Release(t.Context(), c.ID, driving.Fence); err != nil { + t.Fatal(err) + } + check(saved) +} + +func TestAppChatStartupR3QuietRelease(t *testing.T) { + celltest.RequireEngine(t) + svc := relayserve.New(relayserve.Config{Store: t.TempDir()}) + var releases atomic.Int64 + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/release") { + svc.Handler.ServeHTTP(w, r) + return + } + reply := httptest.NewRecorder() + svc.Handler.ServeHTTP(reply, r) + releases.Add(1) + if reply.Code >= 400 { + t.Errorf("R3 refused release: %d %s", reply.Code, reply.Body.String()) + } + for key, values := range reply.Header() { + w.Header()[key] = values + } + w.WriteHeader(reply.Code) + _, _ = w.Write(reply.Body.Bytes()) + })) + t.Cleanup(func() { _ = svc.Close(); relay.Close() }) + t.Setenv(syncsetup.URLVar, relay.URL) + home := t.TempDir() + t.Setenv("CODEAF_HOME", home) + if _, err := identity.Ensure(home); err != nil { + t.Fatal(err) + } + s := appDeviceSync(t, home) + if err := syncsetup.PutOwnDevice(t.Context(), s.Dir, s.Identity, s.Device, "Mac"); err != nil { + t.Fatal(err) + } + c := appFinishedCell(t, home, "Startup R3") + b, _, _ := appDeviceBackend(t, home) + // List joins the finite startup pass without opening a chat. + if _, err := b.ListChats(); err != nil { + t.Fatal(err) + } + v, err := s.Dir.Cell(t.Context(), c.ID) + if err != nil { + t.Fatal(err) + } + head, err := cellstore.Head(c) + if err != nil { + t.Fatal(err) + } + if releases.Load() != 1 || v.Cell.Head != head.Turn.ID || v.Cell.Lease.Device != s.Device.ID() || v.Cell.Lease.Fence != 1 || v.Cell.Lease.Expires != 0 || v.Cell.Lease.ReleasedAt != 0 { + t.Fatalf("R3 startup state: releases=%d cell=%+v", releases.Load(), v.Cell) + } + if err := b.Close(); err != nil { + t.Fatal(err) + } + restarted, _, _ := appDeviceBackend(t, home) + if _, err := restarted.ListChats(); err != nil { + t.Fatal(err) + } + if releases.Load() != 1 { + t.Fatal("R3 restart released an existing chat") + } +} diff --git a/cmd/codeaf/app_chats.go b/cmd/codeaf/app_chats.go new file mode 100644 index 0000000000..6c15fa2784 --- /dev/null +++ b/cmd/codeaf/app_chats.go @@ -0,0 +1,493 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "reflect" + "slices" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +type appChats struct { + start sync.Once + byAgent map[string]appwire.ChatRow + take sync.Mutex + op sync.Mutex + // Offered is guarded by op and records an attempted startup pass. + offered bool + mu sync.Mutex + rows []appwire.ChatRow + facts map[string]chatlist.Row + sent []appwire.ChatRow + version uint64 + source *syncsetup.Sync + feed dirwatch.Follower + cancel context.CancelFunc + done chan struct{} + closed bool +} +type appLocalChat struct { + agent string + copyable bool +} + +// Startup offers local cells without starting a chat-inventory subscriber. +// Device publication owns the paired engine's shared directory follower. +func (b *appBackend) startLocalChatPublication() { + guard.Go("app/existing-chats", func() { + c := &b.chats + c.op.Lock() + defer c.op.Unlock() + if c.isClosed() || c.offered { + return + } + if s, ok, err := syncsetup.Open(b.relayHome()); err == nil && ok && syncsetup.MayTalk(b.relayHome()) { + b.publishLocalChats(s) + c.offered = true + } + }) +} + +// localChats relates physical chat folders to directory ids. Older chats may +// have a separate moving copy, and a drive may publish under a mapped id. +func (b *appBackend) localChats(s *syncsetup.Sync) map[string]appLocalChat { + type entry struct{ id, file string } + var local []entry + for _, r := range session.ReadWorld(filepath.Join(b.relayHome(), "v3", "projects")).Sessions() { + if appChatStaging(r.Dir) { + continue + } + local = append(local, entry{r.ID, r.Transcript}) + } + b.locked(func() { + for id, a := range b.agents { + if a.transcript != "" { + local = append(local, entry{id, a.transcript}) + } + } + }) + out := map[string]appLocalChat{} + for _, item := range local { + agent, file := item.id, item.file + root := session.DirOf(file) + copied := cell.Copied(root) + if _, err := cell.OpenAt(copied, filepath.Base(copied)); err != nil { + continue + } + remote := filepath.Base(copied) + if held, _, err := s.SendLease(remote); err == nil && held != "" { + remote = held + } + copyable := true + if meta, err := session.LoadMeta(root); err == nil && copied != root { + copyable = olderChatCopyable(meta) + } + out[remote] = appLocalChat{agent: agent, copyable: copyable} + } + return out +} + +// appChatStaging excludes private takeover scratch from app agent admission. +// Opening its copied working presence can attach a journal and directory watch +// before the taker renames it, preventing installation on native Windows. +func appChatStaging(dir string) bool { + return strings.HasSuffix(filepath.Clean(dir), handoff.StagingSuffix) +} + +// Local list/open responses keep their disk-only latency. The desktop's +// explicit inventory bootstrap and sync callbacks share this same follower. +func (b *appBackend) startChatInventory() { + c := &b.chats + if c.inventoryActive() { + return + } + c.start.Do(func() { + guard.Go("app/chats-bootstrap", func() { + if !c.inventoryActive() { + _, _ = b.ListChats() + } + }) + }) +} + +func (c *appChats) isClosed() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.closed +} + +func (c *appChats) inventoryActive() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.closed || c.source != nil +} + +func (b *appBackend) ListChats() (appwire.ChatsResult, error) { + return b.listChats(false) +} + +func (b *appBackend) listChats(publishExisting bool) (appwire.ChatsResult, error) { + c := &b.chats + c.op.Lock() + defer c.op.Unlock() + if c.isClosed() { + return appwire.ChatsResult{}, appwire.Errorf(appwire.CodeClosed, "codeaf is shutting down") + } + s, ok, err := syncsetup.Open(b.relayHome()) + if errors.Is(err, syncsetup.ErrNoIdentity) { + err = nil + } + if err != nil { + return appwire.ChatsResult{}, appEngineError(err) + } + if !ok || s == nil || !syncsetup.MayTalk(b.relayHome()) { + readVersion := func() uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.version + } + version := readVersion() + b.publishChats([]appwire.ChatRow{}, version, false) + return appwire.ChatsResult{Chats: []appwire.ChatRow{}, Version: version}, nil + } + feed := b.startChatFeed(s) + if !c.offered || publishExisting { + b.publishLocalChats(s) + c.offered = true + } + return b.readChats(context.Background(), s, feed, false) +} + +func (b *appBackend) startChatFeed(s *syncsetup.Sync) dirwatch.Follower { + c := &b.chats + c.mu.Lock() + defer c.mu.Unlock() + if c.source == nil { + c.source = s + c.feed = s.Follow() + if c.feed != nil { + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + c.done = make(chan struct{}) + feed, done := c.feed, c.done + guard.Go("app/chats", func() { defer close(done); b.followChats(ctx, feed) }) + } + } + return c.feed +} + +// PublishCopy is also the terminal's background publication door. One finite, +// sequential pass offers existing cell chats without opening or resealing them. +// Older chats are left to the terminal's separate conversion policy. +func (b *appBackend) publishLocalChats(s *syncsetup.Sync) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + guard.Go("app/existing-chats-cancel", func() { + select { + case <-b.left: + cancel() + case <-ctx.Done(): + } + }) + // ReadWorld resolves older originals to conversion copies. Enumerate direct + // cells instead so this pass never offers a legacy home-folder conversion. + roots, _ := filepath.Glob(filepath.Join(s.Home, "v3", "projects", "*", "*")) + name := devname.Name(s.Home) + for _, root := range roots { + if ctx.Err() != nil { + return + } + if !cell.ValidID(filepath.Base(root)) { + continue + } + c, err := cell.OpenAt(root, filepath.Base(root)) + if err != nil { + continue + } + // An unopened shell has nothing to share and needs no relay request. + if head, err := cellstore.Head(c); err != nil || head == nil { + continue + } + meta, err := session.LoadMeta(root) + if err != nil { + continue + } + e := cellstore.EngineFor(meta.Workspace) + e.DataRoot = filepath.Join(s.Home, "v3", "stores") + if meta.Owned { + e.Workspace = filepath.Join(root, "work") + } + one, stop := context.WithTimeout(ctx, cellstore.SealWithin) + _ = s.PublishCopy(one, e, c, name, meta.Title) + stop() + } +} +func (b *appBackend) readChats(parent context.Context, s *syncsetup.Sync, feed dirwatch.Follower, emit bool) (appwire.ChatsResult, error) { + ctx, cancel := context.WithTimeout(parent, syncsetup.DeviceRequestWithin) + defer cancel() + rows, version, err := s.RowsAt(ctx) + if err != nil { + return appwire.ChatsResult{}, appEngineError(err) + } + if result, newer := b.chats.newerChats(version); newer { + return result, nil + } + state := dirwatch.State{} + if feed != nil { + state = feed.State() + } + known := b.devices.watch.Snapshot(s.Identity, s.Device).Online + local := b.localChats(s) + out := make([]appwire.ChatRow, 0, len(rows)) + facts := make(map[string]chatlist.Row, len(rows)) + for _, r := range rows { + if r.Parent != "" || r.Status == chatlist.Branch { + continue + } + facts[r.Cell] = r + _, installed := local[r.Cell] + here := r.Mine && (r.Status == chatlist.Here || installed) + row := appwire.ChatRow{Cell: r.Cell, Title: r.Title, Here: here, Status: string(r.Status), Seen: appwire.Seen{Device: r.DeviceID, Fence: r.Fence, Head: r.Head}, CanTake: !here && r.Head != ""} + if service, ok := s.Dir.(interface{ HeadTakes() bool }); ok && !service.HeadTakes() { + row.Seen.Head = "" + } + if r.DeviceID != "" { + row.Holder = &appwire.LocationHolder{ID: r.DeviceID, Name: r.Device} + } + appChatPresence(&row, r, state, known) + if r.SavedAt > 0 { + row.SavedAt = time.UnixMilli(r.SavedAt).UTC() + } + if l, ok := local[r.Cell]; ok { + row.LocalAgent = l.agent + row.CanTake = row.CanTake && l.copyable + } + if row.Here { + row.Status = "here" + } + out = append(out, row) + } + cacheFacts := func() { + b.chats.mu.Lock() + defer b.chats.mu.Unlock() + b.chats.facts = facts + } + cacheFacts() + b.publishChats(out, version, emit) + return appwire.ChatsResult{Chats: out, Version: version}, nil +} + +func (c *appChats) newerChats(version uint64) (appwire.ChatsResult, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if version < c.version { + return appwire.ChatsResult{Chats: slices.Clone(c.rows), Version: c.version}, true + } + return appwire.ChatsResult{}, false +} + +func appChatPresence(row *appwire.ChatRow, fact chatlist.Row, state dirwatch.State, known map[string]bool) { + row.Online = nil + if fact.DeviceID != "" { + if online, ok := known[fact.DeviceID]; ok { + row.Online = &online + } + } + row.Line = appChatStatusLine(fact, row.Online) + row.TakeoverLine = appChatTakeoverLine(fact, row.Online) +} + +// Inventory operations hold op through caching and emission. A list refreshes +// the cache silently; only the follower compares against rows actually emitted. +func (b *appBackend) publishChats(rows []appwire.ChatRow, version uint64, emit bool) { + old, current := b.chats.cacheRows(rows, version, emit) + if !current || !emit { + return + } + before := map[string]appwire.ChatRow{} + for _, r := range old { + before[r.Cell] = r + } + for _, r := range rows { + prev, exists := before[r.Cell] + delete(before, r.Cell) + if !exists || !reflect.DeepEqual(prev, r) { + b.emit.Emit(appwire.EventChatChanged, appwire.ChatChanged{Chat: r, Version: version}) + } + if r.LocalAgent != "" { + var a *appAgent + b.locked(func() { a = b.agents[r.LocalAgent] }) + if a != nil { + b.located(a, chatLocation(r)) + } + } + } + for id := range before { + b.emit.Emit(appwire.EventChatRemoved, appwire.ChatRemoved{Cell: id, Version: version}) + } +} + +func (c *appChats) cacheRows(rows []appwire.ChatRow, version uint64, emit bool) ([]appwire.ChatRow, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if version < c.version { + return nil, false + } + c.rows = rows + c.byAgent = make(map[string]appwire.ChatRow) + for _, row := range rows { + if row.LocalAgent != "" { + c.byAgent[row.LocalAgent] = row + } + } + c.version = version + old := c.sent + if emit { + c.sent = rows + } + return old, true +} + +func chatLocation(r appwire.ChatRow) *appwire.Location { + seen := r.Seen + line := r.Line + if !r.Here && r.Status == string(chatlist.Idle) { + line = chatlist.Available + } else if !r.Here && r.Holder != nil { + line = chatlist.Superseded(r.Holder.Name) + } + return &appwire.Location{Cell: r.Cell, Seen: &seen, Here: r.Here, Holder: r.Holder, CanTake: r.CanTake, Line: line, TakeoverLine: r.TakeoverLine} +} +func (b *appBackend) followChats(ctx context.Context, feed dirwatch.Follower) { + // Device publication may have connected the shared watch before this + // subscriber joined. Changes announces only future frames, so reconcile + // its current state once; a watch that has not spoken still stays silent. + if feed.State().Up && !b.refreshChatInventory(ctx, feed) { + return + } + for { + select { + case <-ctx.Done(): + return + case _, ok := <-feed.Changes(): + if !ok { + return + } + } + if !b.refreshChatInventory(ctx, feed) { + return + } + } +} + +func (b *appBackend) refreshChatInventory(ctx context.Context, feed dirwatch.Follower) bool { + c := &b.chats + c.op.Lock() + defer c.op.Unlock() + if ctx.Err() != nil { + return false + } + readSource := func() (*syncsetup.Sync, uint64) { + c.mu.Lock() + defer c.mu.Unlock() + return c.source, c.version + } + s, version := readSource() + state := feed.State() + if state.Version > version { + _, _ = b.readChats(ctx, s, feed, true) + } else { + // Presence alone changes no directory version and needs no directory GET. + cloneRows := func() []appwire.ChatRow { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.rows) + } + rows := cloneRows() + known := b.devices.watch.Snapshot(s.Identity, s.Device).Online + for i := range rows { + r := &rows[i] + appChatPresence(r, c.facts[r.Cell], state, known) + } + b.publishChats(rows, version, true) + } + return true +} + +func (b *appBackend) closeChats() { + c := &b.chats + stopFollower := func() (context.CancelFunc, dirwatch.Follower, chan struct{}) { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true + return c.cancel, c.feed, c.done + } + cancel, feed, done := stopFollower() + if cancel != nil { + cancel() + } + if feed != nil { + feed.Close() + } + if done != nil { + <-done + } + // Join foreground continuations and inventory reads before the remaining teardown. + c.take.Lock() + defer c.take.Unlock() + c.op.Lock() + defer c.op.Unlock() +} + +// Cached enrichment is also used on agent.changed; emitting a turn's event +// never adds network I/O. Existing refusal facts take precedence until the next +// directory reading explicitly replaces them. +func (b *appBackend) cachedChatLocation(id string) *appwire.Location { + readRow := func() (appwire.ChatRow, bool) { + b.chats.mu.Lock() + defer b.chats.mu.Unlock() + row, ok := b.chats.byAgent[id] + return row, ok + } + row, ok := readRow() + if !ok { + return nil + } + return chatLocation(row) +} + +func (b *appBackend) dressChatLocation(view *appwire.Agent) { + fresh := b.cachedChatLocation(view.ID) + if fresh == nil { + return + } + if view.Location != nil { + copy := *view.Location + if copy.Cell == "" { + copy.Cell = fresh.Cell + } + if copy.Seen == nil { + copy.Seen = fresh.Seen + } + if !copy.Here { + copy.CanTake = fresh.CanTake + copy.TakeoverLine = fresh.TakeoverLine + } + fresh = © + } + view.Location = fresh +} diff --git a/cmd/codeaf/app_chats_e15_test.go b/cmd/codeaf/app_chats_e15_test.go new file mode 100644 index 0000000000..aa7f4eb190 --- /dev/null +++ b/cmd/codeaf/app_chats_e15_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "net/http" + "slices" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// E15 holds the watch at its initial online snapshot. Chat inventory joins +// after those frames, so no later frame can rescue a missed initial event. +func TestAppChatsE15AlreadyConnectedWatchEmitsOnlineRow(t *testing.T) { + celltest.Available(t) + var version uint64 + var peer string + appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/dir/watch" { + next.ServeHTTP(w, r) + return + } + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer conn.CloseNow() + ctx := conn.CloseRead(r.Context()) + for _, frame := range []string{ + fmt.Sprintf(`{"v":%d}`, version), + fmt.Sprintf(`{"t":"presence","device":%q,"online":true,"at":1}`, peer), + } { + if err := conn.Write(ctx, websocket.MessageText, []byte(frame)); err != nil { + return + } + } + <-ctx.Done() + }) + }) + aHome, bHome := t.TempDir(), t.TempDir() + id, err := identity.Ensure(aHome) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(bHome, id, false); err != nil { + t.Fatal(err) + } + a, b := appDeviceSync(t, aHome), appDeviceSync(t, bHome) + peer = a.Device.ID() + for _, s := range []*syncsetup.Sync{a, b} { + if err := syncsetup.PutOwnDevice(t.Context(), s.Dir, s.Identity, s.Device, devname.Name(s.Home)); err != nil { + t.Fatal(err) + } + } + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if _, err := a.Dir.Create(t.Context(), c.ID, directory.CellInit{Head: "saved-head"}); err != nil { + t.Fatal(err) + } + listing, err := a.Dir.List(t.Context()) + if err != nil { + t.Fatal(err) + } + version = listing.Version + _, rec, wire := appDeviceBackend(t, bHome) + feed := b.Follow() + defer feed.Close() + appSyncWait(t, "E15 watch snapshot before chat subscription", func() bool { + state := feed.State() + return state.Up && state.Version == version && slices.Contains(state.Online, peer) + }) + listed := machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + if len(listed.Chats) != 1 || listed.Chats[0].Online == nil || !*listed.Chats[0].Online { + t.Fatalf("E15 initial online chat: %+v", listed) + } + rec.waitFor(t, 5*time.Second, func(name string, data any) bool { + if name != appwire.EventChatChanged { + return false + } + row := data.(appwire.ChatChanged).Chat + return row.Cell == c.ID && row.Online != nil && *row.Online + }) +} diff --git a/cmd/codeaf/app_chats_live_test.go b/cmd/codeaf/app_chats_live_test.go new file mode 100644 index 0000000000..e3fc4841a2 --- /dev/null +++ b/cmd/codeaf/app_chats_live_test.go @@ -0,0 +1,266 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// L2/L6 are observed through chats.list and chat.changed, with real directory +// version changes. A lease means an open chat, independent of a model turn. +func TestAppChatsL2L6(t *testing.T) { + celltest.Available(t) + var clockAdvance atomic.Int64 + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.URL.Path != "/v1/dir/list" { + next.ServeHTTP(w, r) + return + } + reply := httptest.NewRecorder() + next.ServeHTTP(reply, r) + var listing directory.Listing + if err := json.Unmarshal(reply.Body.Bytes(), &listing); err != nil { + t.Errorf("L2 listing clock: %v", err) + return + } + listing.Now += clockAdvance.Load() + for key, values := range reply.Header() { + w.Header()[key] = values + } + w.WriteHeader(reply.Code) + _ = json.NewEncoder(w).Encode(listing) + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + a := appDeviceSync(t, dirA) + feed := a.Follow() + defer feed.Close() + if err = syncsetup.PutOwnDevice(t.Context(), a.Dir, a.Identity, a.Device, "Mac"); err != nil { + t.Fatal(err) + } + ids := []string{} + for _, title := range []string{"first", "unchanged"} { + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + sealed, err := directory.SealName(directory.MetadataKey(id.CellKey()), title) + if err != nil { + t.Fatal(err) + } + if _, err = a.Dir.Create(t.Context(), c.ID, directory.CellInit{Head: "sealed-head", Title: sealed}); err != nil { + t.Fatal(err) + } + ids = append(ids, c.ID) + } + _, rec, w := appDeviceBackend(t, dirB) + machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}) + rec.waitFor(t, 10*time.Second, func(name string, data any) bool { + if name != appwire.EventChatChanged { + return false + } + r := data.(appwire.ChatChanged).Chat + return r.Cell == ids[1] && r.Online != nil && *r.Online + }) + before := machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}) + for _, r := range before.Chats { + if r.Status != "running" || r.Line != "open on Mac" || strings.Contains(r.TakeoverLine, "running") { + t.Errorf("L6 idle open row: %+v", r) + } + } + rec.mu.Lock() + cursor := len(rec.events) + rec.mu.Unlock() + // Cross a whole-second age boundary without a chat change. Advancing the + // directory version for first must not reissue unchanged's relative age. + clockAdvance.Store(2000) + changed, err := directory.SealName(directory.MetadataKey(id.CellKey()), "changed") + if err != nil { + t.Fatal(err) + } + if _, err = a.Dir.Publish(t.Context(), ids[0], directory.Publish{Fence: 1, OldHead: "sealed-head", Head: "next-head", Title: changed}); err != nil { + t.Fatal(err) + } + machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}) + rec.waitFor(t, 10*time.Second, func(name string, data any) bool { + return name == appwire.EventChatChanged && data.(appwire.ChatChanged).Chat.Title == "changed" + }) + rec.mu.Lock() + defer rec.mu.Unlock() + changedRows, removed := []string{}, 0 + for i := cursor; i < len(rec.events); i++ { + switch rec.events[i] { + case appwire.EventChatChanged: + changedRows = append(changedRows, rec.data[i].(appwire.ChatChanged).Chat.Cell) + case appwire.EventChatRemoved: + removed++ + } + } + if len(changedRows) != 1 || changedRows[0] != ids[0] || removed != 0 { + t.Errorf("L2 changes=%v removed=%d", changedRows, removed) + } +} + +// L1: a completed chat precedes pairing. Only app startup and pairing run; +// neither computer opens the chat or sends a turn to publish it. +func TestAppChatsExistingBeforePairL1(t *testing.T) { + celltest.RequireEngine(t) + var creates atomic.Int64 + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" && cell.ValidID(strings.TrimPrefix(r.URL.Path, "/v1/dir/cells/")) { + creates.Add(1) + } + next.ServeHTTP(w, r) + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + if _, err := identity.Ensure(dirA); err != nil { + t.Fatal(err) + } + a, _, wa := appDeviceBackend(t, dirA) + if _, err := a.ListChats(); err != nil { + t.Fatal(err) + } + c := appFinishedCell(t, dirA, "Before pairing") + b, _, wb := appDeviceBackend(t, dirB) + offer := machineCall[appwire.PairResult](t, wa, "pair.start", appwire.PairStartParams{Mode: "offer_code", Name: "A"}).Pairing + code := e2PairWait(t, wa, offer.ID, "waiting").Code + join := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: code, Name: "B"}).Pairing + question := e2PairWait(t, wa, offer.ID, "question") + machineCall[appwire.PairResult](t, wa, "pair.answer", appwire.PairAnswerParams{ID: offer.ID, Question: question.Question.ID, Option: "y"}) + e2PairWait(t, wa, offer.ID, "done") + e2PairWait(t, wb, join.ID, "done") + if err := b.Close(); err != nil { + t.Fatal(err) + } + _, _, wb = appDeviceBackend(t, dirB) + appSyncWait(t, "L1 pre-pair chat on B", func() bool { + rows := machineCall[appwire.ChatsResult](t, wb, "chats.list", struct{}{}).Chats + for _, r := range rows { + if r.Cell == c.ID { + return !r.Here && r.Holder != nil && r.Holder.Name == "A" && r.CanTake && r.LocalAgent == "" + } + } + return false + }) + if err := a.Close(); err != nil { + t.Fatal(err) + } + before := creates.Load() + startup := appFinishedCell(t, dirA, "Before restart") + restarted, _, _ := appDeviceBackend(t, dirA) + appSyncWait(t, "L1 startup publishes without a list call on A", func() bool { + for _, r := range machineCall[appwire.ChatsResult](t, wb, "chats.list", struct{}{}).Chats { + if r.Cell == startup.ID { + return !r.Here && r.Holder != nil && r.Holder.Name == "A" + } + } + return false + }) + // Synchronize with the startup's single publication pass. + if _, err := restarted.ListChats(); err != nil { + t.Fatal(err) + } + if n := creates.Load(); n != before+1 || before != 1 { + t.Fatalf("L1 creates before=%d after=%d; want one create for each distinct cell", before, n) + } + restarted.locked(func() { + if len(restarted.agents) != 0 { + t.Error("L1 startup opened a chat") + } + }) +} + +func appFinishedCell(t *testing.T, dir, title string) cell.Cell { + t.Helper() + c, err := cell.CreateIn(filepath.Join(dir, "v3", "projects", "local"), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + if err = session.SaveMeta(c.Root, session.Meta{ID: c.ID, Title: title, Workspace: work, LastUserAt: time.Now()}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(c.Root, cell.TranscriptPath), nil, 0600); err != nil { + t.Fatal(err) + } + eng := cellstore.Engine{DataRoot: filepath.Join(dir, "v3", "stores"), Workspace: work} + if _, err = eng.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + return c +} + +// L1 retains #2002 even when an older home-folder chat already has a +// completed conversion copy from an earlier release. +func TestAppChatsOlderCopyStaysLocalL1(t *testing.T) { + celltest.RequireEngine(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + dir := t.TempDir() + if _, err := identity.Ensure(dir); err != nil { + t.Fatal(err) + } + original := filepath.Join(dir, "v3", "projects", "old", "old-chat") + userHome, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + if err = session.SaveMeta(original, session.Meta{ID: "old-chat", Workspace: userHome, LastUserAt: time.Now()}); err != nil { + t.Fatal(err) + } + copied := cell.CopyRoot(original) + c, err := cell.CreateIn(filepath.Dir(copied), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(c.Root, copied); err != nil { + t.Fatal(err) + } + c.Root = copied + c.ID = filepath.Base(copied) + if err = session.SaveMeta(copied, session.Meta{ID: c.ID, Title: "Older home chat", Owned: true, LastUserAt: time.Now()}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(copied, cell.TranscriptPath), nil, 0600); err != nil { + t.Fatal(err) + } + e := cellstore.Engine{DataRoot: filepath.Join(dir, "v3", "stores")} + if _, err = e.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(copied, cell.StateDir, "copy-ready"), nil, 0600); err != nil { + t.Fatal(err) + } + _, _, w := appDeviceBackend(t, dir) + rows := machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}).Chats + if len(rows) != 0 { + t.Fatalf("L1 older home-folder copy was published: %+v", rows) + } +} diff --git a/cmd/codeaf/app_device_publish.go b/cmd/codeaf/app_device_publish.go new file mode 100644 index 0000000000..7e9df188b7 --- /dev/null +++ b/cmd/codeaf/app_device_publish.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/Agent-Field/codeaf/internal/cellsync" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +func (d *appDevices) wakePublication() { + select { + case d.wake <- struct{}{}: + default: + } +} + +// publishOwn owns one retry and shares the inventory's existing watch. A paired +// computer is present even when it has no chats; a successful upsert sleeps +// until a name, pairing or connection changes. +func (d *appDevices) publishOwn(ctx context.Context) { + var source *syncsetup.Sync + var feed dirwatch.Follower + var changes <-chan struct{} + var retry *time.Timer + var later <-chan time.Time + var wait time.Duration + var up bool + var connected bool + var version uint64 + defer func() { + if retry != nil { + retry.Stop() + } + if feed != nil { + feed.Close() + } + }() + attempt := func() error { + d.op.Lock() + defer d.op.Unlock() + if !syncsetup.MayTalk(d.home) { + return nil + } + s, ok, err := syncsetup.Open(d.home) + if err != nil || !ok { + return err + } + if source != nil && (source.Relay != s.Relay || source.Identity.ID() != s.Identity.ID() || source.Device.ID() != s.Device.ID()) && feed != nil { + feed.Close() + feed, changes, up, connected, version = nil, nil, false, false, 0 + } + source = s + // Bind the observer before this new pairing's first request returns. + d.watch.Snapshot(s.Identity, s.Device) + one, cancel := context.WithTimeout(ctx, syncsetup.DeviceRequestWithin) + defer cancel() + if feed == nil { + feed = s.Follow() + if feed != nil { + changes = feed.Changes() + state := feed.State() + up, connected, version = state.Up, state.Up, state.Version + } + } + // Subscribe before the upsert's listing so an intervening directory + // version remains queued. The listing and successful PUT already leave + // a complete roster; a second read must not restart a successful write. + return syncsetup.PutOwnDevice(one, s.Dir, s.Identity, s.Device, devname.Name(d.home)) + } + refresh := func() { + d.op.Lock() + defer d.op.Unlock() + one, cancel := context.WithTimeout(ctx, syncsetup.DeviceRequestWithin) + defer cancel() + _, _ = source.Dir.List(one) + } + for { + err := attempt() + if retry != nil { + retry.Stop() + later = nil + } + if err != nil && !errors.Is(err, directory.ErrRevoked) && !errors.Is(err, directory.ErrRotated) { + if wait == 0 { + wait = cellsync.DefaultInterval + } else { + wait = min(2*wait, cellsync.MaxBackoff) + } + retry = time.NewTimer(wait) + later = retry.C + } else { + wait = 0 + } + waiting: + for { + select { + case <-ctx.Done(): + return + case <-d.wake: + break waiting + case <-later: + break waiting + case <-changes: + state := feed.State() + reconnected := state.Up && !up && connected + changed := state.Version != version + up = state.Up + connected = connected || up + version = state.Version + if reconnected { + break waiting + } + if state.Up && changed { + refresh() + } + } + } + } +} diff --git a/cmd/codeaf/app_devices_a2_test.go b/cmd/codeaf/app_devices_a2_test.go new file mode 100644 index 0000000000..6361f000b4 --- /dev/null +++ b/cmd/codeaf/app_devices_a2_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "net/http" + "testing" + + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/identity" +) + +func TestA2RelayChangeAndMissingSelfRetainReachableRoster(t *testing.T) { + celltest.Available(t) + appDevicesRelay(t) + home := t.TempDir() + if _, err := identity.Ensure(home); err != nil { + t.Fatal(err) + } + old := appDeviceSync(t, home) + if err := old.Rename(t.Context(), "old relay name"); err != nil { + t.Fatal(err) + } + // The new relay has no self record. Its roster is readable even while + // the independent presence door is temporarily unavailable. + appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/dir/presence" || r.Method == http.MethodPut { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"err":"unreachable"}`)) + return + } + next.ServeHTTP(w, r) + }) + }) + b, _, wire := appDeviceBackend(t, home) + view := appDeviceList(t, wire) + if len(view.Devices) != 1 || !view.Devices[0].Self || view.Devices[0].ID != old.Device.ID() || view.Devices[0].Online != nil { + t.Fatalf("A2: readable directory without self: %+v", view) + } + _ = b.Close() +} diff --git a/cmd/codeaf/app_devices_e14_test.go b/cmd/codeaf/app_devices_e14_test.go new file mode 100644 index 0000000000..09e92ff60e --- /dev/null +++ b/cmd/codeaf/app_devices_e14_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "net/http" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +func e14PairWait(t *testing.T, b *appBackend, id string, ready func(appwire.Pairing) bool) appwire.Pairing { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + p, err := b.GetPair(id) + if err != nil { + t.Fatal(err) + } + if p != nil && ready(*p) { + return *p + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("A1: pairing did not reach its expected state") + return appwire.Pairing{} +} + +func TestA1PairedEnginesRetryOwnRecordsWithoutChats(t *testing.T) { + e14PairedEnginesRetry(t, true) +} + +func TestA1PairedEnginesRetryImmediatelyAfterPairingWithoutChats(t *testing.T) { + e14PairedEnginesRetry(t, false) +} + +func e14PairedEnginesRetry(t *testing.T, restart bool) { + t.Helper() + celltest.Available(t) + svc := relayserve.New(relayserve.Config{}) + defer svc.Close() + var allow atomic.Bool + var puts, refused atomic.Int64 + _, _, _ = appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/v1/dir/devices/") { + if !allow.Load() { + refused.Add(1) + http.Error(w, "temporarily unavailable", 503) + return + } + puts.Add(1) + } + if strings.HasPrefix(r.URL.Path, "/v1/dir/") { + next.ServeHTTP(w, r) + } else { + svc.Handler.ServeHTTP(w, r) + } + }) + }) + a, b := t.TempDir(), t.TempDir() + ba, ra, wa := appDeviceBackend(t, a) + bb, rb, wb := appDeviceBackend(t, b) + wa.replyWithin, wb.replyWithin = 90*time.Second, 90*time.Second + offer := machineCall[appwire.PairResult](t, wa, "pair.start", appwire.PairStartParams{Mode: "offer_code", Name: "Abir"}).Pairing + shown := e14PairWait(t, ba, offer.ID, func(p appwire.Pairing) bool { return p.Code != "" }) + join := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: shown.Code, Name: "Abir"}).Pairing + question := e14PairWait(t, ba, offer.ID, func(p appwire.Pairing) bool { return p.Question != nil }) + platform := map[string]string{"windows": "Windows", "darwin": "Mac", "linux": "Linux"}[runtime.GOOS] + if want := "Pair with Abir (" + platform + ")?"; question.Question.Title != want { + t.Fatalf("A3: same-name approval = %q, want %q", question.Question.Title, want) + } + _ = machineCall[appwire.PairResult](t, wa, "pair.answer", appwire.PairAnswerParams{ID: offer.ID, Question: question.Question.ID, Option: "y"}) + for _, side := range []struct { + backend *appBackend + id string + }{{ba, offer.ID}, {bb, join.ID}} { + done := e14PairWait(t, side.backend, side.id, func(p appwire.Pairing) bool { return p.State == "done" || p.State == "failed" }) + if done.State != "done" { + t.Fatalf("A1: pairing failed: %+v", done) + } + for _, line := range done.Lines { + if strings.Contains(line, "could not be shared") || strings.Contains(line, "chat next connects") { + t.Fatalf("A3: pairing success warned about a retried device record: %q", line) + } + } + } + if refused.Load() < 2 { + t.Fatal("A1: both pairing-time writes must fail") + } + if restart { + _ = ba.Close() + _ = bb.Close() + _, ra, wa = appDeviceBackend(t, a) + _, rb, wb = appDeviceBackend(t, b) + wa.replyWithin, wb.replyWithin = 90*time.Second, 90*time.Second + } + allow.Store(true) + for _, rec := range []*appRecorder{ra, rb} { + rec.waitFor(t, 15*time.Second, func(event string, data any) bool { + return event == appwire.EventDevicesChanged && len(data.(appwire.DevicesResult).Devices.Devices) == 2 + }) + } + for _, wire := range []*actionWire{wa, wb} { + if view := appDeviceList(t, wire); len(view.Devices) != 2 || !view.Devices[0].Self { + t.Fatalf("A1: restarted engine roster: %+v", view) + } + } + if puts.Load() != 2 { + t.Fatalf("A1: %d successful PUT attempts, want one per computer", puts.Load()) + } + renamed := machineCall[appwire.DeviceRenameResult](t, wa, "devices.rename", appwire.DeviceRenameParams{Name: "studio"}) + if renamed.Line != "This computer is now called studio." || puts.Load() != 3 { + t.Fatalf("A1/A3: rename was not published immediately: %+v, PUTs=%d", renamed, puts.Load()) + } + appDeviceEvent(t, rb, func(v appwire.Devices) bool { return len(v.Devices) == 2 && v.Devices[1].Name == "studio" }) + _ = machineCall[appwire.DeviceRenameResult](t, wa, "devices.rename", appwire.DeviceRenameParams{Name: "studio"}) + if puts.Load() != 3 { + t.Fatal("A1: unchanged rename rewrote the device") + } + allow.Store(false) + pending := machineCall[appwire.DeviceRenameResult](t, wa, "devices.rename", appwire.DeviceRenameParams{Name: "office"}) + if pending.Line != "This computer is now called office. Your other computers will see it once this computer is back online." { + t.Fatalf("A3: pending rename receipt: %q", pending.Line) + } + allow.Store(true) + rb.waitFor(t, 15*time.Second, func(event string, data any) bool { + if event != appwire.EventDevicesChanged { + return false + } + v := data.(appwire.DevicesResult).Devices + return len(v.Devices) == 2 && v.Devices[1].Name == "office" + }) + if puts.Load() != 4 { + t.Fatalf("A1: retried rename sent %d PUTs, want four total changes", puts.Load()) + } +} diff --git a/cmd/codeaf/app_idle_release_test.go b/cmd/codeaf/app_idle_release_test.go new file mode 100644 index 0000000000..654e15bf17 --- /dev/null +++ b/cmd/codeaf/app_idle_release_test.go @@ -0,0 +1,437 @@ +package main + +import ( + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// I1/I2 use the app's real opener, two homes and a loopback relay. Only the +// idle clock is advanced; ordinary watchers and admission remain installed. +func TestAppIdleReleaseI1I2OpenViewKeepsItsFence(t *testing.T) { appIdleReleaseI1I2(t, true) } +func TestAppIdleReleaseI1I2BackgroundChatStaysReleasedUntilSend(t *testing.T) { + appIdleReleaseI1I2(t, false) +} + +func TestAppBackgroundE13B1B3B4StoppedProcessStaysReleasedAcrossRestart(t *testing.T) { + appIdleReleaseI1I2(t, false, true) +} + +func TestAppBackgroundE13B2QueuedTaskAcquiresAndSettlesAtStartup(t *testing.T) { + celltest.RequireEngine(t) + var acquires atomic.Int32 + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/acquire") { + acquires.Add(1) + } + next.ServeHTTP(w, r) + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + a, rec, _, workspace := appSyncBackend(t) + homeA, _ := outageHomes(t, relay.URL) + t.Cleanup(func() { _ = a.Close() }) + agent, err := mustCreate(t, a, appwire.CreateAsk{Workspace: workspace}) + if err != nil { + t.Fatal(err) + } + turn, err := a.Send(agent.ID, "! echo before-recovery", "") + if err != nil { + t.Fatal(err) + } + appTurnDoneFor(t, rec, turn) + local := mustFind(t, a, agent.ID) + var transcript string + a.locked(func() { transcript = local.transcript }) + if err := a.Close(); err != nil { + t.Fatal(err) + } + seedAppE13StoppedProcess(t, transcript) + checkpoint := filepath.Join(filepath.Dir(transcript), "tasks.json") + raw, err := os.ReadFile(checkpoint) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatal(err) + } + job := document["runs"].([]any)[0].(map[string]any) + job["state"], job["stopped"] = "running", false + delete(job, "endedAt") + // This is genuinely runnable recovery, beyond the old command whose + // process died with its engine. It must retain an automatic opener even + // after that command's two saved records have been reconciled locally. + document["seq"] = 2 + document["nodes"] = []any{map[string]any{ + "id": 2, "title": "B2 queued answer", "brief": "Reply with recovered.", + "state": "queued", "acceptance": "An answer is recorded.", + }} + raw, err = json.Marshal(document) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(checkpoint, raw, 0600); err != nil { + t.Fatal(err) + } + acquires.Store(0) + restarted := e2StartApp(t, homeA) + defer func() { + if t.Failed() { + n, found, err := session.ReadProcessRecord(transcript, 1) + raw, _ := os.ReadFile(checkpoint) + t.Logf("B2 startup: acquires=%d process=%+v found=%v err=%v checkpoint=%s", acquires.Load(), n, found, err, raw) + } + }() + appSyncWait(t, "B2 startup recovery acquired and settled the queued task", func() bool { + n, found, err := session.ReadProcessRecord(transcript, 1) + if acquires.Load() != 1 || err != nil || !found || n.State != session.JobStopped { + return false + } + for _, task := range session.ReadSessionTasks(transcript) { + if task.ID == 2 && (task.State == session.TaskDone || task.State == session.TaskFailed || task.State == session.TaskUnverified) && !task.EndedAt.IsZero() { + t.Logf("B2 queued task recovered as %s: %s", task.State, task.Report) + return true + } + } + return false + }) + e2Quit(t, restarted) +} + +func appIdleReleaseI1I2(t *testing.T, viewed bool, stopped ...bool) { + celltest.RequireEngine(t) + var acquires, releases, beats atomic.Int32 + var watches, devices, vaults atomic.Int32 + leftover := len(stopped) > 0 && stopped[0] + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/watch") { + watches.Add(1) + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/devices/") { + devices.Add(1) + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/vault") { + vaults.Add(1) + } + if strings.HasSuffix(r.URL.Path, "/acquire") { + acquires.Add(1) + } + if strings.HasSuffix(r.URL.Path, "/release") { + releases.Add(1) + } + if strings.HasSuffix(r.URL.Path, "/heartbeat") { + beats.Add(1) + } + next.ServeHTTP(w, r) + }) + }) + // Keep the relay setting alive through the parent's registered teardown. + // outageHomes repeats these values after that teardown has been registered. + t.Setenv(syncsetup.URLVar, relay.URL) + t.Setenv(syncsetup.IntervalVar, "50") + a, rec, _, workspace := appSyncBackend(t) + _, homeB := outageHomes(t, relay.URL) + var idleClock atomic.Int64 + if leftover { + clock := a.now + a.now = func() time.Time { return clock().Add(time.Duration(idleClock.Load())) } + } + // outageHomes stops spawned stores. Close the parent first so its final + // save cannot restart a store while the fixture is being dismantled. + t.Cleanup(func() { _ = a.Close() }) + homeA := home.Dir() + _, peer, wire := appDeviceBackend(t, homeB) + t.Setenv(home.EnvVar, homeA) + a.snapshotCloseClaims() + run := a.crews() + a.startCloseWatch() + agent, err := mustCreate(t, a, appwire.CreateAsk{Workspace: workspace}) + if err != nil { + t.Fatal(err) + } + turn, err := a.Send(agent.ID, "! echo before-idle", "") + if err != nil { + t.Fatal(err) + } + appTurnDoneFor(t, rec, turn) + own := appSyncOwn(t) + row := idleReleaseRow(t, wire, agent.ID, func(r appwire.ChatRow) bool { return r.Seen.Head != "" }) + appSyncWait(t, "I1 idle turn settled", func() bool { + var idle bool + a.locked(func() { idle = a.agents[agent.ID].live != nil && !a.agents[agent.ID].live.running }) + return idle + }) + var transcript string + if leftover { + // Model the old engine's mismatch after its final checkpoint: the + // stopped job is in tasks.json, but its process file still says running. + func() { + a := mustFind(t, a, agent.ID) + a.gate.Lock() + defer a.gate.Unlock() + live := a.live + transcript = a.transcript + release := live.conn.Release + live.conn.Release = func() { + release() + seedAppE13StoppedProcess(t, transcript) + } + }() + } + if viewed { + if _, err := a.OpenAgent(agent.ID); err != nil { + t.Fatal(err) + } + } + peer.waitFor(t, 10*time.Second, func(_ string, data any) bool { + changed, ok := data.(appwire.ChatChanged) + return ok && changed.Chat.Cell == agent.ID && changed.Chat.Seen == row.Seen && changed.Chat.Online != nil && *changed.Chat.Online + }) + peer.mu.Lock() + cursor := len(peer.events) + peer.mu.Unlock() + before, err := own.Dir.Cell(t.Context(), agent.ID) + if err != nil { + t.Fatal(err) + } + acquires.Store(0) + releases.Store(0) + network := [3]int32{watches.Load(), devices.Load(), vaults.Load()} + for period := 0; period < 2; period++ { + if leftover { + idleClock.Add(int64(appAgentIdle + time.Second)) + } else { + a.locked(func() { + if live := a.agents[agent.ID].live; live != nil { + live.lastUse = a.now().Add(-appAgentIdle - time.Second) + } + }) + } + a.sweepOnce() + a.crewTick(run, false) + // Live read doors must keep using the existing open conversation. + if viewed { + if _, err := a.ListSkills(agent.ID); err != nil { + t.Fatal(err) + } + if _, err := a.GetAutonomy(agent.ID); err != nil { + t.Fatal(err) + } + if _, err := a.ConversationPoints(agent.ID); err != nil { + t.Fatal(err) + } + } + if _, err := a.ListAgents(""); err != nil { + t.Fatal(err) + } + if _, _, err := a.Entries(agent.ID, "", appwire.MaxEntries); err != nil { + t.Fatal(err) + } + if _, err := a.ListTasks(agent.ID); err != nil { + t.Fatal(err) + } + if !viewed && period == 0 { + peer.waitFor(t, 10*time.Second, func(_ string, data any) bool { + changed, ok := data.(appwire.ChatChanged) + return ok && changed.Chat.Cell == agent.ID && changed.Chat.Status == "idle" + }) + } + // Give the automatic opener and directory follower time to react. + time.Sleep(250 * time.Millisecond) + v, err := own.Dir.Cell(t.Context(), agent.ID) + if err != nil { + t.Fatal(err) + } + wantReleases := int32(1) + if viewed { + wantReleases = 0 + } + live := v.Cell.Lease.Expires > v.Now + if live != viewed || v.Cell.Lease.Fence != row.Seen.Fence || acquires.Load() != 0 || releases.Load() != wantReleases { + t.Fatalf("I1/I2 period %d: lease=%+v acquires=%d releases=%d", period+1, v.Cell.Lease, acquires.Load(), releases.Load()) + } + if leftover { + if got := [3]int32{watches.Load(), devices.Load(), vaults.Load()}; got != network { + t.Fatalf("B4 idle release restarted sync traffic: before=%v after=%v (watch/device/vault)", network, got) + } + } + } + if viewed { + appSyncWait(t, "I1 open idle hold to renew beyond its original deadline", func() bool { + v, err := own.Dir.Cell(t.Context(), agent.ID) + return err == nil && v.Cell.Lease.Expires > before.Cell.Lease.Expires + }) + if acquires.Load() != 0 || releases.Load() != 0 { + t.Fatalf("I1 renewal acquired=%d released=%d", acquires.Load(), releases.Load()) + } + } + peer.mu.Lock() + changes := 0 + for _, data := range peer.data[cursor:] { + if changed, ok := data.(appwire.ChatChanged); ok && changed.Chat.Cell == agent.ID { + changes++ + } + } + peer.mu.Unlock() + wantChanges := 1 + if viewed { + wantChanges = 0 + } + if changes != wantChanges { + t.Fatalf("I2 idle chat emitted %d changes, want %d", changes, wantChanges) + } + t.Logf("I1/I2 viewed=%v: idle acquires=%d releases=%d heartbeat requests=%d row changes=%d fence=%d", viewed, acquires.Load(), releases.Load(), beats.Load(), changes, row.Seen.Fence) + if leftover { + if session.NeedsProcessRecovery(transcript) { + t.Fatal("B1 settled process still demands recovery after the sweep") + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + // A fresh engine must also leave the known ending closed without acquiring. + seedAppE13StoppedProcess(t, transcript) + restarted := e2StartApp(t, homeA) + for range 2 { + time.Sleep(250 * time.Millisecond) + v, err := own.Dir.Cell(t.Context(), agent.ID) + if err != nil || v.Cell.Lease.Fence != row.Seen.Fence || acquires.Load() != 0 || releases.Load() != 1 { + t.Fatalf("B1/B3 restart acquired: lease=%+v acquires=%d releases=%d err=%v", v.Cell.Lease, acquires.Load(), releases.Load(), err) + } + } + e2Turn(t, restarted, agent.ID, "send after background restart") + newRow := idleReleaseRow(t, wire, agent.ID, func(r appwire.ChatRow) bool { return r.Seen.Fence == row.Seen.Fence+1 && r.Seen.Head != row.Seen.Head }) + if acquires.Load() != 1 || newRow.Holder == nil || newRow.Holder.ID != own.Device.ID() { + t.Fatalf("B3 send after restart: row=%+v acquires=%d", newRow, acquires.Load()) + } + return + } + turn, err = a.Send(agent.ID, "! echo after-idle", "") + if err != nil { + t.Fatal(err) + } + appTurnDoneFor(t, rec, turn) + wantFence, wantAcquires := row.Seen.Fence+1, int32(1) + if viewed { + wantFence, wantAcquires = row.Seen.Fence, 0 + } + newRow := idleReleaseRow(t, wire, agent.ID, func(r appwire.ChatRow) bool { return r.Seen.Fence == wantFence && r.Seen.Head != row.Seen.Head }) + if acquires.Load() != wantAcquires || newRow.Holder == nil || newRow.Holder.ID != own.Device.ID() { + t.Fatalf("I1 send: row=%+v acquires=%d", newRow, acquires.Load()) + } +} + +func seedAppE13StoppedProcess(t *testing.T, transcript string) { + t.Helper() + dir := filepath.Dir(transcript) + if err := os.MkdirAll(filepath.Join(dir, "processes"), 0700); err != nil { + t.Fatal(err) + } + for name, raw := range map[string]string{ + "processes/head.json": `{"epoch":"e13","high":1}`, + "processes/1.json": `{"ID":1,"Command":"say hi","Kind":"command","State":"running","Started":"2026-10-27T23:58:18Z","Elapsed":379625,"ToolCallID":"call_e13","StartedBy":"chat"}`, + "tasks.json": `{"type":"tasks","version":1,"seq":1,"runs":[{"id":1,"kind":"job","title":"say hi","state":"failed","stopped":true,"report":"job 1","job_command":"say hi","job_call_id":"call_e13","startedAt":"2026-10-27T23:58:18Z","endedAt":"2026-10-27T23:58:18.000379625Z"}]}`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(raw), 0600); err != nil { + t.Fatal(err) + } + } +} + +func idleReleaseRow(t *testing.T, wire *actionWire, id string, ready func(appwire.ChatRow) bool) appwire.ChatRow { + t.Helper() + var found appwire.ChatRow + appSyncWait(t, "I1/I2 peer row "+id, func() bool { + for _, row := range machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats { + if row.Cell == id && ready(row) { + found = row + return true + } + } + return false + }) + return found +} + +func TestAppIdleReleaseI1WorkDoorsStillOpenTheConversation(t *testing.T) { + for _, door := range []string{"send", "queue", "steer", "retry", "close then open"} { + t.Run(door, func(t *testing.T) { + var opens atomic.Int32 + conv := &fakeConversation{script: []session.Event{{Kind: session.EventTurnDone}}} + rec := newAppRecorder() + b := newFakeBackend(rec, func(ask appOpenAsk) (*appConn, error) { + opens.Add(1) + return &appConn{Conv: conv, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() {}}, nil + }) + t.Cleanup(func() { _ = b.Close() }) + agent, err := mustCreate(t, b, appwire.CreateAsk{Workspace: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if door == "retry" { + conv.mu.Lock() + conv.script = []session.Event{{Kind: session.EventError, Err: errors.New("try again")}} + conv.mu.Unlock() + turn, err := b.Send(agent.ID, "work", "") + if err != nil { + t.Fatal(err) + } + rec.waitFor(t, time.Second, func(_ string, data any) bool { + done, ok := data.(appwire.TurnDone) + return ok && done.Turn == turn && !done.OK + }) + appSyncWait(t, "I1 failed turn idle", func() bool { + var idle bool + b.locked(func() { idle = !b.agents[agent.ID].live.running }) + return idle + }) + conv.mu.Lock() + conv.script = []session.Event{{Kind: session.EventTurnDone}} + conv.mu.Unlock() + } + b.locked(func() { b.agents[agent.ID].live.lastUse = b.now().Add(-appAgentIdle - time.Second) }) + b.sweepOnce() + if opens.Load() != 1 { + t.Fatalf("I1 view reopened %d times", opens.Load()) + } + switch door { + case "send": + _, err = b.Send(agent.ID, "work", "") + case "queue": + _, err = b.Queue(agent.ID, "work", "") + case "steer": + err = b.Steer(agent.ID, "work", "") + // Steering requires a running turn; an idle refusal does not + // admit work or need a writer. + if wireCode(err) != appwire.CodeBusy || opens.Load() != 1 { + t.Fatalf("I1 idle steer: %d opens, %v", opens.Load(), err) + } + return + case "retry": + _, err = b.Retry(agent.ID) + case "close then open": + err = b.CloseAgent(agent.ID) + if err == nil { + _, err = b.OpenAgent(agent.ID) + } + } + if err != nil || opens.Load() != 2 { + t.Fatalf("I1 %s: %d opens, %v", door, opens.Load(), err) + } + }) + } +} diff --git a/cmd/codeaf/app_outage_test.go b/cmd/codeaf/app_outage_test.go new file mode 100644 index 0000000000..bfc7205af5 --- /dev/null +++ b/cmd/codeaf/app_outage_test.go @@ -0,0 +1,427 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// The relay retains its directory and store while the listener +// and every connection (including hijacked watch sockets) go away and return. +type outageRelay struct { + t *testing.T + clock *outageClock + handler http.Handler + directory *directory.Memory + server *httptest.Server + address string + mu sync.Mutex + connections map[net.Conn]bool + blocked atomic.Value // certificate held offline until the competing take finishes +} + +type outageClock struct{ offset atomic.Int64 } + +func (c *outageClock) Now() time.Time { return time.Now().Add(time.Duration(c.offset.Load())) } +func (c *outageClock) Advance(d time.Duration) { c.offset.Add(int64(d)) } + +func newOutageRelay(t *testing.T) *outageRelay { + t.Helper() + clock := &outageClock{} + r := &outageRelay{t: t, clock: clock, connections: map[net.Conn]bool{}} + r.directory = directory.NewMemory(clock.Now) + mux := http.NewServeMux() + auth := reqsign.AuthenticateAt(clock.Now) + mux.Handle("/v1/dir/", directory.Handler(auth, func(string) (directory.Directory, error) { return r.directory, nil })) + store := blobstore.NewMemory() + mux.Handle("/v1/store/", blobstore.Handler(auth, func(string) (blobstore.Store, error) { return store, nil })) + r.handler = mux + r.blocked.Store("") + r.start() + t.Cleanup(func() { r.stop(); _ = r.directory.Close() }) + return r +} + +func (r *outageRelay) start() { + r.t.Helper() + address := r.address + if address == "" { + address = "127.0.0.1:0" + } + listener, err := net.Listen("tcp", address) + if err != nil { + r.t.Fatal(err) + } + r.address = listener.Addr().String() + s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Header.Get(reqsign.HeaderCert) == r.blocked.Load().(string) { + http.Error(w, "test computer still disconnected", http.StatusServiceUnavailable) + return + } + r.handler.ServeHTTP(w, req) + })) + _ = s.Listener.Close() + s.Listener = listener + s.Config.ConnState = func(c net.Conn, state http.ConnState) { + r.mu.Lock() + defer r.mu.Unlock() + if state == http.StateClosed { + delete(r.connections, c) + } else { + r.connections[c] = true + } + } + s.Start() + r.server = s +} + +func (r *outageRelay) stop() { + if r.server == nil { + return + } + _ = r.server.Listener.Close() + r.mu.Lock() + for c := range r.connections { + _ = c.Close() + } + r.mu.Unlock() + r.server.Close() + r.server = nil +} + +func outageHomes(t *testing.T, relayURL string) (string, string) { + t.Helper() + t.Setenv(syncsetup.URLVar, relayURL) + t.Setenv(syncsetup.IntervalVar, "50") + a, b := home.Dir(), e2TempDir(t) + id, err := identity.Ensure(a) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(b, id, false); err != nil { + t.Fatal(err) + } + for dir, name := range map[string]string{a: "A", b: "B"} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"tools.approvalMode":"allow"}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := devname.Set(dir, name); err != nil { + t.Fatal(err) + } + s := appDeviceSync(t, dir) + if err := syncsetup.PutOwnDevice(t.Context(), s.Dir, s.Identity, s.Device, name); err != nil { + t.Fatal(err) + } + } + t.Cleanup(cellstore.StopSpawned) + return a, b +} + +func outageChatRow(t *testing.T, app *e2App, id string, ready func(appwire.ChatRow) bool) appwire.ChatRow { + t.Helper() + var found appwire.ChatRow + outageWait(t, "E8 chat row "+id, func() bool { + rows := machineCall[appwire.ChatsResult](t, app.w, "chats.list", struct{}{}) + for _, row := range rows.Chats { + if (id == "" || row.Cell == id) && ready(row) { + found = row + return true + } + } + return false + }) + return found +} + +func TestAppRelayOutageO1O2O3O4O5(t *testing.T) { appRelayOutage(t, false) } +func TestAppRelayOutageO2CompetingTakeKeepsWork(t *testing.T) { appRelayOutage(t, true) } + +func TestAppRelayOutageO3O4StartupRecovery(t *testing.T) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + model := appTwoMessageSource(t, "echo offline-start") + t.Cleanup(model.Close) + t.Setenv("CODEAF_BASE_URL", model.URL) + relay := newOutageRelay(t) + dirA, dirB := outageHomes(t, relay.server.URL) + a := e2StartApp(t, dirA) + relay.stop() + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, a.w, "agents.create", map[string]any{"workspace": workspace}).Agent + e2Turn(t, a, created.ID, "opened while relay was down") + relay.start() + b := e2StartApp(t, dirB) + outageChatRow(t, b, "", func(r appwire.ChatRow) bool { return r.Title != "" && r.CanTake && !r.SavedAt.IsZero() }) +} + +func TestAppTakeC1UnavailableModelUsesLocalDefaultOnce(t *testing.T) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + t.Setenv(config.APIKeyEnv, "") + model := appTwoMessageSource(t, "echo model-recovery") + t.Cleanup(model.Close) + var requestedMu sync.Mutex + var requested []string + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(raw)) + var request struct { + Model string `json:"model"` + } + _ = json.Unmarshal(raw, &request) + if request.Model != "" { + requestedMu.Lock() + requested = append(requested, request.Model) + requestedMu.Unlock() + } + if strings.HasPrefix(request.Model, "ollama/") || strings.HasPrefix(request.Model, "receiver/") { + http.Error(w, "unavailable model reached the router", http.StatusBadRequest) + return + } + model.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(endpoint.Close) + t.Setenv("CODEAF_BASE_URL", endpoint.URL) + relay := e2WireRelay(t) + dirA, dirB := outageHomes(t, relay.URL) + if err := config.WriteSources(dirA, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: endpoint.URL}}); err != nil { + t.Fatal(err) + } + if err := config.WriteChatModel(dirA, "ollama/llama3.2:latest"); err != nil { + t.Fatal(err) + } + if err := config.WriteSources(dirB, []config.PersistedSource{{ID: "custom-receiver", Written: "receiver", Address: endpoint.URL, Key: "receiver-key"}}); err != nil { + t.Fatal(err) + } + if err := config.WriteChatModel(dirB, "receiver/local-default"); err != nil { + t.Fatal(err) + } + configured := func(dir string) *e2App { + return e2StartAppWithSetup(t, dir, func(app *e2App) { + machineCall[any](t, app.w, "setup.get", struct{}{}) + machineCall[appwire.ChatsResult](t, app.w, "chats.list", struct{}{}) + }) + } + a, b := configured(dirA), configured(dirB) + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, a.w, "agents.create", map[string]any{"workspace": workspace, "model": "ollama/llama3.2:latest"}).Agent + if created.Model != "ollama/llama3.2:latest" { + t.Fatalf("C1 source model = %q", created.Model) + } + e2Turn(t, a, created.ID, "from Ollama") + e2Quit(t, a) + row := outageChatRow(t, b, "", func(r appwire.ChatRow) bool { return r.Title != "" && r.CanTake }) + meta, err := session.LoadMeta(takeRootForAt(dirA, row.Cell)) + if err != nil { + t.Fatal(err) + } + if meta.Model != "ollama/llama3.2:latest" { + t.Fatalf("C1 source saved model = %q", meta.Model) + } + taken := machineCall[appwire.TakeResult](t, b.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}).Agent + if taken.Model != "receiver/local-default" { + t.Fatalf("C1 taken model = %q, want the usable local default", taken.Model) + } + e2Turn(t, b, taken.ID, "first local turn") + e2Turn(t, b, taken.ID, "second local turn") + requestedMu.Lock() + usedDefault := false + for _, name := range requested { + usedDefault = usedDefault || name == "local-default" + } + requestedMu.Unlock() + if !usedDefault { + t.Fatal("C1 no request used the receiver's default") + } + b.mu.Lock() + events := append([]e2Frame(nil), b.events...) + b.mu.Unlock() + count := 0 + for _, f := range events { + if f.Event != appwire.EventEntryAdded { + continue + } + var event appwire.EntryAdded + _ = json.Unmarshal(f.Data, &event) + if strings.Contains(event.Entry.Text, "isn't set up here") && strings.Contains(event.Entry.Text, "receiver/local-default") { + count++ + } + } + if count != 1 { + t.Fatalf("C1 model recovery notices = %d, want one", count) + } + // The new main has never been called, so the last request's provider is + // not enough to describe the saved model choice on the returning computer. + machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, b.w, "agents.set_models", map[string]any{"id": taken.ID, "main": "receiver/never-called", "fallback": ""}) + e2Quit(t, b) + if err := config.WriteAPIKey(dirA, "original-router-key"); err != nil { + t.Fatal(err) + } + a = configured(dirA) + back := outageChatRow(t, a, row.Cell, func(r appwire.ChatRow) bool { return r.CanTake && r.Seen.Device != row.Seen.Device }) + returned := machineCall[appwire.TakeResult](t, a.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: back.Seen}).Agent + if returned.Model != "ollama/llama3.2:latest" { + t.Fatalf("C1 returning home used %q", returned.Model) + } + e2Turn(t, a, returned.ID, "back on original computer") +} + +func appRelayOutage(t *testing.T, competing bool) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + model := appTwoMessageSource(t, "echo outage") + t.Cleanup(model.Close) + t.Setenv("CODEAF_BASE_URL", model.URL) + relay := newOutageRelay(t) + dirA, dirB := outageHomes(t, relay.server.URL) + a, b := e2StartApp(t, dirA), e2StartApp(t, dirB) + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, a.w, "agents.create", map[string]any{"workspace": workspace}).Agent + e2Turn(t, a, created.ID, "before take") + e2Quit(t, a) + row := outageChatRow(t, b, "", func(r appwire.ChatRow) bool { return r.Title != "" && r.CanTake }) + taken := machineCall[appwire.TakeResult](t, b.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}).Agent + e2Turn(t, b, taken.ID, "before outage") + a = e2StartApp(t, dirA) + before := outageChatRow(t, a, row.Cell, func(r appwire.ChatRow) bool { return r.SavedAt.After(row.SavedAt) && r.Seen.Device != row.Seen.Device }) + relay.stop() + relay.clock.Advance(directory.LeaseTTL + time.Second) + e2Turn(t, b, taken.ID, "saved during outage") + c, err := cell.OpenAt(takeRootForAt(dirB, row.Cell), row.Cell) + if err != nil { + t.Fatal(err) + } + transcript, err := os.ReadFile(session.Place{Dir: c.Root}.Transcript()) + if err != nil || !bytes.Contains(transcript, []byte("saved during outage")) { + t.Fatalf("O4 outage transcript missing: %v", err) + } + end := int64(len(transcript)) + var latest *cellstore.Sealed + appSyncWait(t, "O4 local outage turn saved", func() bool { + latest, err = cellstore.Head(c) + return err == nil && latest != nil && latest.Receipt.Transcript.End >= end + }) + if competing { + s := appDeviceSync(t, dirB) + cert, _ := json.Marshal(s.Device.Cert) + relay.blocked.Store(base64.RawURLEncoding.EncodeToString(cert)) + } + relay.start() + if competing { + fresh, err := appDeviceSync(t, dirA).Dir.Cell(t.Context(), row.Cell) + if err != nil { + t.Fatal(err) + } + machineCall[appwire.TakeResult](t, a.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: appwire.Seen{Device: fresh.Cell.Lease.Device, Fence: fresh.Cell.Lease.Fence, Head: fresh.Cell.Head}}) + relay.blocked.Store("") + b.wait(t, func(f e2Frame) bool { + if f.Event != appwire.EventAgentChanged { + return false + } + var event appwire.AgentChanged + _ = json.Unmarshal(f.Data, &event) + l := event.Agent.Location + return event.Agent.ID == taken.ID && l != nil && !l.Here && l.Holder != nil && l.Holder.Name == "A" && strings.Contains(l.Line, "A continued this chat") + }) + outageWait(t, "O2 pending work kept on relay", func() bool { + listing, err := appDeviceSync(t, dirA).Dir.List(t.Context()) + if err != nil { + return false + } + for _, record := range listing.Cells { + if record.ParentCell == row.Cell && record.OrphanTurns > 0 { + // A late accounting seal may extend the outage turn while + // displacement settles; the kept head must include that turn. + turns, err := cellstore.Turns(c) + if err != nil { + t.Fatal(err) + } + contains := false + for _, turn := range turns { + contains = contains || turn.ID == latest.Turn.ID + if turn.ID == record.Head && contains { + return true + } + } + } + } + return false + }) + _, refusal := b.w.call("agents.send", map[string]any{"id": taken.ID, "text": "after A took", "client_id": "after-take"}) + if refusal == nil || !strings.Contains(string(e2JSON(refusal.Detail)), "held_elsewhere") { + t.Fatalf("O2 lost ownership did not refuse plainly: %+v", refusal) + } + return + } + saved, err := cellstore.SavedAt(c, latest.Turn.ID) + if err != nil { + t.Fatal(err) + } + recovered := outageChatRow(t, a, row.Cell, func(r appwire.ChatRow) bool { + return r.Seen.Head == latest.Turn.ID && r.SavedAt.UnixMilli() == saved && r.Online != nil && *r.Online && !strings.Contains(r.Line, "offline") + }) + if recovered.Seen.Device != before.Seen.Device || recovered.Seen.Fence < before.Seen.Fence { + t.Fatalf("O5 wrong recovered holder: %+v", recovered) + } + // O3 recovery above required no send, quit or reopen. A later send still works. + e2Turn(t, b, taken.ID, "after recovery") + b.mu.Lock() + defer b.mu.Unlock() + for _, f := range b.events { + if f.Event == appwire.EventAgentChanged { + var event appwire.AgentChanged + _ = json.Unmarshal(f.Data, &event) + l := event.Agent.Location + if event.Agent.ID == taken.ID && l != nil && !l.Here && l.Holder != nil && l.Holder.ID == before.Seen.Device { + t.Fatalf("O1 own chat became a viewer: %s", f.Data) + } + } + if f.Event == appwire.EventEntryAdded && strings.Contains(string(f.Data), "B continued this chat") { + t.Fatalf("O1 self displacement notice: %s", f.Data) + } + } +} + +func outageWait(t *testing.T, what string, ready func() bool) { + t.Helper() + deadline := time.Now().Add(90 * time.Second) + for !ready() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/cmd/codeaf/app_pair.go b/cmd/codeaf/app_pair.go new file mode 100644 index 0000000000..766406a177 --- /dev/null +++ b/cmd/codeaf/app_pair.go @@ -0,0 +1,414 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/pair" +) + +const appJoinBusy = "Finish the work on this computer before joining another computer." +const appJoining = "Pairing is joining another computer. Finish pairing and restart codeaf before starting work." + +type appPair struct { + last *appPairOperation + mu sync.Mutex + next uint64 + current *appPairOperation + restart bool +} +type appPairOperation struct { + backend *appBackend + ctx context.Context + cancel context.CancelFunc + done chan struct{} + answer chan string + view appwire.Pairing + question uint64 + joining bool + finished bool +} + +func pairSnapshot(p appwire.Pairing) appwire.Pairing { + p.Lines = slices.Clone(p.Lines) + if p.Question != nil { + q := *p.Question + q.Options = slices.Clone(q.Options) + p.Question = &q + } + if p.Error != nil { + e := *p.Error + p.Error = &e + } + return p +} +func (*appBackend) PairingAvailable() bool { return cell.Available() } + +func (b *appBackend) StartPair(p appwire.PairStartParams) (appwire.Pairing, error) { + switch p.Mode { + case "offer_code", "request_link": + case "use_code": + if strings.TrimSpace(p.Code) == "" { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBadParams, "code is required") + } + case "approve_link": + if strings.TrimSpace(p.Link) == "" { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBadParams, "link is required") + } + default: + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBadParams, "unknown pairing mode") + } + if p.Name != "" { + if _, err := devname.Clean(p.Name); err != nil { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBadParams, "%s", err) + } + } + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + if b.pair.current != nil && !b.pair.current.finished { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBusy, "Another pairing is running on this computer.") + } + if b.pair.restart { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBusy, "Restart codeaf to finish joining this computer.") + } + joining := p.Mode == "use_code" || p.Mode == "request_link" + busy, closed := false, false + b.locked(func() { + closed = b.closing + if joining { + for _, a := range b.agents { + if a.opening || a.starting || a.view.State == appwire.StateWorking || a.view.State == appwire.StateNeedsYou || a.live != nil && (a.live.running || a.live.needsYou()) { + busy = true + break + } + } + } + }) + if closed { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeClosed, "codeaf is shutting down") + } + if busy { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBusy, appJoinBusy) + } + ctx, cancel := context.WithCancel(context.Background()) + b.pair.next++ + op := &appPairOperation{backend: b, ctx: ctx, cancel: cancel, done: make(chan struct{}), answer: make(chan string, 1), joining: joining, + view: appwire.Pairing{ID: fmt.Sprintf("pair-%d", b.pair.next), Mode: p.Mode, State: "preparing", Lines: []string{}}} + b.pair.current = op + op.emitLocked() + guard.Go("app/pair", func() { defer close(op.done); defer cancel(); op.run(p) }) + return pairSnapshot(op.view), nil +} + +// The previous finished operation remains addressable while the next runs. +func (b *appBackend) pairOperationLocked(id string) *appPairOperation { + if op := b.pair.current; op != nil && (id == "" || op.view.ID == id) { + return op + } + if op := b.pair.last; op != nil && (id == "" || op.view.ID == id) { + return op + } + return nil +} + +func (b *appBackend) GetPair(id string) (*appwire.Pairing, error) { + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + op := b.pairOperationLocked(id) + if op == nil && id == "" { + return nil, nil + } + if op == nil || id != "" && op.view.ID != id { + return nil, appwire.Errorf(appwire.CodeNotFound, "that pairing is not here") + } + p := pairSnapshot(op.view) + return &p, nil +} +func (b *appBackend) AnswerPair(p appwire.PairAnswerParams) (appwire.Pairing, error) { + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + op := b.pairOperationLocked(p.ID) + if op == nil || op.view.ID != p.ID { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeNotFound, "that pairing is not here") + } + q := op.view.Question + if op.finished || q == nil || q.ID != p.Question || !slices.ContainsFunc(q.Options, func(o appwire.PairOption) bool { return o.ID == p.Option }) { + return appwire.Pairing{}, appwire.Errorf(appwire.CodeBadParams, "that answer is not waiting") + } + op.view.Question, op.view.State = nil, "waiting" + op.emitLocked() + op.answer <- p.Option + return pairSnapshot(op.view), nil +} +func (b *appBackend) CancelPair(id string) (appwire.Pairing, error) { + cancelPair := func() (*appPairOperation, error) { + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + op := b.pairOperationLocked(id) + if op == nil || op.view.ID != id { + return nil, appwire.Errorf(appwire.CodeNotFound, "that pairing is not here") + } + op.cancel() + return op, nil + } + op, err := cancelPair() + if err != nil { + return appwire.Pairing{}, err + } + <-op.done + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + return pairSnapshot(op.view), nil +} +func (b *appBackend) closePair() { + cancelCurrent := func() *appPairOperation { + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + op := b.pair.current + if op != nil { + op.cancel() + } + return op + } + op := cancelCurrent() + if op != nil { + <-op.done + } +} +func (b *appBackend) pairingWorkRefusal() error { + b.pair.mu.Lock() + defer b.pair.mu.Unlock() + if b.pair.restart || b.pair.current != nil && b.pair.current.joining && !b.pair.current.finished { + return appwire.Errorf(appwire.CodeBusy, appJoining) + } + return nil +} +func (o *appPairOperation) emitLocked() { + o.backend.emit.Emit(appwire.EventPairChanged, appwire.PairResult{Pairing: pairSnapshot(o.view)}) +} +func (o *appPairOperation) update(fn func(*appwire.Pairing)) { + o.backend.pair.mu.Lock() + defer o.backend.pair.mu.Unlock() + if o.finished { + return + } + fn(&o.view) + o.emitLocked() +} +func (o *appPairOperation) line(line string) { + o.update(func(p *appwire.Pairing) { p.Lines = append(p.Lines, strings.Split(line, "\n")...) }) +} +func (o *appPairOperation) ask(ctx context.Context, kind, title, detail string, options []appwire.PairOption) string { + o.update(func(p *appwire.Pairing) { + o.question++ + p.State = "question" + p.Question = &appwire.PairQuestion{ID: fmt.Sprint(o.question), Kind: kind, Title: title, Detail: detail, Options: options} + }) + select { + case a := <-o.answer: + return a + case <-ctx.Done(): + return "" + case <-o.ctx.Done(): + return "" + } +} +func (o *appPairOperation) finish(state, line string, err error, restart bool) { + o.backend.pair.mu.Lock() + defer o.backend.pair.mu.Unlock() + if o.finished { + return + } + o.finished = true + o.backend.pair.last = o + o.view.State = state + o.view.Question = nil + o.view.Code = "" + o.view.Link = "" + o.view.ExpiresAt = time.Time{} + if line != "" { + o.view.Lines = append(o.view.Lines, strings.Split(line, "\n")...) + } + if err != nil { + o.view.Error = &appwire.PairError{Code: appwire.CodeEngine, Message: err.Error()} + } + o.view.Restart = restart + o.backend.pair.restart = restart + o.emitLocked() +} +func (o *appPairOperation) run(p appwire.PairStartParams) { + err, restart := o.perform(p) + switch { + case err == nil: + o.backend.devices.wakePublication() + if !restart { + guard.Go("app/paired-chats", func() { _, _ = o.backend.listChats(true) }) + } + // These events precede done: a desktop may immediately restart on done. + o.backend.publishDevices() + sync, _ := o.backend.SyncStatus() + o.backend.emit.Emit(appwire.EventSyncChanged, appwire.SyncChanged{Sync: sync}) + o.finish("done", "", nil, restart) + case o.ctx.Err() != nil || errors.Is(err, context.Canceled): + o.finish("cancelled", appPairCancelled, nil, false) + case errors.Is(err, pair.ErrRefused): + o.finish("cancelled", appPairDeclined, nil, false) + default: + o.finish("failed", appPairFailure(err), errors.New(appPairFailure(err)), false) + } +} +func (o *appPairOperation) perform(p appwire.PairStartParams) (error, bool) { + dir := o.backend.relayHome() + if reason := cell.UnavailableReason(); reason != "" { + return errors.New(reason), false + } + if p.Name != "" { + if _, err := devname.Set(dir, p.Name); err != nil { + return err, false + } + } + replace := false + if o.joining { + if _, err := identity.Load(dir); err == nil { + fresh, err := identity.Joinable(dir) + if err != nil { + return err, false + } + if !fresh { + a := o.ask(o.ctx, "replace_own_chats", appPairReplaceTitle, appPairReplaceDetail, appPairReplaceOptions()) + if a != "replace" { + return context.Canceled, false + } + replace = true + } + } else if !errors.Is(err, identity.ErrNone) { + return err, false + } + } + via := "" + if p.Mode == "approve_link" { + via = pair.LinkRelay(p.Link) + } + route, err := pairMailboxAt(dir, via) + if err != nil { + return err, false + } + record := func() { + _ = ownDeviceRecord(o.ctx, dir, route) + } + switch p.Mode { + case "offer_code": + grant, err := pairGrantAt(dir, route) + if err != nil { + return err, false + } + name, err := pair.Offer(o.ctx, route, grant, o) + if err != nil { + return err, false + } + record() + o.line(appPairPaired(name)) + return nil, false + case "use_code": + joined, err := pair.Join(o.ctx, route, pairJoiningAt(dir)(replace), p.Code, o) + if err != nil { + return err, false + } + record() + o.line(appPairJoinedLine(joined)) + return nil, !joined.Already + case "request_link": + joined, err := pair.JoinByLink(o.ctx, linkRequests(route), linkJoining(dir)(route, replace), o) + if err != nil && !errors.Is(err, pair.ErrPairedNotRead) { + return err, false + } + record() + if err != nil { + o.line(appPairListUnread) + } else { + o.line(appPairJoined) + } + return nil, !joined.Already + case "approve_link": + ref, err := pair.ReadLink(p.Link) + if err != nil { + return err, false + } + who, err := linkApprover(dir)(route) + if err != nil { + return err, false + } + who.BeforeApprove = func(ctx context.Context) error { + _ = ownDeviceRecord(ctx, dir, route) + return nil + } + asking, err := who.Look(o.ctx, ref) + if err != nil { + return err, false + } + label := pair.ComputerLabel(asking.Name, asking.Platform) + o.line(appPairApproveTitle(label)) + answer := o.ask(o.ctx, "approve", appPairApproveTitle(label), appPairCheck(asking.Check), pairApprovalOptions()) + if answer != "y" { + if o.ctx.Err() != nil { + return o.ctx.Err(), false + } + if err := who.Deny(o.ctx, asking); err != nil { + return err, false + } + return pair.ErrRefused, false + } + if err := who.Approve(o.ctx, asking); err != nil { + return err, false + } + if err := identity.EndSolo(dir); err != nil { + return err, false + } + record() + o.line(appPairPaired(asking.Name)) + return nil, false + } + return errors.New("unknown pairing mode"), false +} +func (o *appPairOperation) Show(code *pair.Code, lines string) { + o.update(func(p *appwire.Pairing) { + p.State = "waiting" + p.Code = code.Shown() + p.ExpiresAt = code.Born().Add(pair.CodeValidFor) + p.Lines = append(p.Lines, appPairCodeLines(code.Shown())...) + }) +} +func (o *appPairOperation) Burned() { o.line(appPairBurned) } +func (o *appPairOperation) Ask(ctx context.Context, label, words string) bool { + o.update(func(p *appwire.Pairing) { p.Words = words }) + return o.ask(ctx, "approve", appPairApproveTitle(label), appPairWords(words), pairApprovalOptions()) == "y" +} +func (o *appPairOperation) Waiting(words string) { + o.update(func(p *appwire.Pairing) { + p.State = "waiting" + p.Words = words + p.Lines = append(p.Lines, appPairWaiting(words)) + }) +} +func (o *appPairOperation) Invited(in pair.Invite) { + o.update(func(p *appwire.Pairing) { + p.State = "waiting" + p.Link = in.Ref.URL() + if in.Via != "" { + p.Link = in.Ref.URLAt(in.Via) + } + p.Words = in.Check + p.ExpiresAt = time.Now().Add(in.ExpiresIn) + p.Lines = append(p.Lines, appPairInviteLines(in.Check, in.ExpiresIn)...) + }) +} diff --git a/cmd/codeaf/app_pair_copy.go b/cmd/codeaf/app_pair_copy.go new file mode 100644 index 0000000000..b70c1dc3c6 --- /dev/null +++ b/cmd/codeaf/app_pair_copy.go @@ -0,0 +1,78 @@ +package main + +import ( + "errors" + "fmt" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/pair" +) + +// Desktop pairing copy belongs here: terminal prompts contain commands and +// keyboard answers that do not describe the app's controls. +const ( + appPairReplaceTitle = "Replace this computer's chats?" + appPairReplaceDetail = "This computer already has its own chats. Pairing replaces them with the chats from your other computer." + appPairCancelled = "Cancelled. Nothing was paired." + appPairDeclined = "Pairing was declined. Nothing was paired." + appPairBurned = "Someone entered a wrong code. Here is a new code." + appPairJoined = "Paired. This computer now shares your chats." + appPairAlready = "This computer is already paired with your chats." + appPairFollowed = "Paired. This computer now shares your current chats and keeps its saved keys." + appPairListUnread = "Paired. Your computers could not be listed yet. Open codeaf again in a moment." + appPairFailed = "This computer could not finish pairing. Try again." +) + +func pairApprovalOptions() []appwire.PairOption { + return []appwire.PairOption{{ID: "y", Label: "Pair"}, {ID: "n", Label: "Don't pair"}} +} + +func appPairReplaceOptions() []appwire.PairOption { + return []appwire.PairOption{{ID: "replace", Label: "Replace this computer's chats"}, {ID: "cancel", Label: "Not now"}} +} +func appPairCodeLines(code string) []string { + return []string{"Pairing shares your chats with another computer.", fmt.Sprintf("On your other computer, choose Pair a computer, then Enter a code from the other computer, and type %s. This code lasts %d minutes.", code, int(pair.CodeValidFor/time.Minute))} +} +func appPairApproveTitle(name string) string { return fmt.Sprintf("Pair with %s?", name) } +func appPairWords(words string) string { + return "Pair only if your other computer shows the same three words: " + words + "." +} +func appPairWaiting(words string) string { + return "Waiting for approval on your other computer. It should show: " + words + "." +} +func appPairPaired(name string) string { + return fmt.Sprintf("Paired with %s. Your chats can continue on both computers.", name) +} +func appPairJoinedLine(j pair.Joined) string { + switch { + case j.Already: + return appPairAlready + case j.Successor: + return appPairFollowed + default: + return appPairJoined + } +} +func appPairInviteLines(check string, expires time.Duration) []string { + return []string{"Open this pairing link on a computer you already use.", "Compare the check number on both computers: " + check + ".", fmt.Sprintf("Waiting for approval. This link lasts %d minutes.", int(expires/time.Minute))} +} +func appPairCheck(check string) string { + return "Pair only if your other computer shows the same check number: " + check + "." +} +func appPairFailure(err error) string { + switch { + case errors.Is(err, pair.ErrCodeDidNotWork): + return "That code did not work, or it timed out. Ask for a new one." + case errors.Is(err, pair.ErrCodeShape): + return "Enter the pairing code shown on your other computer." + case errors.Is(err, pair.ErrSyncOff): + return appSyncOff + case errors.Is(err, pair.ErrSomeoneElse): + return "Someone else used that code. Ask for a new one." + case errors.Is(err, pair.ErrStopped): + return "Your other computer stopped pairing. Start pairing there again." + default: + return appPairFailed + } +} diff --git a/cmd/codeaf/app_pair_e14_test.go b/cmd/codeaf/app_pair_e14_test.go new file mode 100644 index 0000000000..8cfeb7d7d6 --- /dev/null +++ b/cmd/codeaf/app_pair_e14_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "strings" + "testing" +) + +func TestA3PairingInstructionsNameDesktopControls(t *testing.T) { + want := "On your other computer, choose Pair a computer, then Enter a code from the other computer, and type 42-715-302. This code lasts 10 minutes." + if got := appPairCodeLines("42-715-302")[1]; got != want { + t.Fatalf("A3: instructions = %q, want %q", got, want) + } + if got := strings.Join(appPairCodeLines("42-715-302"), " "); strings.Contains(got, "Join another computer") { + t.Fatal("A3: obsolete desktop control") + } +} diff --git a/cmd/codeaf/app_presence_restart_test.go b/cmd/codeaf/app_presence_restart_test.go new file mode 100644 index 0000000000..f58988b6f6 --- /dev/null +++ b/cmd/codeaf/app_presence_restart_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "net" + "net/http" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// P2: reopen the durable relay on the same address with empty presence memory, +// while the apps stay alive. Delay B's reconnect until A has read the fresh relay. +func TestAppRelayRestartP2UnknownUntilPeerReturnsThenActuallyOffline(t *testing.T) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + model := appTwoMessageSource(t, "echo presence") + t.Cleanup(model.Close) + t.Setenv("CODEAF_BASE_URL", model.URL) + r := &outageRelay{t: t, clock: &outageClock{}, connections: map[net.Conn]bool{}} + r.blocked.Store("") + store := t.TempDir() + var db *directory.SQLite + blobs := blobstore.NewMemory() + var reads atomic.Int64 + start := func() { + var err error + db, err = directory.OpenSQLite(filepath.Join(store, "directory.db"), r.clock.Now) + if err != nil { + t.Fatal(err) + } + mux := http.NewServeMux() + auth := reqsign.AuthenticateAt(r.clock.Now) + mux.Handle("/v1/dir/", directory.Handler(auth, func(string) (directory.Directory, error) { return db, nil })) + mux.Handle("/v1/store/", blobstore.Handler(auth, func(string) (blobstore.Store, error) { return blobs, nil })) + r.handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mux.ServeHTTP(w, req) + if req.URL.Path == "/v1/dir/presence" { + reads.Add(1) + } + }) + r.start() + } + start() + t.Cleanup(func() { r.stop(); _ = db.Close() }) + dirA, dirB := outageHomes(t, r.server.URL) + a, b := e2StartApp(t, dirA), e2StartApp(t, dirB) + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, b.w, "agents.create", map[string]any{"workspace": workspace}).Agent + e2Turn(t, b, created.ID, "B stays running") + row := outageChatRow(t, a, "", func(row appwire.ChatRow) bool { return row.Online != nil && *row.Online && !row.Here }) + dev, err := identity.LoadDevice(dirB) + if err != nil { + t.Fatal(err) + } + cert, _ := json.Marshal(dev.Cert) + r.blocked.Store(base64.RawURLEncoding.EncodeToString(cert)) + a.mu.Lock() + outageMark := len(a.events) + a.mu.Unlock() + r.stop() + if err := db.Close(); err != nil { + t.Fatal(err) + } + // Wait for the observing app's ordinary outage expiry, so the restart + // cannot merely retain its old online value and hide a bad refresh. + seen := 0 + a.wait(t, func(f e2Frame) bool { + seen++ + if seen <= outageMark { + return false + } + var changed appwire.DevicesResult + if f.Event != appwire.EventDevicesChanged || json.Unmarshal(f.Data, &changed) != nil { + return false + } + for _, peer := range changed.Devices.Devices { + if peer.ID == dev.ID() && peer.Online == nil { + return true + } + } + return false + }) + a.mu.Lock() + mark := len(a.events) + a.mu.Unlock() + before := reads.Load() + start() + outageWait(t, "P2 A refreshed presence after relay restart", func() bool { return reads.Load() > before }) + unknown := outageChatRow(t, a, row.Cell, func(row appwire.ChatRow) bool { return row.Online == nil }) + if strings.Contains(unknown.Line, "offline") { + t.Fatalf("P2 unknown peer has an offline line: %+v", unknown) + } + r.blocked.Store("") + outageChatRow(t, a, row.Cell, func(row appwire.ChatRow) bool { return row.Online != nil && *row.Online }) + a.mu.Lock() + events := append([]e2Frame(nil), a.events[mark:]...) + a.mu.Unlock() + for _, f := range events { + var devices appwire.DevicesResult + if f.Event == appwire.EventDevicesChanged && json.Unmarshal(f.Data, &devices) == nil { + for _, peer := range devices.Devices.Devices { + if peer.ID == dev.ID() && peer.Online != nil && !*peer.Online { + t.Fatalf("P2 running peer was announced offline after restart: %s", f.Data) + } + } + } + var changed appwire.ChatChanged + if f.Event == appwire.EventChatChanged && json.Unmarshal(f.Data, &changed) == nil && changed.Chat.Cell == row.Cell && (changed.Chat.Online != nil && !*changed.Chat.Online || strings.Contains(changed.Chat.Line, "offline")) { + t.Fatalf("P2 running peer's chat announced offline: %s", f.Data) + } + } + e2Quit(t, b) + r.clock.Advance(directory.Window(directory.DefaultBeat) + time.Second) + outageChatRow(t, a, row.Cell, func(row appwire.ChatRow) bool { return row.Online != nil && !*row.Online }) +} diff --git a/cmd/codeaf/app_reasoning_retry_test.go b/cmd/codeaf/app_reasoning_retry_test.go new file mode 100644 index 0000000000..4efc4ccfd0 --- /dev/null +++ b/cmd/codeaf/app_reasoning_retry_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestAppReasoningRefusalNotRetryableS2(t *testing.T) { + conv := &fakeConversation{script: []session.Event{{Kind: session.EventError, ErrKind: session.TurnErrorProvider, ErrStatus: 400, ErrWords: `"llama3.2:latest" does not support thinking`}}} + var mu sync.Mutex + var opens, releases int + rec := newAppRecorder() + b := newFakeBackend(rec, fakeOpener(conv, &opens, &releases, &mu)) + defer b.Close() + agent, err := mustCreate(t, b, appwire.CreateAsk{Workspace: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if _, err = b.Send(agent.ID, "hello", "s2"); err != nil { + t.Fatal(err) + } + rec.waitFor(t, 5*time.Second, appTurnDone) + appSyncWait(t, "S2 failed turn to settle", func() bool { + idle := false + b.locked(func() { idle = !b.agents[agent.ID].live.running }) + return idle + }) + b.locked(func() { + a := b.agents[agent.ID] + if a.view.Retryable || a.failed != "" { + t.Errorf("S2 permanent refusal retained retryable work: %+v", a.view) + } + }) +} diff --git a/cmd/codeaf/app_reconnect_test.go b/cmd/codeaf/app_reconnect_test.go new file mode 100644 index 0000000000..3e17a7efee --- /dev/null +++ b/cmd/codeaf/app_reconnect_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "bufio" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" +) + +type appReconnectWriter struct { + http.ResponseWriter + connections chan net.Conn +} + +func (w appReconnectWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + c, rw, err := w.ResponseWriter.(http.Hijacker).Hijack() + if err == nil { + w.connections <- c + } + return c, rw, err +} + +func TestAppPresenceP1ReconnectKeepsFreshPeerAndMonotonicLastSeen(t *testing.T) { + appPresenceReconnect(t, "fresh") +} + +func TestAppPresenceP1ReconnectReconcilesMissedOffline(t *testing.T) { + appPresenceReconnect(t, "missed offline") +} + +func TestAppPresenceP1OutageExpiresToUnknown(t *testing.T) { + appPresenceReconnect(t, "outage") +} + +func appPresenceReconnect(t *testing.T, scenario string) { + missedOffline := scenario == "missed offline" + celltest.Available(t) + connections := make(chan net.Conn, 8) + var stale, blocked atomic.Bool + appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/dir/watch" { + if blocked.Load() { + http.Error(w, "reconnect paused", http.StatusServiceUnavailable) + return + } + next.ServeHTTP(appReconnectWriter{w, connections}, r) + return + } + if r.URL.Path == "/v1/dir/list" && stale.Load() { + rec := httptest.NewRecorder() + next.ServeHTTP(rec, r) + var listing directory.Listing + if err := json.Unmarshal(rec.Body.Bytes(), &listing); err != nil { + t.Error(err) + return + } + for id, d := range listing.Devices { + d.LastSeen = 1 + listing.Devices[id] = d + } + for k, v := range rec.Header() { + w.Header()[k] = v + } + w.WriteHeader(rec.Code) + _ = json.NewEncoder(w).Encode(listing) + return + } + next.ServeHTTP(w, r) + }) + }) + a, b := t.TempDir(), t.TempDir() + id, err := identity.Ensure(a) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(b, id, false); err != nil { + t.Fatal(err) + } + sa, sb := appDeviceSync(t, a), appDeviceSync(t, b) + if err = sa.Rename(t.Context(), "A"); err != nil { + t.Fatal(err) + } + if err = sb.Rename(t.Context(), "B"); err != nil { + t.Fatal(err) + } + peer := sb.Follow() + defer peer.Close() + select { + case <-connections: + case <-time.After(10 * time.Second): + t.Fatal("P1 B did not connect") + } + appSyncWait(t, "P1 B's live watch", func() bool { return peer.State().Up }) + backend, rec, wire := appDeviceBackend(t, a) + machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}) + var own net.Conn + select { + case own = <-connections: + case <-time.After(10 * time.Second): + t.Fatal("P1 A did not connect") + } + appDeviceList(t, wire) + appDeviceEvent(t, rec, func(v appwire.Devices) bool { + for _, d := range v.Devices { + if d.ID == sb.Device.ID() { + return d.Online != nil && *d.Online + } + } + return false + }) + rec.mu.Lock() + before := len(rec.events) + rec.mu.Unlock() + if scenario != "fresh" { + blocked.Store(true) + } + if err = own.Close(); err != nil { + t.Fatal(err) + } + if scenario == "outage" { + seen := 0 + rec.waitFor(t, dirwatch.DeadAfter+5*time.Second, func(event string, data any) bool { + i := seen + seen++ + if i < before { + return false + } + if event != appwire.EventDevicesChanged { + return false + } + for _, d := range data.(appwire.DevicesResult).Devices.Devices { + if d.ID == sb.Device.ID() { + return d.Online == nil + } + } + return false + }) + return + } + if missedOffline { + peer.Close() + appSyncWait(t, "P1 B's presence actually lapses", func() bool { + p, err := sb.Dir.Presence(t.Context()) + return err == nil && !p.Devices[sb.Device.ID()].Online + }) + blocked.Store(false) + } else { + peer.Probe() + } + select { + case <-connections: + case <-time.After(30 * time.Second): + t.Fatal("P1 A did not reconnect") + } + if missedOffline { + // No devices.list call repairs A: the watch reconnect owns reconciliation. + appDeviceEvent(t, rec, func(v appwire.Devices) bool { + for _, d := range v.Devices { + if d.ID == sb.Device.ID() { + return d.Online != nil && !*d.Online + } + } + return false + }) + return + } + // A fresh read during the reconnect must agree with B's uninterrupted pings. + fresh := appDeviceList(t, wire) + stale.Store(true) + older := appDeviceList(t, wire) + for i, d := range older.Devices { + if d.LastSeen.Before(fresh.Devices[i].LastSeen) { + t.Fatalf("P1 last_seen moved backwards: %v -> %v", fresh.Devices[i].LastSeen, d.LastSeen) + } + if d.ID == sb.Device.ID() && (d.Online == nil || !*d.Online) { + t.Fatal("P1 reconnect lost fresh peer presence") + } + } + // Serialize the final observer publication before inspecting the wire. + backend.devices.op.Lock() + backend.devices.publish(backend.emit) + backend.devices.op.Unlock() + rec.mu.Lock() + defer rec.mu.Unlock() + last := map[string]time.Time{} + for i, event := range rec.events { + if event != appwire.EventDevicesChanged { + continue + } + for _, d := range rec.data[i].(appwire.DevicesResult).Devices.Devices { + if d.LastSeen.Before(last[d.ID]) { + t.Fatal("P1 emitted last_seen backwards") + } + last[d.ID] = d.LastSeen + if i >= before && d.ID == sb.Device.ID() && d.Online != nil && !*d.Online { + t.Fatal("P1 reconnect emitted fresh B offline") + } + } + } +} diff --git a/cmd/codeaf/app_relay.go b/cmd/codeaf/app_relay.go new file mode 100644 index 0000000000..36045bda2d --- /dev/null +++ b/cmd/codeaf/app_relay.go @@ -0,0 +1,35 @@ +package main + +import ( + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/env" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +func (b *appBackend) relayHome() string { + if b.devices != nil { + return b.devices.home + } + return home.Dir() +} +func (*appBackend) RelayKnown() bool { return true } + +// The resolver owns precedence; these facts name which of its inputs won. +func appRelay(dir string) *appwire.Relay { + resolved := syncsetup.Resolve(dir) + source := "none" + switch { + case env.Get(syncsetup.URLVar) != "": + source = "environment" + case resolved.Hosted: + source = "default" + case resolved.URL != "": + source = "paired" + } + url := resolved.URL + if resolved.Off { + url = "" + } + return &appwire.Relay{URL: url, Source: source, Off: resolved.Off} +} diff --git a/cmd/codeaf/app_relay_test.go b/cmd/codeaf/app_relay_test.go new file mode 100644 index 0000000000..0a3be545f0 --- /dev/null +++ b/cmd/codeaf/app_relay_test.go @@ -0,0 +1,477 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// E-R1: sync.get reports the relay setting without contacting it. +func TestAppRelayER1(t *testing.T) { + _, _, w := appDeviceBackend(t, t.TempDir()) + for _, tc := range []struct { + value, source string + off bool + }{ + {"http://127.0.0.1:1", "environment", false}, {"off", "environment", true}, {"", "default", false}, + } { + t.Setenv(syncsetup.URLVar, tc.value) + var got struct { + Sync struct { + Relay *struct { + URL, Source string + Off bool + } + } + } + raw, _ := json.Marshal(machineCall[any](t, w, "sync.get", struct{}{})) + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if got.Sync.Relay == nil || got.Sync.Relay.Source != tc.source || got.Sync.Relay.Off != tc.off || !tc.off && tc.value != "" && got.Sync.Relay.URL != tc.value { + t.Fatalf("%q: %s", tc.value, raw) + } + } +} + +// E-P3: cancellation is terminal and recoverable on the wire. +func TestAppPairEP3(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + _, rec, w := appDeviceBackend(t, t.TempDir()) + started := machineCall[appwire.PairResult](t, w, "pair.start", appwire.PairStartParams{Mode: "offer_code"}).Pairing + if started.Lines == nil { + t.Fatal("pair lines must be an array") + } + waiting := e2PairWait(t, w, started.ID, "waiting") + if _, err := w.call("pair.start", appwire.PairStartParams{Mode: "offer_code"}); err == nil || err.Code != appwire.CodeBusy { + t.Fatalf("second pairing: %+v", err) + } + if waiting.Code == "" { + t.Fatal("no code") + } + got := machineCall[appwire.PairResult](t, w, "pair.cancel", appwire.IDParams{ID: started.ID}).Pairing + if got.State != "cancelled" { + t.Fatalf("cancel: %+v", got) + } + last := machineCall[appwire.PairGetResult](t, w, "pair.get", struct{}{}).Pairing + if last == nil || last.ID != started.ID || last.State != "cancelled" { + t.Fatalf("last: %+v", last) + } + n := 0 + rec.mu.Lock() + for i, event := range rec.events { + if event == appwire.EventPairChanged && rec.data[i].(appwire.PairResult).Pairing.State == "cancelled" { + n++ + } + } + rec.mu.Unlock() + if n != 1 { + t.Fatalf("final events: %d", n) + } + _, _, other := appDeviceBackend(t, t.TempDir()) + attempt := machineCall[appwire.PairResult](t, other, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: waiting.Code}).Pairing + e2PairWait(t, other, attempt.ID, "failed") + next := machineCall[appwire.PairResult](t, w, "pair.start", appwire.PairStartParams{Mode: "offer_code"}).Pairing + previous := machineCall[appwire.PairGetResult](t, w, "pair.get", appwire.PairGetParams{ID: started.ID}).Pairing + if previous == nil || previous.State != "cancelled" { + t.Fatalf("previous operation lost: %+v", previous) + } + machineCall[appwire.PairResult](t, w, "pair.cancel", appwire.IDParams{ID: next.ID}) + +} + +func e2PairWait(t *testing.T, w *actionWire, id, state string) appwire.Pairing { + t.Helper() + var got appwire.Pairing + appSyncWait(t, "pair "+state, func() bool { + p := machineCall[appwire.PairGetResult](t, w, "pair.get", appwire.PairGetParams{ID: id}).Pairing + if p == nil { + return false + } + got = *p + if got.State == "failed" && state != "failed" { + t.Fatalf("pair failed: %+v", got) + } + return got.State == state + }) + return got +} + +// E-C1: inventory events arrive without another list call; local ids are folder ids. +func TestAppChatsEC1(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + a := appDeviceSync(t, dirA) + b := appDeviceSync(t, dirB) + for _, s := range []*syncsetup.Sync{a, b} { + name := map[string]string{dirA: "A", dirB: "B"}[s.Home] + if _, err := devname.Set(s.Home, name); err != nil { + t.Fatal(err) + } + if err := syncsetup.PutOwnDevice(context.Background(), s.Dir, s.Identity, s.Device, name); err != nil { + t.Fatal(err) + } + } + backendA, _, wa := appDeviceBackend(t, dirA) + _, recB, wb := appDeviceBackend(t, dirB) + first := machineCall[appwire.ChatsResult](t, wb, "chats.list", struct{}{}) + if len(first.Chats) != 0 { + t.Fatal(first) + } + c, err := cell.CreateIn(filepath.Join(dirA, "v3", "projects", "project"), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + localID := "saved-folder" + if err := session.SaveMeta(c.Root, session.Meta{ID: localID, Workspace: t.TempDir(), Title: "A chat"}); err != nil { + t.Fatal(err) + } + // The app id deliberately differs from the directory id. + agent := backendA.agent(localID) + backendA.locked(func() { agent.transcript = filepath.Join(c.Root, cell.TranscriptPath); agent.view.ID = localID }) + title, err := directory.SealName(directory.MetadataKey(id.CellKey()), "A chat") + if err != nil { + t.Fatal(err) + } + if _, err := a.Dir.Create(context.Background(), c.ID, directory.CellInit{Title: title}); err != nil { + t.Fatal(err) + } + recB.waitFor(t, 10*time.Second, func(name string, data any) bool { + if name != appwire.EventChatChanged { + return false + } + row := data.(appwire.ChatChanged).Chat + return row.Cell == c.ID && !row.Here && row.LocalAgent == "" && row.Holder != nil && row.Holder.Name == "A" + }) + rows := machineCall[appwire.ChatsResult](t, wa, "chats.list", struct{}{}) + if len(rows.Chats) != 1 || !rows.Chats[0].Here || rows.Chats[0].LocalAgent != localID { + t.Fatalf("local rows: %+v", rows) + } + + recB.waitFor(t, 10*time.Second, func(name string, data any) bool { + if name != appwire.EventChatChanged { + return false + } + row := data.(appwire.ChatChanged).Chat + return row.Cell == c.ID && row.Online != nil && *row.Online + }) + if err := backendA.Close(); err != nil { + t.Fatal(err) + } + + if err := a.Dir.Archive(context.Background(), c.ID); err != nil { + t.Fatal(err) + } + recB.waitFor(t, 10*time.Second, func(name string, data any) bool { + return name == appwire.EventChatRemoved && data.(appwire.ChatRemoved).Cell == c.ID + }) +} + +// E-P1 and E-P2: explicit replacement precedes any exchange; both pairing sides +// answer the same typed app door, with two homes and a loopback relay. +func TestAppPairEP1EP2L3(t *testing.T) { + for _, used := range []bool{false, true} { + t.Run(map[bool]string{false: "empty", true: "replace"}[used], func(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + // E-P1 starts with a saved chat on the offering computer. + source := filepath.Join(dirA, "v3", "projects", "local", "source") + if err := session.SaveMeta(source, session.Meta{ID: "source", Title: "A saved chat", Workspace: t.TempDir()}); err != nil { + t.Fatal(err) + } + _, recA, wa := appDeviceBackend(t, dirA) + for _, feature := range []string{appwire.FeatureDevicesPair, appwire.FeatureChatsTake, appwire.FeatureSyncRelay} { + if !slices.Contains(wa.hello, feature) { + t.Fatalf("missing feature %s", feature) + } + } + _, recB, wb := appDeviceBackend(t, dirB) + defer assertAppPairCopyL3(t, recA) + defer assertAppPairCopyL3(t, recB) + var kept string + if used { + if _, err := identity.Ensure(dirB); err != nil { + t.Fatal(err) + } + kept = filepath.Join(dirB, "v3", "projects", "local", "saved", "meta.json") + if err := os.MkdirAll(filepath.Dir(kept), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(kept, []byte(`{"id":"saved","title":"keep me"}`), 0600); err != nil { + t.Fatal(err) + } + } + offer := machineCall[appwire.PairResult](t, wa, "pair.start", appwire.PairStartParams{Mode: "offer_code", Name: "A"}).Pairing + waiting := e2PairWait(t, wa, offer.ID, "waiting") + if used { + join := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: waiting.Code, Name: "B"}).Pairing + question := e2PairWait(t, wb, join.ID, "question") + if question.Question.Kind != "replace_own_chats" { + t.Fatal(question) + } + machineCall[appwire.PairResult](t, wb, "pair.answer", appwire.PairAnswerParams{ID: join.ID, Question: question.Question.ID, Option: "cancel"}) + e2PairWait(t, wb, join.ID, "cancelled") + if raw, err := os.ReadFile(kept); err != nil || !strings.Contains(string(raw), "keep me") { + t.Fatalf("own chat lost: %s %v", raw, err) + } + } + join := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: waiting.Code, Name: "B"}).Pairing + if used { + q := e2PairWait(t, wb, join.ID, "question") + machineCall[appwire.PairResult](t, wb, "pair.answer", appwire.PairAnswerParams{ID: join.ID, Question: q.Question.ID, Option: "replace"}) + } + q := e2PairWait(t, wa, offer.ID, "question") + if q.Question.Kind != "approve" || q.Words == "" { + t.Fatal(q) + } + machineCall[appwire.PairResult](t, wa, "pair.answer", appwire.PairAnswerParams{ID: offer.ID, Question: q.Question.ID, Option: "y"}) + e2PairWait(t, wa, offer.ID, "done") + done := e2PairWait(t, wb, join.ID, "done") + if !done.Restart { + t.Fatal("join did not request restart") + } + if got := appDeviceList(t, wa); len(got.Devices) != 2 { + t.Fatalf("A devices: %+v", got) + } + if got := appDeviceList(t, wb); len(got.Devices) != 2 { + t.Fatalf("B devices: %+v", got) + } + t.Setenv(syncsetup.URLVar, "") + relayFact := machineCall[appwire.SyncResult](t, wb, "sync.get", struct{}{}).Sync.Relay + if relayFact == nil || relayFact.Source != "paired" || relayFact.URL != relay.URL { + t.Fatalf("joined relay: %+v", relayFact) + } + }) + } +} + +func assertAppPairCopyL3(t *testing.T, rec *appRecorder) { + t.Helper() + allowed := regexp.MustCompile(`^(Pairing shares your chats with another computer\.|On your other computer, choose Pair a computer, then Enter a code from the other computer, and type [0-9-]+\. This code lasts 10 minutes\.|Waiting for approval on your other computer\. It should show: .+\.|Paired with .+\. Your chats can continue on both computers\.|Paired\. This computer now shares your chats\.|Cancelled\. Nothing was paired\.|Pair with .+\?|Replace this computer's chats\?|This computer already has its own chats\. Pairing replaces them with the chats from your other computer\.|Pair only if your other computer shows the same (three words|check number): .+\.|Pair|Don't pair|Replace this computer's chats|Not now|Open this pairing link on a computer you already use\.|Compare the check number on both computers: .+\.|Waiting for approval\. This link lasts [0-9]+ minutes\.)$`) + rec.mu.Lock() + defer rec.mu.Unlock() + for i, name := range rec.events { + if name != appwire.EventPairChanged { + continue + } + p := rec.data[i].(appwire.PairResult).Pairing + visible := slices.Clone(p.Lines) + if q := p.Question; q != nil { + visible = append(visible, q.Title, q.Detail) + for _, option := range q.Options { + visible = append(visible, option.Label) + if !slices.Contains([]string{"Pair", "Don't pair", "Replace this computer's chats", "Not now"}, option.Label) { + t.Errorf("L3 option: %q", option.Label) + } + } + } + for _, line := range visible { + if line != "" && !allowed.MatchString(line) { + t.Errorf("L3 unexpected app copy: %q", line) + } + + for _, banned := range []string{"codeaf pair", "--replace", "device"} { + if strings.Contains(strings.ToLower(line), banned) { + t.Errorf("L3 terminal copy: %q", line) + } + } + } + } +} + +// E-P4: a wrong code uses the engine's failure sentence and adds no computer. +func TestAppPairEP4(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + _, _, wa := appDeviceBackend(t, t.TempDir()) + _, _, wb := appDeviceBackend(t, t.TempDir()) + offer := machineCall[appwire.PairResult](t, wa, "pair.start", appwire.PairStartParams{Mode: "offer_code"}).Pairing + p := e2PairWait(t, wa, offer.ID, "waiting") + last := p.Code[len(p.Code)-1] + wrong := p.Code[:len(p.Code)-1] + string('0'+(last-'0'+1)%10) + join := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "use_code", Code: wrong}).Pairing + failed := e2PairWait(t, wb, join.ID, "failed") + if failed.Error == nil || failed.Error.Message != "That code did not work, or it timed out. Ask for a new one." { + t.Fatalf("failure: %+v", failed) + } + if got := appDeviceList(t, wa); len(got.Devices) != 1 { + t.Fatal(got) + } + machineCall[appwire.PairResult](t, wa, "pair.cancel", appwire.IDParams{ID: offer.ID}) +} + +// E-P1's link counterparts: the advertisement includes all four modes. +func TestAppPairLinksEP1(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + a, recA, wa := appDeviceBackend(t, t.TempDir()) + _, recB, wb := appDeviceBackend(t, t.TempDir()) + defer assertAppPairCopyL3(t, recA) + defer assertAppPairCopyL3(t, recB) + if _, err := identity.Ensure(a.relayHome()); err != nil { + t.Fatal(err) + } + request := machineCall[appwire.PairResult](t, wb, "pair.start", appwire.PairStartParams{Mode: "request_link", Name: "B"}).Pairing + waiting := e2PairWait(t, wb, request.ID, "waiting") + approve := machineCall[appwire.PairResult](t, wa, "pair.start", appwire.PairStartParams{Mode: "approve_link", Link: waiting.Link}).Pairing + q := e2PairWait(t, wa, approve.ID, "question") + machineCall[appwire.PairResult](t, wa, "pair.answer", appwire.PairAnswerParams{ID: approve.ID, Question: q.Question.ID, Option: "y"}) + e2PairWait(t, wa, approve.ID, "done") + done := e2PairWait(t, wb, request.ID, "done") + if !done.Restart { + t.Fatal("link join needs restart") + } + if got := appDeviceList(t, wb); len(got.Devices) != 2 { + t.Fatal(got) + } +} + +// E-C2: even a completed independent copy must not offer an older home-folder +// chat. The original local agent id differs from the directory copy id. +func TestAppChatsEC2(t *testing.T) { + celltest.Available(t) + relay := e2Relay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + a := appDeviceSync(t, dirA) + backend, _, w := appDeviceBackend(t, dirB) + original := filepath.Join(dirB, "v3", "projects", "old", "old-chat") + userHome, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + if err := session.SaveMeta(original, session.Meta{ID: "old-chat", Workspace: userHome}); err != nil { + t.Fatal(err) + } + copied := cell.CopyRoot(original) + c, err := cell.CreateIn(filepath.Dir(copied), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(c.Root, copied); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(copied, cell.StateDir, "copy-ready"), nil, 0600); err != nil { + t.Fatal(err) + } + agent := backend.agent("old-chat") + backend.locked(func() { agent.transcript = filepath.Join(original, "transcript.jsonl") }) + remote := filepath.Base(copied) + if _, err := a.Dir.Create(context.Background(), remote, directory.CellInit{Head: "saved-head"}); err != nil { + t.Fatal(err) + } + rows := machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}) + if len(rows.Chats) != 1 || rows.Chats[0].CanTake || rows.Chats[0].LocalAgent != "old-chat" { + t.Fatalf("older chat: %+v", rows) + } +} + +// E-P2: joining while an ordinary app turn runs is refused before pairing starts. +func TestAppPairLiveWorkBusyEP2(t *testing.T) { + celltest.Available(t) + t.Setenv(syncsetup.URLVar, "off") + hold := make(chan struct{}) + conv := &fakeConversation{hold: hold} + var mu sync.Mutex + opens, releases := 0, 0 + backend := newFakeBackend(newAppRecorder(), fakeOpener(conv, &opens, &releases, &mu)) + defer backend.Close() + defer close(hold) + agent, err := mustCreate(t, backend, appwire.CreateAsk{Workspace: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if _, err := backend.Send(agent.ID, "working", "busy"); err != nil { + t.Fatal(err) + } + w := newActionWire(t, backend) + _, failure := w.call("pair.start", appwire.PairStartParams{Mode: "use_code", Code: "42-123-456"}) + if failure == nil || failure.Code != appwire.CodeBusy || failure.Message != appJoinBusy { + t.Fatalf("join during work: %+v", failure) + } + last := machineCall[appwire.PairGetResult](t, w, "pair.get", struct{}{}) + if last.Pairing != nil { + t.Fatal("busy join started a pairing") + } +} + +// E-C1: a blocked inventory request cannot hold up the ordinary local list. +func TestAppChatsLocalListDoesNotWaitEC1(t *testing.T) { + celltest.Available(t) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/dir/list" { + once.Do(func() { close(entered) }) + select { + case <-release: + case <-r.Context().Done(): + return + } + } + next.ServeHTTP(w, r) + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + dir := t.TempDir() + if _, err := identity.Ensure(dir); err != nil { + t.Fatal(err) + } + backend, _, _ := appDeviceBackend(t, dir) + t.Cleanup(func() { close(release) }) + done := make(chan error, 1) + go func() { _, err := backend.ListAgents(""); done <- err }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("inventory read did not start") + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("local list waited for the blocked relay") + } +} diff --git a/cmd/codeaf/app_settings_sync_test.go b/cmd/codeaf/app_settings_sync_test.go new file mode 100644 index 0000000000..65737eb322 --- /dev/null +++ b/cmd/codeaf/app_settings_sync_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/effort" + "github.com/Agent-Field/codeaf/internal/identity" +) + +func TestAppModelSyncM5SkippedChoiceIsSilent(t *testing.T) { + celltest.Available(t) + relay := e2WireRelay(t) + t.Setenv("CODEAF_SYNC_URL", relay.URL) + a, b := t.TempDir(), t.TempDir() + id, err := identity.Ensure(a) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(b, id, false); err != nil { + t.Fatal(err) + } + sa, sb := appDeviceSync(t, a), appDeviceSync(t, b) + backend, rec, _ := appDeviceBackend(t, b) + if err = config.WriteChatModel(b, "openrouter/deepseek/deepseek-chat"); err != nil { + t.Fatal(err) + } + if err = sb.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + if err = config.WriteChatModel(a, "ollama/llama3.2:latest"); err != nil { + t.Fatal(err) + } + if err = sa.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + rec.mu.Lock() + before := len(rec.events) + rec.mu.Unlock() + if err = sb.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + settings, err := backend.Settings() + if err != nil { + t.Fatal(err) + } + if settings.DefaultModel != "openrouter/deepseek/deepseek-chat" { + t.Fatalf("M5 default changed: %+v", settings) + } + rec.mu.Lock() + defer rec.mu.Unlock() + for _, event := range rec.events[before:] { + if event == appwire.EventSettingsChanged { + t.Fatal("M5 skipped-only round emitted settings.changed") + } + } +} + +func TestAppDevicesF3CompleteStartupAndNoDuplicates(t *testing.T) { + celltest.Available(t) + appDevicesRelay(t) + a, b := t.TempDir(), t.TempDir() + id, err := identity.Ensure(a) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(b, id, false); err != nil { + t.Fatal(err) + } + sa, sb := appDeviceSync(t, a), appDeviceSync(t, b) + if err = sa.Rename(t.Context(), "desk"); err != nil { + t.Fatal(err) + } + if err = sb.Rename(t.Context(), "laptop"); err != nil { + t.Fatal(err) + } + if _, err = devname.Set(a, "desk"); err != nil { + t.Fatal(err) + } + backend, rec, wire := appDeviceBackend(t, a) + // Chat startup and device publication both compare the existing self record. + if err = sa.Rename(t.Context(), "desk"); err != nil { + t.Fatal(err) + } + if _, err = backend.RenameDevice("desk"); err != nil { + t.Fatal(err) + } + appDeviceList(t, wire) + appDeviceEvent(t, rec, func(v appwire.Devices) bool { return len(v.Devices) == 2 }) + for range 3 { + appDeviceList(t, wire) + if _, err = backend.RenameDevice("desk"); err != nil { + t.Fatal(err) + } + } + backend.publishDevices() + rec.mu.Lock() + defer rec.mu.Unlock() + n := 0 + var previous *appwire.DevicesResult + for i, event := range rec.events { + if event == appwire.EventDevicesChanged { + n++ + next := rec.data[i].(appwire.DevicesResult) + if len(next.Devices.Devices) != 2 { + t.Fatal("F3 emitted a partial roster") + } + if previous != nil && reflect.DeepEqual(*previous, next) { + t.Fatal("A1/F3 repeated an unchanged roster") + } + previous = &next + } + } + // The engine now opens its shared watch at startup. Learning self presence + // is a real change; both its initial roster and that change stay complete. + if n < 1 { + t.Fatal("A1/F3 startup did not emit its complete roster") + } +} + +func TestAppSyncedSettingsS1(t *testing.T) { + celltest.Available(t) + relay := e2WireRelay(t) + t.Setenv("CODEAF_SYNC_URL", relay.URL) + a, b := t.TempDir(), t.TempDir() + id, err := identity.Ensure(a) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(b, id, false); err != nil { + t.Fatal(err) + } + sa, sb := appDeviceSync(t, a), appDeviceSync(t, b) + backend, rec, wire := appDeviceBackend(t, b) + _ = machineCall[appwire.Settings](t, wire, "settings.get", struct{}{}) + if err = config.WriteDefaultEffort(a, effort.High); err != nil { + t.Fatal(err) + } + if err = sa.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + if err = sb.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + settings, err := backend.Settings() + if err != nil { + t.Fatal(err) + } + if settings.Effort != "high" { + t.Fatalf("S1 preference did not arrive: %+v", settings) + } + rec.mu.Lock() + defer rec.mu.Unlock() + var events []appwire.SettingsChanged + for i, e := range rec.events { + if e == appwire.EventSettingsChanged { + events = append(events, rec.data[i].(appwire.SettingsChanged)) + } + } + if len(events) != 1 || !reflect.DeepEqual(events[0].Settings, settings) { + t.Fatalf("S1 events=%+v, want one full settings object %+v", events, settings) + } + rec.mu.Unlock() + if err = sb.SyncState(t.Context()); err != nil { + t.Fatal(err) + } + rec.mu.Lock() + count := 0 + for _, e := range rec.events { + if e == appwire.EventSettingsChanged { + count++ + } + } + if count != 1 { + t.Fatalf("S1 unchanged round emitted again: %d", count) + } +} + +func TestAppDevicesChangedS3(t *testing.T) { + celltest.Available(t) + appDevicesRelay(t) + backend, rec, wire := appDeviceBackend(t, t.TempDir()) + initial := appDeviceList(t, wire) + for range 3 { + if _, err := backend.RenameDevice("S3 computer"); err != nil { + t.Fatal(err) + } + } + appDeviceEvent(t, rec, func(v appwire.Devices) bool { return v.Devices[0].Name == "S3 computer" }) + rec.mu.Lock() + defer rec.mu.Unlock() + previous := initial + for i, e := range rec.events { + if e != appwire.EventDevicesChanged { + continue + } + view := rec.data[i].(appwire.DevicesResult).Devices + if reflect.DeepEqual(previous, view) { + t.Fatalf("S3 duplicate devices.changed: %+v", view) + } + previous = view + } +} diff --git a/cmd/codeaf/app_staging_test.go b/cmd/codeaf/app_staging_test.go new file mode 100644 index 0000000000..d79bea3c2a --- /dev/null +++ b/cmd/codeaf/app_staging_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/session" + teamstore "github.com/Agent-Field/codeaf/internal/teams" +) + +// S6 gives the app a fetched staging tree with the sender's working presence. +// Listing, startup recovery and lazy opening must never admit it as an agent. +func TestAppStagingAdmissionS6(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_HOME", dir) + c, err := cell.CreateIn(filepath.Join(dir, "v3", "projects", "copies"), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if err = session.SaveMeta(c.Root, session.Meta{ID: c.ID, Title: "Staging only", LastUserAt: time.Now(), Workspace: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(c.Root, cell.TranscriptPath), []byte("{}\n"), 0600); err != nil { + t.Fatal(err) + } + presence, err := json.Marshal(session.SessionPresence{State: session.PresenceWorking, PID: 123, UpdatedAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(c.Root, "presence.json"), presence, 0600); err != nil { + t.Fatal(err) + } + stage := c.Root + handoff.StagingSuffix + if err = os.Rename(c.Root, stage); err != nil { + t.Fatal(err) + } + b, _, _ := appDeviceBackend(t, dir) + rows, err := b.ListAgents("") + if err != nil { + t.Fatal(err) + } + if len(rows) != 0 { + t.Errorf("S6 staging tree reached agents.list: %+v", rows) + } + run := &appCrewRun{} + if got := b.lazyCandidates(run, appCrewRead{file: &teamstore.File{}}, true); len(got) != 0 { + t.Errorf("S6 staging tree reached lazy open: %+v", got) + } + b.snapshotCloseClaims() + if len(b.closeRecovery.claims) != 0 || appCloseConversationExists(stage) { + t.Fatal("S6 staging tree reached close recovery") + } +} diff --git a/cmd/codeaf/app_take.go b/cmd/codeaf/app_take.go new file mode 100644 index 0000000000..1d341a555c --- /dev/null +++ b/cmd/codeaf/app_take.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +func appTakeError(err error) error { + if errors.Is(err, directory.ErrLeaseHeld) || errors.Is(err, directory.ErrFenceStale) { + return &appwire.Error{Code: appwire.CodeConflict, Message: chatlist.LostRace, Detail: appwire.HolderChanged{Kind: "holder_changed"}} + } + if errors.Is(err, directory.ErrNotFound) { + return appwire.Errorf(appwire.CodeNotFound, "that chat is not available") + } + var wire *appwire.Error + if errors.As(err, &wire) { + if wire.Code != appwire.CodeEngine { + return wire + } + message := appTakeWords(wire.Message) + if strings.HasPrefix(message, "This chat used ") { + return &appwire.Error{Code: wire.Code, Message: message, Detail: appTakeDetail(err)} + } + } + var missing *continuedModelUnavailable + if errors.As(err, &missing) { + return &appwire.Error{Code: appwire.CodeEngine, Message: missing.Error(), Detail: fmt.Sprintf("model %q, provider %q: no usable local default model", missing.model, missing.provider)} + } + return &appwire.Error{Code: appwire.CodeEngine, Message: appTakeFailed, Detail: appTakeDetail(err)} +} + +func appTakeDetail(err error) any { + var wire *appwire.Error + detail := err.Error() + if errors.As(err, &wire) && wire.Detail != nil { + if words, ok := wire.Detail.(string); ok { + detail = words + } else { + return wire.Detail + } + } + return appTakeWords(detail) +} + +// Loopback and app codecs can each have added their own engine label. +func appTakeWords(words string) string { + const prefix = appwire.CodeEngine + ": " + for strings.HasPrefix(words, prefix) { + words = strings.TrimPrefix(words, prefix) + } + return words +} +func (b *appBackend) TakeAgent(p appwire.TakeParams) (appwire.Agent, error) { + if p.Cell == "" || p.Seen.Device == "" { + return appwire.Agent{}, appwire.Errorf(appwire.CodeBadParams, "a chat and its seen computer are required") + } + if !cell.ValidID(p.Cell) { + return appwire.Agent{}, appwire.Errorf(appwire.CodeNotFound, "that chat is not available") + } + if err := b.pairingWorkRefusal(); err != nil { + return appwire.Agent{}, err + } + b.chats.take.Lock() + defer b.chats.take.Unlock() + s, ok, err := syncsetup.Open(b.relayHome()) + if err != nil { + return appwire.Agent{}, appTakeError(err) + } + if !ok { + return appwire.Agent{}, appwire.Errorf(appwire.CodeEngine, appSyncOff) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + guard.Go("app/take-cancel", func() { + select { + case <-b.left: + cancel() + case <-ctx.Done(): + } + }) + // Check even the already-here fast path against what the person saw. + view, err := s.Dir.Cell(ctx, p.Cell) + if err != nil { + return appwire.Agent{}, appTakeError(err) + } + seen := &directory.LeaseExpectation{Device: p.Seen.Device, Fence: p.Seen.Fence, Head: p.Seen.Head} + if !seen.Matches(view.Cell) { + return appwire.Agent{}, appTakeError(directory.ErrLeaseHeld) + } + if view.Cell.Archived || view.Cell.ParentCell != "" { + return appwire.Agent{}, appwire.Errorf(appwire.CodeNotFound, "that chat is not available") + } + locals := b.localChats(s) + var existing *appAgent + if local, found := locals[p.Cell]; found { + if !local.copyable { + return appwire.Agent{}, appwire.Errorf(appwire.CodeEngine, "This older chat stays on this computer.") + } + b.locked(func() { existing = b.agents[local.agent] }) + } + if existing != nil { + releaseForTake := func() error { + existing.gate.Lock() + defer existing.gate.Unlock() + busy := false + b.locked(func() { + busy = existing.opening || existing.starting || existing.live != nil && (existing.live.running || existing.live.needsYou()) + }) + if busy { + return appwire.Errorf(appwire.CodeBusy, "Finish this chat's work before continuing it here.") + } + if view.Cell.Lease.Device != s.Device.ID() { + b.release(existing) + } + return nil + } + if err := releaseForTake(); err != nil { + return appwire.Agent{}, err + } + } + got, err := chatContinuer(s, nil).TakeSeen(ctx, p.Cell, seen) + if err != nil { + return appwire.Agent{}, appTakeError(err) + } + // The continuer's already-here shortcut may observe a local acquisition + // that happened after our preflight read. It must not accept older consent. + if got.AlreadyHere && (p.Seen.Device != s.Device.ID() || !seen.Matches(directory.Cell{Head: got.Taken.Driving.Head, Lease: directory.Lease{Device: s.Device.ID(), Fence: got.Taken.Driving.Fence}})) { + return appwire.Agent{}, appTakeError(directory.ErrLeaseHeld) + } + root := got.Taken.Cell.Root + meta, err := session.LoadMeta(root) + if err != nil { + return appwire.Agent{}, appTakeError(err) + } + transcript := filepath.Join(root, cell.TranscriptPath) + id := appAgentID(transcript) + if existing != nil { + b.locked(func() { id = existing.view.ID }) + } + b.remember(session.SessionRow{ID: id, Dir: root, Transcript: transcript, Workspace: meta.Workspace, Title: meta.Title, Created: meta.Created, Model: meta.Model, ModelIdentity: meta.ModelIdentity}) + a := b.agent(id) + b.locked(func() { + a.transcript, a.windowMoved = transcript, false + if !got.AlreadyHere { + // Installation replaced the saved conversation. A resting local + // agent may still remember the model from before another computer + // continued it; only the newly installed model belongs to this take. + a.view.Model, a.view.ModelIdentity = meta.Model, meta.ModelIdentity + } + }) + location := &appwire.Location{Here: true, Cell: p.Cell, Seen: &appwire.Seen{Device: s.Device.ID(), Fence: got.Taken.Driving.Fence, Head: got.Taken.Driving.Head}, Holder: &appwire.LocationHolder{ID: s.Device.ID(), Name: got.Device}} + if service, ok := s.Dir.(interface{ HeadTakes() bool }); ok && !service.HeadTakes() { + location.Seen.Head = "" + } + b.located(a, location) + agent, err := b.OpenAgent(id) + if err != nil { + return appwire.Agent{}, appTakeError(err) + } + // A local display replica may still be catching up to the acquired receipt. + if agent.Location != nil { + fresh := *agent.Location + fresh.Cell = p.Cell + fresh.Here = true + fresh.Seen = location.Seen + fresh.Holder = location.Holder + fresh.CanTake = false + fresh.Line = "" + location = &fresh + } + b.located(a, location) + agent.Location = location + // Acquisition may have been observed before installation and admission. + // Reconcile the local id even when no later directory version will arrive. + reconcileInventory := func() error { + b.chats.op.Lock() + defer b.chats.op.Unlock() + _, err := b.readChats(ctx, s, b.chats.feed, true) + return err + } + err = reconcileInventory() + if err != nil { + // The take has completed. Inventory recovery belongs to its follower. + b.startChatInventory() + } + return agent, nil +} diff --git a/cmd/codeaf/app_take_live_test.go b/cmd/codeaf/app_take_live_test.go new file mode 100644 index 0000000000..abefd051da --- /dev/null +++ b/cmd/codeaf/app_take_live_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// L4's error envelope is observable through agents.take. The relay injects an +// install failure by making the copy parent unwritable just after acquisition. +func TestAppTakeInstallFailureMessageL4(t *testing.T) { + celltest.RequireEngine(t) + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("failure injection needs ordinary file permissions") + } + dirA, dirB := t.TempDir(), t.TempDir() + parent := filepath.Join(dirB, "v3", "projects", "copies") + var injected atomic.Bool + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + if r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/acquire") { + if err := os.Chmod(parent, 0500); err != nil { + t.Errorf("inject install failure: %v", err) + } else { + injected.Store(true) + } + } + }) + }) + t.Setenv(syncsetup.URLVar, relay.URL) + t.Cleanup(func() { _ = os.Chmod(parent, 0700) }) + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + t.Setenv("CODEAF_HOME", dirA) + c, err := cell.CreateIn(filepath.Join(dirA, "v3", "projects", "local"), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + if err = session.SaveMeta(c.Root, session.Meta{ID: c.ID, Title: "Source chat", Workspace: work}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(c.Root, cell.TranscriptPath), nil, 0600); err != nil { + t.Fatal(err) + } + eng := cellstore.Engine{DataRoot: filepath.Join(dirA, "v3", "stores"), Workspace: work} + if _, err = eng.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + a := appDeviceSync(t, dirA) + if err = a.PublishCopy(t.Context(), eng, c, "Mac", "Source chat"); err != nil { + t.Fatal(err) + } + _, _, w := appDeviceBackend(t, dirB) + rows := machineCall[appwire.ChatsResult](t, w, "chats.list", struct{}{}).Chats + if len(rows) != 1 { + t.Fatal(rows) + } + _, failure := w.call("agents.take", appwire.TakeParams{Cell: c.ID, Seen: rows[0].Seen}) + _ = os.Chmod(parent, 0700) + if !injected.Load() || failure == nil { + t.Fatalf("L4 injection=%v error=%+v", injected.Load(), failure) + } + after, err := a.Dir.Cell(t.Context(), c.ID) + if err != nil { + t.Fatal(err) + } + _, admitted := a.Dir.Heartbeat(t.Context(), c.ID, directory.Beat{Fence: rows[0].Seen.Fence}) + t.Logf("L4 recovery evidence: before=%+v after=%+v source heartbeat=%v", rows[0].Seen, after.Cell.Lease, admitted) + if failure.Code != appwire.CodeEngine || failure.Message != "This chat could not be moved to this computer." || !strings.Contains(string(e2JSON(failure.Detail)), "permission denied") { + t.Fatalf("L4 error envelope: %+v", failure) + } +} diff --git a/cmd/codeaf/app_take_model_test.go b/cmd/codeaf/app_take_model_test.go new file mode 100644 index 0000000000..847d7a7875 --- /dev/null +++ b/cmd/codeaf/app_take_model_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/config" +) + +// E10 T1/T2/T3: moving a chat and setting up its model are separate actions. +// Both computers use real app commands; only the provider is scripted. +func TestAppTakeModelT1T2T3N1NoUsableDefault(t *testing.T) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + model := appTwoMessageSource(t, "echo model-ready") + t.Cleanup(model.Close) + var calls atomic.Int64 + var usedDefault atomic.Bool + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + calls.Add(1) + raw, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(raw)) + var request struct{ Model string } + _ = json.Unmarshal(raw, &request) + if request.Model == "fixture/default" { + usedDefault.Store(true) + } + if strings.HasPrefix(request.Model, "ollama/") { + http.Error(w, "unavailable model reached the router", http.StatusBadRequest) + return + } + } + model.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(endpoint.Close) + t.Setenv("CODEAF_BASE_URL", endpoint.URL) + relay := e2WireRelay(t) + dirA, dirB := outageHomes(t, relay.URL) + if err := config.WriteSources(dirA, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: endpoint.URL}}); err != nil { + t.Fatal(err) + } + for _, dir := range []string{dirA, dirB} { + if err := config.WriteChatModel(dir, "ollama/llama3.2:latest"); err != nil { + t.Fatal(err) + } + } + a, b := e2StartApp(t, dirA), e2StartApp(t, dirB) + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, a.w, "agents.create", map[string]any{"workspace": workspace, "model": "ollama/llama3.2:latest"}).Agent + e2Turn(t, a, created.ID, "from A") + e2Quit(t, a) + row := outageChatRow(t, b, "", func(r appwire.ChatRow) bool { return r.Title != "" && r.CanTake }) + taken := machineCall[appwire.TakeResult](t, b.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}).Agent + if taken.Location == nil || !taken.Location.Here { + t.Fatalf("T1 take did not complete here: %+v", taken) + } + b.wait(t, func(f e2Frame) bool { + var changed appwire.ChatChanged + return f.Event == appwire.EventChatChanged && json.Unmarshal(f.Data, &changed) == nil && changed.Chat.Cell == row.Cell && changed.Chat.Here && changed.Chat.LocalAgent == taken.ID + }) + local := outageChatRow(t, b, row.Cell, func(r appwire.ChatRow) bool { return r.Here && r.LocalAgent == taken.ID }) + if local.Seen.Device != taken.Location.Holder.ID || local.Seen.Fence != taken.Location.Seen.Fence { + t.Fatalf("T2 row/agent disagree: %+v / %+v", local, taken.Location) + } + before := calls.Load() + turn := machineCall[struct { + Turn uint64 `json:"turn"` + }](t, b.w, "agents.send", map[string]any{"id": taken.ID, "text": "before setup", "client_id": "before-setup"}).Turn + const reason = "This chat used llama3.2 (Ollama), which isn't set up on this computer. Set up a model in Settings, then send again." + b.wait(t, func(f e2Frame) bool { + var done appwire.TurnDone + if f.Event != appwire.EventTurnDone || json.Unmarshal(f.Data, &done) != nil || done.Turn != turn { + return false + } + if done.OK || done.Error != reason || done.ErrorKind == "provider" { + t.Fatalf("T1/T3 first send: %s", f.Data) + } + return true + }) + if calls.Load() != before { + t.Fatal("T1 unavailable model reached a provider") + } + // The ordinary profile-model door is what Settings uses for this choice. + machineCall[any](t, b.w, "settings.set", map[string]any{"default_model": "fixture/default"}) + e2Turn(t, b, taken.ID, "after setup") + e2Turn(t, b, taken.ID, "second ready turn") + if !usedDefault.Load() { + t.Fatal("T1 the next send did not use B's usable default") + } + b.mu.Lock() + defer b.mu.Unlock() + failures, notices, switches := 0, 0, 0 + for _, f := range b.events { + var added appwire.EntryAdded + if f.Event != appwire.EventEntryAdded || json.Unmarshal(f.Data, &added) != nil || added.Entry.Agent != taken.ID { + continue + } + if added.Entry.Text == reason { + failures++ + } + if added.Entry.NoticeKind == "model_switch" { + switches++ + } + if strings.Contains(added.Entry.Text, "isn't set up here") && strings.Contains(added.Entry.Text, "fixture/default") { + notices++ + } + } + if failures != 1 || notices != 1 || switches != 1 { + t.Fatalf("T1/T3/N1 model reasons = %d, specific notices = %d, all model notices = %d; want one each", failures, notices, switches) + } +} + +func TestAppTakeT3ActionableMessageAndTechnicalDetail(t *testing.T) { + missing := &continuedModelUnavailable{model: "ollama/llama3.2:latest", provider: "ollama"} + for _, cause := range []error{ + fmt.Errorf("engine: %w", missing), + &appwire.Error{Code: appwire.CodeEngine, Message: missing.Error(), Detail: "engine: engine: model ollama/llama3.2:latest: no usable default"}, + &appwire.Error{Code: appwire.CodeEngine, Message: "engine: engine: " + missing.Error(), Detail: "engine: model ollama/llama3.2:latest: no usable default"}, + } { + var wire *appwire.Error + if !errors.As(appTakeError(cause), &wire) || wire.Message != missing.Error() { + t.Fatalf("T3 actionable message lost: %+v", wire) + } + detail, _ := wire.Detail.(string) + if !strings.Contains(detail, "ollama/llama3.2:latest") || strings.Contains(detail, "engine:") { + t.Fatalf("T3 technical detail lost or prefixed: %q", detail) + } + } +} diff --git a/cmd/codeaf/app_take_recovery_test.go b/cmd/codeaf/app_take_recovery_test.go new file mode 100644 index 0000000000..033366ca28 --- /dev/null +++ b/cmd/codeaf/app_take_recovery_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// S5 uses L4's post-acquisition permission failure with a real open source +// conversation, then submits through the source app's ordinary send door. +func TestAppTakeInstallFailureS5(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("L4 injection needs ordinary file permissions") + } + celltest.RequireEngine(t) + dirB := t.TempDir() + parent := filepath.Join(dirB, "v3", "projects", "copies") + var armed, injected atomic.Bool + relay := e2WireRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + if r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/acquire") && armed.CompareAndSwap(true, false) { + if err := os.Chmod(parent, 0500); err != nil { + t.Errorf("S5 injection: %v", err) + } else { + injected.Store(true) + } + } + }) + }) + t.Cleanup(func() { _ = os.Chmod(parent, 0700) }) + source, rec, _, workspace := appSyncBackend(t) + t.Setenv(syncsetup.URLVar, relay.URL) + t.Setenv("CODEAF_MODEL", "test/model") + dirA := home.Dir() + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + c, err := cell.CreateIn(filepath.Join(dirA, "v3", "projects", "local"), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + private := (session.Place{Dir: c.Root, Owned: true}).Work() + if err = os.MkdirAll(private, 0700); err != nil { + t.Fatal(err) + } + if err = session.SaveMeta(c.Root, session.Meta{ID: c.ID, Title: "Source chat", Workspace: workspace, Owned: true, LastUserAt: time.Now()}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(c.Root, cell.TranscriptPath), []byte(`{"type":"message","role":"user","content":"seed"}`+"\n"), 0600); err != nil { + t.Fatal(err) + } + eng := cellstore.Engine{DataRoot: filepath.Join(dirA, "v3", "stores"), Workspace: private} + if _, err = eng.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + owner := appDeviceSync(t, dirA) + if err = owner.PublishCopy(t.Context(), eng, c, "Source S5", "Source chat"); err != nil { + t.Fatal(err) + } + agent, err := source.OpenAgent(c.ID) + if err != nil { + t.Fatal(err) + } + source.drives.mu.Lock() + drive := source.drives.drive[c.ID] + source.drives.mu.Unlock() + if drive == nil { + t.Fatal("S5 source has no drive") + } + if err = drive.Gate(); err != nil { + t.Fatal(err) + } + if _, err = identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + _, _, wire := appDeviceBackend(t, dirB) + rows := machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats + if len(rows) != 1 { + t.Fatalf("S5 rows before take: %+v", rows) + } + armed.Store(true) + _, failure := wire.call("agents.take", appwire.TakeParams{Cell: c.ID, Seen: rows[0].Seen}) + _ = os.Chmod(parent, 0700) + if !injected.Load() || failure == nil || failure.Message != appTakeFailed { + t.Fatalf("S5 injection=%v failure=%+v", injected.Load(), failure) + } + after, err := owner.Dir.Cell(t.Context(), c.ID) + if err != nil { + t.Fatal(err) + } + if after.Cell.Lease.Expires != 0 || after.Cell.Lease.Fence != rows[0].Seen.Fence+1 { + t.Fatalf("S5 failed take lease: %+v", after.Cell.Lease) + } + rows = machineCall[appwire.ChatsResult](t, wire, "chats.list", struct{}{}).Chats + if len(rows) != 1 || rows[0].Here || rows[0].LocalAgent != "" || !rows[0].CanTake || rows[0].Holder == nil || rows[0].Holder.ID != after.Cell.Lease.Device || strings.Contains(rows[0].Line, "open on") { + t.Fatalf("S5 failed taker row: %+v", rows) + } + t.Setenv(home.EnvVar, dirA) + if err = drive.started().CheckLease(t.Context()); err != nil { + t.Fatal(err) + } + var view appwire.Agent + source.locked(func() { view = source.agents[agent.ID].view }) + if view.Location != nil && strings.Contains(view.Location.Line, "continued") { + t.Fatalf("S5 released chat claims a continuation: %+v", view.Location) + } + turn, err := source.Send(agent.ID, "continue after the failed move", "s5-recover") + if err != nil { + t.Fatalf("S5 source send: %v", err) + } + appTurnDoneFor(t, rec, turn) + recovered, err := owner.Dir.Cell(t.Context(), c.ID) + if err != nil { + t.Fatal(err) + } + if recovered.Cell.Lease.Device != owner.Device.ID() || recovered.Cell.Lease.Fence <= after.Cell.Lease.Fence || recovered.Cell.Lease.Expires <= recovered.Now { + t.Fatalf("S5 source did not reacquire: %+v", recovered.Cell.Lease) + } +} diff --git a/cmd/codeaf/app_take_test.go b/cmd/codeaf/app_take_test.go new file mode 100644 index 0000000000..25a712a33f --- /dev/null +++ b/cmd/codeaf/app_take_test.go @@ -0,0 +1,561 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/devname" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/env" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/pairbox" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +type e2Frame struct { + Event string `json:"event"` + Data json.RawMessage `json:"data"` +} +type e2App struct { + w *actionWire + mu sync.Mutex + events []e2Frame + more chan struct{} + done chan error + log string +} + +// e2TempDir retains native evidence and avoids deleting state still held by +// the shared daemon, which outlives the app processes until its idle shutdown. +func e2TempDir(t *testing.T) string { + t.Helper() + if os.Getenv("CODEAF_E2_KEEP_FAILED") != "1" { + return resolvedTempDir(t) + } + dir, err := os.MkdirTemp("", "e2-app-state-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + t.Logf("retained app state: %s", dir) + }) + real, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + return real +} + +func e2StartApp(t *testing.T, dir string, daemon ...bool) *e2App { + t.Helper() + return e2StartAppWithSetup(t, dir, func(app *e2App) { + machineCall[any](t, app.w, "setup.get", struct{}{}) + machineCall[appwire.Empty](t, app.w, "keys.set", map[string]any{"provider": "openrouter", "key": "test-key"}) + machineCall[appwire.ChatsResult](t, app.w, "chats.list", struct{}{}) + }, daemon...) +} + +func e2StartAppWithSetup(t *testing.T, dir string, setup func(*e2App), daemon ...bool) *e2App { + t.Helper() + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(exe, "app", "--home", dir) + cmd.Env = append(env.EnvironWithout(home.EnvVar, config.ProfileDirEnv), carriedChildEnv+"=real", "CODEAF_ENGINE_DAEMON=0") + if len(daemon) > 0 && daemon[0] { + cmd.Env = append(cmd.Env, "CODEAF_ENGINE_DAEMON=1") + } + inR, inW := io.Pipe() + cmd.Stdin = inR + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + f, err := os.CreateTemp(os.TempDir(), "e2-app-*.log") + log := "" + if f != nil { + log = f.Name() + } + if err != nil { + t.Fatal(err) + } + cmd.Stderr = f + t.Logf("app %s log: %s", dir, log) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + app := &e2App{w: &actionWire{t: t, in: inW, replyWithin: 2 * time.Minute, waiting: map[int]chan actionFrame{}}, more: make(chan struct{}, 1), done: make(chan error, 1), log: log} + go func() { app.done <- cmd.Wait(); _ = f.Close(); _ = inR.Close() }() + t.Cleanup(func() { + app.w.close() + select { + case err := <-app.done: + if err != nil { + raw, _ := os.ReadFile(log) + t.Errorf("app exit: %v\n%s", err, raw) + } + case <-time.After(90 * time.Second): + _ = cmd.Process.Kill() + t.Error("app did not exit") + } + if t.Failed() { + raw, err := os.ReadFile(log) + t.Logf("app %s log: %s (read: %v)\n%s", dir, log, err, raw) + } + }) + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 4<<20), 4<<20) + hello := make(chan struct{}) + go func() { + if !scanner.Scan() { + close(hello) + return + } + var first struct { + Data appwire.Hello `json:"data"` + } + _ = json.Unmarshal(scanner.Bytes(), &first) + app.w.hello = first.Data.Features + close(hello) + for scanner.Scan() { + var reply actionFrame + if json.Unmarshal(scanner.Bytes(), &reply) != nil { + continue + } + if reply.ID != 0 { + if reply.Error != nil && len(daemon) > 0 && daemon[0] { + t.Logf("S6 app error envelope: %s", scanner.Bytes()) + } + app.w.mu.Lock() + ch := app.w.waiting[reply.ID] + delete(app.w.waiting, reply.ID) + app.w.mu.Unlock() + if ch != nil { + ch <- reply + } + continue + } + var frame e2Frame + _ = json.Unmarshal(scanner.Bytes(), &frame) + app.mu.Lock() + app.events = append(app.events, frame) + app.mu.Unlock() + select { + case app.more <- struct{}{}: + default: + } + } + // The channel is closed only once; normal output termination is separate. + }() + select { + case <-hello: + case <-time.After(30 * time.Second): + t.Fatal("app hello timed out") + } + + if setup != nil { + setup(app) + } + return app +} +func (a *e2App) wait(t *testing.T, match func(e2Frame) bool) { + t.Helper() + deadline := time.After(45 * time.Second) + seen := 0 + for { + a.mu.Lock() + for ; seen < len(a.events); seen++ { + if match(a.events[seen]) { + a.mu.Unlock() + return + } + } + a.mu.Unlock() + select { + case <-a.more: + case <-deadline: + raw, _ := os.ReadFile(a.log) + t.Fatalf("app event timed out\n%s", raw) + } + } +} +func e2Turn(t *testing.T, a *e2App, id, text string) { + t.Helper() + got := machineCall[struct { + Turn uint64 `json:"turn"` + }](t, a.w, "agents.send", map[string]any{"id": id, "text": text, "client_id": text}) + a.wait(t, func(f e2Frame) bool { + if f.Event != appwire.EventTurnDone { + return false + } + var d appwire.TurnDone + _ = json.Unmarshal(f.Data, &d) + if d.Turn != got.Turn { + return false + } + if !d.OK { + t.Fatalf("turn failed: %s", f.Data) + } + return true + }) +} + +// E-T1, E-T2, E-Q1: actual app commands, two homes, a scripted model and a +// loopback relay exercise takeover, idle notification, stale consent and quit. +func TestAppTakeET1ET2QuitEQ1(t *testing.T) { + appTakeRoundTrip(t, false) +} + +// S6 keeps both app processes, their presence watchers and their native-store +// daemons alive across the same real fetch, install and take-back acceptance. +func TestAppTakeProcessS6(t *testing.T) { + appTakeRoundTrip(t, true) +} + +func TestAppTakeF1P2FenceAndPromptPublication(t *testing.T) { + appTakeRoundTrip(t, false) +} + +// W5 removes the sender's workspace before the receiver takes its empty chat. +func TestAppTakeMissingWorkspaceW5(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("removing the sender's watched workspace needs Unix rename semantics; native W5 is in handoff") + } + appTakeRoundTrip(t, true, true) +} + +func appTakeRoundTrip(t *testing.T, daemon bool, missingWorkspace ...bool) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + absent := len(missingWorkspace) > 0 && missingWorkspace[0] + if absent { + workspace = filepath.Join(e2TempDir(t), "sender-project") + if err := os.MkdirAll(workspace, 0700); err != nil { + t.Fatal(err) + } + } + model := appTwoMessageSource(t, "echo codeaf-e2") + t.Cleanup(model.Close) + t.Setenv("CODEAF_BASE_URL", model.URL) + + var publicationMu sync.Mutex + var publications []string + relay := e2Relay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/publish") { + next.ServeHTTP(w, r) + return + } + reply := httptest.NewRecorder() + next.ServeHTTP(reply, r) + publicationMu.Lock() + publications = append(publications, r.URL.Path+" "+reply.Result().Status+" "+reply.Body.String()) + publicationMu.Unlock() + for k, v := range reply.Header() { + w.Header()[k] = v + } + w.WriteHeader(reply.Code) + _, _ = w.Write(reply.Body.Bytes()) + }) + }) + t.Cleanup(func() { + if t.Failed() { + publicationMu.Lock() + defer publicationMu.Unlock() + t.Logf("publication receipts: %v", publications) + } + }) + t.Setenv(syncsetup.URLVar, relay.URL) + t.Setenv(syncsetup.IntervalVar, "50") + dirA, dirB := home.Dir(), e2TempDir(t) + t.Cleanup(cellstore.StopSpawned) + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + for dir, name := range map[string]string{dirA: "A", dirB: "B"} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"tools.approvalMode":"allow"}`), 0600); err != nil { + t.Fatal(err) + } + + if _, err := devname.Set(dir, name); err != nil { + t.Fatal(err) + } + } + // A completed pairing has published both computers before continuation. + for _, dir := range []string{dirA, dirB} { + s := appDeviceSync(t, dir) + if err := syncsetup.PutOwnDevice(context.Background(), s.Dir, s.Identity, s.Device, devname.Name(dir)); err != nil { + t.Fatal(err) + } + } + a, b := e2StartApp(t, dirA, daemon), e2StartApp(t, dirB, daemon) + created := machineCall[struct { + Agent appwire.Agent `json:"agent"` + }](t, a.w, "agents.create", map[string]any{"workspace": workspace}).Agent + e2Turn(t, a, created.ID, "first") + // Start continuation from a chat saved by the real app leaving path. This + // exercises a saved chat independently of background first-publication timing. + e2Quit(t, a) + if absent { + if err := os.Rename(workspace, workspace+".away"); err != nil { + t.Fatal(err) + } + } else { + a = e2StartApp(t, dirA, daemon) + machineCall[appwire.TakeResult](t, a.w, "agents.open", appwire.IDParams{ID: created.ID}) + } + + t.Logf("created %s", string(e2JSON(created))) + if !absent { + t.Logf("A sync: %s", string(e2JSON(machineCall[appwire.SyncResult](t, a.w, "sync.get", struct{}{})))) + } + var row appwire.ChatRow + appSyncWait(t, "published chat", func() bool { + rows := machineCall[appwire.ChatsResult](t, b.w, "chats.list", struct{}{}) + for _, r := range rows.Chats { + if r.Title != "" && r.CanTake { + row = r + return true + } + } + return false + }) + taken := machineCall[appwire.TakeResult](t, b.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}).Agent + if taken.Location == nil || !taken.Location.Here { + t.Fatalf("taken: %+v", taken) + } + appSyncWait(t, "F1 take fence reaches the next row", func() bool { + rows := machineCall[appwire.ChatsResult](t, b.w, "chats.list", struct{}{}) + for _, r := range rows.Chats { + if r.Cell == row.Cell { + if r.Seen.Fence > taken.Location.Seen.Fence { + t.Fatalf("F1 take returned fence %d, next row has %d", taken.Location.Seen.Fence, r.Seen.Fence) + } + return r.Seen.Fence == taken.Location.Seen.Fence + } + } + return false + }) + if absent { + info, err := os.Stat(taken.Workspace) + if err != nil || !info.IsDir() || !strings.HasPrefix(taken.Workspace, dirB+string(filepath.Separator)) { + t.Fatalf("W5 receiver workspace %q: %v", taken.Workspace, err) + } + bound, err := os.ReadFile(filepath.Join(dirB, "v3", "stores", row.Cell, "engine-tree")) + if err != nil || string(bound) != taken.Workspace { + t.Fatalf("W5 followed tree %q, want %q: %v", bound, taken.Workspace, err) + } + if _, err := os.Stat(workspace); !os.IsNotExist(err) { + t.Fatalf("W5 sender workspace was recreated: %v", err) + } + e2Turn(t, b, taken.ID, "after take") + e2Quit(t, b) + return + } + a.wait(t, func(f e2Frame) bool { + if f.Event != appwire.EventAgentChanged { + return false + } + var d appwire.AgentChanged + _ = json.Unmarshal(f.Data, &d) + return d.Agent.ID == created.ID && d.Agent.Location != nil && !d.Agent.Location.Here && d.Agent.Location.Holder != nil && d.Agent.Location.Holder.Name == "B" + }) + _, refusal := a.w.call("agents.send", map[string]any{"id": created.ID, "text": "kept draft", "client_id": "refused"}) + if refusal == nil || refusal.Code != appwire.CodeConflict || !strings.Contains(string(e2JSON(refusal.Detail)), "held_elsewhere") { + t.Fatalf("send refusal: %+v", refusal) + } + _, conflict := a.w.call("agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}) + if conflict == nil || conflict.Code != appwire.CodeConflict || !strings.Contains(string(e2JSON(conflict.Detail)), "holder_changed") { + t.Fatalf("stale: %+v", conflict) + } + // A backend remembering a saved chat for the first time also refuses work + // from the inventory's known location, before opening a conversation. + cold, _, coldWire := appDeviceBackend(t, dirA) + machineCall[appwire.ChatsResult](t, coldWire, "chats.list", struct{}{}) + _, savedRefusal := coldWire.call("agents.send", map[string]any{"id": created.ID, "text": "saved draft", "client_id": "saved-refusal"}) + if savedRefusal == nil || savedRefusal.Code != appwire.CodeConflict || !strings.Contains(string(e2JSON(savedRefusal.Detail)), "held_elsewhere") { + t.Fatalf("saved chat refusal: %+v", savedRefusal) + } + coldWire.close() + _ = cold.Close() + e2Turn(t, b, taken.ID, "after take") + appSyncWait(t, "P2 completed taken turn reaches A before B quits", func() bool { + rows := machineCall[appwire.ChatsResult](t, a.w, "chats.list", struct{}{}) + for _, r := range rows.Chats { + if r.Cell == row.Cell && r.SavedAt.After(row.SavedAt) { + return true + } + } + return false + }) + // EOF immediately after turn.done must flush and release, with no empty turn. + e2Quit(t, b) + s := appDeviceSync(t, dirA) + v, err := s.Dir.Cell(context.Background(), row.Cell) + if err != nil { + t.Fatal(err) + } + if v.Cell.Lease.Expires != 0 { + t.Fatalf("quit stranded the hold: %+v", v.Cell.Lease) + } + if v.Cell.OrphanTurns != 0 { + t.Fatalf("empty turn branch: %+v", v.Cell) + } + again := machineCall[appwire.TakeResult](t, a.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: appwire.Seen{Device: v.Cell.Lease.Device, Fence: v.Cell.Lease.Fence}}).Agent + if again.Location == nil || !again.Location.Here { + t.Fatal(again) + } + entries, readErr := a.w.call("agents.entries", appwire.IDParams{ID: again.ID}) + if readErr != nil || !strings.Contains(string(entries), "after take") { + t.Fatalf("quit lost the final turn: %s %+v", entries, readErr) + } + listing, err := s.Dir.List(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, c := range listing.Cells { + if c.ParentCell != "" { + t.Fatalf("quit created a branch: %+v", c) + } + } +} +func e2JSON(v any) []byte { raw, _ := json.Marshal(v); return raw } + +// The engine door is exercised against the established directory/store wire +// rig. CODEAF_E2_DURABLE_RELAY=1 selects relayserve's durable backend. +func e2WireRelay(t *testing.T, wrappers ...func(http.Handler) http.Handler) *httptest.Server { + t.Helper() + var mu sync.Mutex + links := directory.NewLinks(time.Now, directory.DefaultLinkLimits) + dirs := map[string]directory.Directory{} + stores := map[string]blobstore.Store{} + auth := reqsign.AuthenticateAt(time.Now) + mux := http.NewServeMux() + mux.Handle(directory.LinkPath+"/", directory.LinkHandler(links, pairbox.SocketPeer)) + box := pairbox.Handler(pairbox.NewMemory(pairbox.DefaultLimits, time.Now), pairbox.SocketPeer) + mux.Handle(pairbox.Path, box) + mux.Handle(pairbox.Path+"/", box) + + mux.Handle("/v1/dir/", directory.Handler(auth, func(id string) (directory.Directory, error) { + mu.Lock() + defer mu.Unlock() + if dirs[id] == nil { + m := directory.NewMemory(time.Now) + m.SetLinks(links) + dirs[id] = m + } + return dirs[id], nil + })) + mux.Handle("/v1/store/", blobstore.Handler(auth, func(id string) (blobstore.Store, error) { + mu.Lock() + defer mu.Unlock() + if stores[id] == nil { + stores[id] = blobstore.NewMemory() + } + return stores[id], nil + })) + var handler http.Handler = mux + for _, wrap := range wrappers { + handler = wrap(handler) + } + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return server +} + +func e2Quit(t *testing.T, a *e2App) { + t.Helper() + a.w.close() + select { + case err := <-a.done: + if err != nil { + raw, _ := os.ReadFile(a.log) + t.Fatalf("quit: %v\n%s", err, raw) + } + a.done <- nil + case <-time.After(90 * time.Second): + t.Fatal("quit did not finish") + } + raw, _ := os.ReadFile(a.log) + if strings.Contains(string(raw), "could not save") { + t.Fatalf("save failure: %s", raw) + } +} + +func e2Relay(t *testing.T, wrappers ...func(http.Handler) http.Handler) *httptest.Server { + t.Helper() + if os.Getenv("CODEAF_E2_DURABLE_RELAY") == "1" { + return appSyncRelay(t) + } + return e2WireRelay(t, wrappers...) +} + +// E-T2: stale consent is refused before any restore, even for a chat that has +// never been copied here; the directory's computer and fence do not move. +func TestAppTakeET2(t *testing.T) { + celltest.Available(t) + relay := e2WireRelay(t) + t.Setenv(syncsetup.URLVar, relay.URL) + dirA, dirB := t.TempDir(), t.TempDir() + id, err := identity.Ensure(dirA) + if err != nil { + t.Fatal(err) + } + if _, err := identity.Adopt(dirB, id, false); err != nil { + t.Fatal(err) + } + s := appDeviceSync(t, dirA) + chat, err := cell.NewID(time.Now()) + if err != nil { + t.Fatal(err) + } + before, err := s.Dir.Create(context.Background(), chat, directory.CellInit{}) + if err != nil { + t.Fatal(err) + } + _, _, w := appDeviceBackend(t, dirB) + _, missing := w.call("agents.take", appwire.TakeParams{Cell: "unknown-chat", Seen: appwire.Seen{Device: before.Cell.Lease.Device, Fence: 0}}) + if missing == nil || missing.Code != appwire.CodeNotFound { + t.Fatalf("unknown chat: %+v", missing) + } + _, failure := w.call("agents.take", appwire.TakeParams{Cell: chat, Seen: appwire.Seen{Device: before.Cell.Lease.Device, Fence: 0}}) + if failure == nil || failure.Code != appwire.CodeConflict || failure.Message != chatlist.LostRace || !strings.Contains(string(e2JSON(failure.Detail)), "holder_changed") { + t.Fatalf("stale consent: %+v", failure) + } + after, err := s.Dir.Cell(context.Background(), chat) + if err != nil { + t.Fatal(err) + } + if after.Cell.Lease.Device != before.Cell.Lease.Device || after.Cell.Lease.Fence != before.Cell.Lease.Fence { + t.Fatalf("stale consent moved the chat: %+v", after.Cell.Lease) + } +} diff --git a/cmd/codeaf/app_takeback_model_test.go b/cmd/codeaf/app_takeback_model_test.go new file mode 100644 index 0000000000..1cb2c42f64 --- /dev/null +++ b/cmd/codeaf/app_takeback_model_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/config" +) + +func TestAppTakeC1bSavedModelTravelsBack(t *testing.T) { + for _, available := range []bool{true, false} { + t.Run(map[bool]string{true: "shared provider", false: "B-only provider"}[available], func(t *testing.T) { + celltest.RequireEngine(t) + _, workspace := wakeEnvironment(t, e2TempDir(t)) + model := appTwoMessageSource(t, "echo takeback-model") + t.Cleanup(model.Close) + requested := make(chan string, 1024) + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct{ Model string } + if r.Method == http.MethodPost { + // Restore the body so the ordinary scripted provider handles it. + raw := json.RawMessage{} + if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { + t.Error(err) + return + } + _ = json.Unmarshal(raw, &body) + r.Body = io.NopCloser(strings.NewReader(string(raw))) + requested <- body.Model + } + model.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(endpoint.Close) + t.Setenv("CODEAF_BASE_URL", endpoint.URL) + relay := e2WireRelay(t) + homeA, homeB := outageHomes(t, relay.URL) + const x = "fixture/model-x" + y := "fixture/model-y" + if !available { + y = "ollama/model-y" + if err := config.WriteSources(homeB, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: endpoint.URL}}); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{homeA, homeB} { + if err := config.WriteChatModel(dir, x); err != nil { + t.Fatal(err) + } + } + a, b := e2StartApp(t, homeA), e2StartApp(t, homeB) + created := machineCall[struct{ Agent appwire.Agent }](t, a.w, "agents.create", map[string]any{"workspace": workspace, "model": x}).Agent + e2Turn(t, a, created.ID, "A on X") + row := outageChatRow(t, b, "", func(r appwire.ChatRow) bool { return r.Title != "" && r.CanTake && r.Seen.Head != "" }) + fresh := machineCall[appwire.TakeResult](t, b.w, "agents.take", appwire.TakeParams{Cell: row.Cell, Seen: row.Seen}).Agent + if fresh.Model != x { + t.Fatalf("C1b fresh receiver used %q, want %q", fresh.Model, x) + } + assertC1bNotices(t, b, fresh.ID, 0, "") + const fallback = "fixture/fallback-y" + machineCall[any](t, b.w, "agents.set_models", map[string]any{"id": fresh.ID, "main": y, "fallback": fallback}) + e2Turn(t, b, fresh.ID, "B on Y") + // B saves a turn in several publications. Take back only what A has + // seen of B's last one: a head that both computers report, twice. + var back appwire.ChatRow + settled := "" + outageWait(t, "C1b A sees B's last save", func() bool { + mine := outageChatRow(t, b, row.Cell, func(r appwire.ChatRow) bool { return r.Seen.Head != row.Seen.Head }) + back = outageChatRow(t, a, row.Cell, func(r appwire.ChatRow) bool { return r.CanTake && r.Seen.Head != row.Seen.Head }) + if back.Seen.Head != mine.Seen.Head { + settled = "" + return false + } + if settled != back.Seen.Head { + settled = back.Seen.Head + time.Sleep(time.Second) + return false + } + return true + }) + cursor := func() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.events) + }() + taken := machineCall[appwire.TakeResult](t, a.w, "agents.take", appwire.TakeParams{Cell: back.Cell, Seen: back.Seen}).Agent + want, notices, line := y, 0, "" + if !available { + want, notices = x, 1 + line = "This chat used " + y + " on B. It isn't set up here, so it continues with " + x + "." + } + if taken.Model != want || taken.ModelIdentity == nil || taken.ModelIdentity.Models == nil || taken.ModelIdentity.Models.Main != want || taken.ModelIdentity.Models.Fallback != fallback { + t.Fatalf("C1b take back: model=%q identity=%+v, want main=%s fallback=%s", taken.Model, taken.ModelIdentity, want, fallback) + } + for len(requested) > 0 { + <-requested + } + e2Turn(t, a, taken.ID, "A after taking back") + var seen []string + for len(requested) > 0 { + seen = append(seen, <-requested) + } + wireModel := strings.TrimPrefix(want, "ollama/") + found := false + for _, model := range seen { + found = found || model == wireModel + if model == "model-y" && !available { + t.Fatal("C1b unavailable model reached A's provider") + } + } + if !found { + t.Fatalf("C1b turn did not use %s: %q", want, seen) + } + assertC1bNotices(t, a, taken.ID, cursor, line, notices) + }) + } +} + +func assertC1bNotices(t *testing.T, a *e2App, id string, cursor int, line string, counts ...int) { + t.Helper() + want := 0 + if len(counts) > 0 { + want = counts[0] + } + a.mu.Lock() + defer a.mu.Unlock() + var notices []string + for _, frame := range a.events[cursor:] { + var added appwire.EntryAdded + if frame.Event == appwire.EventEntryAdded && json.Unmarshal(frame.Data, &added) == nil && added.Entry.Agent == id && (added.Entry.NoticeKind == "model_switch" || strings.HasPrefix(added.Entry.Text, "This chat used ")) { + notices = append(notices, added.Entry.Text) + } + } + if len(notices) != want || want == 1 && notices[0] != line { + t.Fatalf("C1b notices = %q, want %d copies of %q", notices, want, line) + } +} diff --git a/cmd/codeaf/appclose.go b/cmd/codeaf/appclose.go index d34ab55c50..123ff9da92 100644 --- a/cmd/codeaf/appclose.go +++ b/cmd/codeaf/appclose.go @@ -851,6 +851,9 @@ func (b *appBackend) heldTaskViews(a *appAgent) map[string]appwire.Task { // Deleted journals keep their directory for deliverables. Only their lifetime // marker, not directory existence alone, can establish that work is still owed. func appCloseConversationExists(dir string) bool { + if appChatStaging(dir) { + return false + } if _, err := os.Stat(filepath.Join(dir, teams.ConversationDeletedFile)); !os.IsNotExist(err) { return false } diff --git a/cmd/codeaf/appdevices.go b/cmd/codeaf/appdevices.go index c406114b31..b70e35fcfd 100644 --- a/cmd/codeaf/appdevices.go +++ b/cmd/codeaf/appdevices.go @@ -3,6 +3,7 @@ package main import ( "cmp" "context" + "encoding/json" "errors" "runtime" "slices" @@ -23,50 +24,99 @@ import ( "github.com/Agent-Field/codeaf/internal/wireauth" ) -// appDevices observes traffic already owned by the engine. Opening Settings -// does not start a follower; only a person's list request reads the relay. +// appDevicesQuiet is the desktop's line for a home that has not paired yet: the +// desktop pairs by itself (devices.pair), so it never sends people to a terminal. +const appDevicesQuiet = "Only this computer is here so far." + +// appSyncOff is the desktop's line when sync is off: the desktop turns it on and +// off in its Relay settings, so the terminal's environment-variable advice does not apply. +const appSyncOff = "Sync is off on this computer." + +// appDevices owns the paired engine's device publication and shares its +// directory follower with chat inventory. Opening Settings adds no follower. type appDevices struct { - home string - watch *syncsetup.DeviceObserver - op sync.Mutex - done chan struct{} + home string + watch *syncsetup.DeviceObserver + op sync.Mutex + done chan struct{} + emitMu sync.Mutex + lastWire string + cancel context.CancelFunc + wake chan struct{} + owned chan struct{} } func (b *appBackend) listenDevices() *appDevices { - d := &appDevices{home: home.Dir(), done: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + d := &appDevices{home: home.Dir(), done: make(chan struct{}), cancel: cancel, wake: make(chan struct{}, 1), owned: make(chan struct{})} d.watch = syncsetup.ObserveDevices(d.home) + publishChange := func() { + d.op.Lock() + defer d.op.Unlock() + d.publish(b.emit) + } guard.Go("app/devices", func() { defer close(d.done) for range d.watch.Changes() { - if view, err := d.snapshot(); err == nil { - b.emit.Emit(appwire.EventDevicesChanged, appwire.DevicesResult{Devices: view}) - } + publishChange() } }) + guard.Go("app/own-device", func() { defer close(d.owned); d.publishOwn(ctx) }) return d } +// publishDevices compares the wire value because observer hints can describe +// unrelated directory changes, and explicit actions also publish their result. +func (b *appBackend) publishDevices() { + d := b.devices + d.op.Lock() + defer d.op.Unlock() + d.publish(b.emit) +} + +func (d *appDevices) publish(emit appwire.Emitter) { + d.emitMu.Lock() + defer d.emitMu.Unlock() + view, complete, err := d.snapshotState() + if err != nil || view.Mode == appwire.DevicesOn && !complete { + return + } + encoded, err := json.Marshal(appwire.DevicesResult{Devices: view}) + if err != nil || string(encoded) == d.lastWire { + return + } + d.lastWire = string(encoded) + emit.Emit(appwire.EventDevicesChanged, appwire.DevicesResult{Devices: view}) +} + func (d *appDevices) close() { + d.cancel() + <-d.owned d.watch.Close() <-d.done } func (d *appDevices) snapshot() (appwire.Devices, error) { + view, _, err := d.snapshotState() + return view, err +} + +func (d *appDevices) snapshotState() (appwire.Devices, bool, error) { // A first read needs a stable local device id without turning a fresh // identity into a fleet that is allowed to contact the relay. id, err := identity.EnsureSolo(d.home) if err != nil { - return appwire.Devices{}, err + return appwire.Devices{}, false, err } self, err := identity.Device(d.home) if err != nil { - return appwire.Devices{}, err + return appwire.Devices{}, false, err } view := appwire.Devices{Mode: appwire.DevicesOn} known := d.watch.Snapshot(id, self) switch { case !syncsetup.Resolve(d.home).On(): - view.Mode, view.Line = appwire.DevicesOff, chatlist.SyncOff + view.Mode, view.Line = appwire.DevicesOff, appSyncOff case !cell.Available(): view.Mode, view.Line = appwire.DevicesUnavailable, cell.UnavailableReason() case errors.Is(known.Refused, dirwatch.ErrRevoked): @@ -74,7 +124,7 @@ func (d *appDevices) snapshot() (appwire.Devices, error) { case errors.Is(known.Refused, dirwatch.ErrRotated): view.Mode, view.Line = appwire.DevicesUnavailable, chatlist.Replaced case !syncsetup.MayTalk(d.home): - view.Mode, view.Line = appwire.DevicesQuiet, chatlist.DevicesQuiet + view.Mode, view.Line = appwire.DevicesQuiet, appDevicesQuiet } if view.Mode != appwire.DevicesOn { if rec, ok := known.Devices[self.ID()]; ok { @@ -110,7 +160,7 @@ func (d *appDevices) snapshot() (appwire.Devices, error) { slices.SortFunc(view.Devices, func(a, b appwire.Device) int { return cmp.Or(-cmp.Compare(rank(a.Self), rank(b.Self)), cmp.Compare(rank(a.Removed), rank(b.Removed)), cmp.Compare(a.Name, b.Name), cmp.Compare(a.ID, b.ID)) }) - return view, nil + return view, known.Complete, nil } func (b *appBackend) ListDevices() (appwire.Devices, error) { @@ -125,7 +175,7 @@ func (b *appBackend) ListDevices() (appwire.Devices, error) { if err != nil || !ok { view.Devices, view.Mode = view.Devices[:1], appwire.DevicesUnavailable view.Devices[0].Online = nil - view.Line = chatlist.SyncOff + view.Line = appSyncOff if err != nil { view.Line = err.Error() } @@ -136,8 +186,12 @@ func (b *appBackend) ListDevices() (appwire.Devices, error) { if _, err := s.Dir.List(ctx); err != nil { return appwire.Devices{}, deviceError(err) } - if _, err := s.Dir.Presence(ctx); err != nil && !errors.Is(err, directory.ErrTooOld) { - return appwire.Devices{}, deviceError(err) + if _, err := s.Dir.Presence(ctx); err != nil { + // The successful roster establishes reachability. Presence is advisory + // and can fail independently; only an authority refusal invalidates it. + if errors.Is(err, directory.ErrRevoked) || errors.Is(err, directory.ErrRotated) { + return appwire.Devices{}, deviceError(err) + } } return d.snapshot() } @@ -156,11 +210,12 @@ func (b *appBackend) RenameDevice(typed string) (appwire.DeviceRenameResult, err if err != nil { return appwire.DeviceRenameResult{}, err } + d.wakePublication() view, err := d.snapshot() if err != nil { return appwire.DeviceRenameResult{}, err } - b.emit.Emit(appwire.EventDevicesChanged, appwire.DevicesResult{Devices: view}) + d.publish(b.emit) return appwire.DeviceRenameResult{Device: view.Devices[0], Line: line}, nil } @@ -183,7 +238,7 @@ func (b *appBackend) RemoveDevice(deviceID string) (appwire.DeviceRemoveResult, return appwire.DeviceRemoveResult{}, deviceError(err) } if !ok { - return appwire.DeviceRemoveResult{}, appwire.Errorf(appwire.CodeEngine, "%s", chatlist.SyncOff) + return appwire.DeviceRemoveResult{}, appwire.Errorf(appwire.CodeEngine, "%s", appSyncOff) } ctx, cancel := context.WithTimeout(context.Background(), syncsetup.DeviceRequestWithin) defer cancel() @@ -210,7 +265,7 @@ func (b *appBackend) RemoveDevice(deviceID string) (appwire.DeviceRemoveResult, // the relay. The revoke already updated this process's remembered record. view, err = d.snapshot() if err == nil { - b.emit.Emit(appwire.EventDevicesChanged, appwire.DevicesResult{Devices: view}) + d.publish(b.emit) } return appwire.DeviceRemoveResult{ID: deviceID, Line: line}, nil } diff --git a/cmd/codeaf/appdevices_binding_test.go b/cmd/codeaf/appdevices_binding_test.go index 7d024c5f39..8f371426bb 100644 --- a/cmd/codeaf/appdevices_binding_test.go +++ b/cmd/codeaf/appdevices_binding_test.go @@ -56,7 +56,7 @@ func TestAppDevicesV1V3RePairClearsRemovalWithoutRestart(t *testing.T) { if err := sa.Dir.Revoke(context.Background(), old.Device.ID()); err != nil { t.Fatal(err) } - if _, err := w.call("devices.list", struct{}{}); err == nil || err.Code != appwire.CodeEngine || err.Message != chatlist.Removed { + if _, err := w.call("devices.list", struct{}{}); err != nil && (err.Code != appwire.CodeEngine || err.Message != chatlist.Removed) { t.Fatalf("learned removal: %v", err) } if view := appDeviceList(t, w); view.Mode != appwire.DevicesUnavailable || view.Line != chatlist.Removed || len(view.Devices) != 1 || !view.Devices[0].Removed { @@ -86,7 +86,7 @@ func TestAppDevicesV1V3RePairClearsRemovalWithoutRestart(t *testing.T) { // V1 amendment: replacing the pairing accepts its lower directory version and // drops the old roster, presence and callbacks, including a delayed old list. -func TestAppDevicesV1ReplacementPairingDropsOldRoster(t *testing.T) { +func TestAppDevicesV1ReplacementPairingDropsOldRosterP2(t *testing.T) { celltest.Available(t) delay := &appDeviceDelayedList{ready: make(chan struct{}), release: make(chan struct{})} _, _, down := appDevicesRelay(t, delay.wrap) @@ -108,7 +108,7 @@ func TestAppDevicesV1ReplacementPairingDropsOldRoster(t *testing.T) { if err := peer.Rename(context.Background(), "old partner"); err != nil { t.Fatal(err) } - _, rec, w := appDeviceBackend(t, a) + backend, rec, w := appDeviceBackend(t, a) _ = appDeviceList(t, w) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -122,9 +122,9 @@ func TestAppDevicesV1ReplacementPairingDropsOldRoster(t *testing.T) { } defer watch.Close() done := make(chan error, 1) - delay.armed.Store(true) - go func() { _, err := old.Dir.List(ctx); done <- err }() - delay.wait(t) + delay.capture(t, backend, func() { + go func() { _, err := old.Dir.List(ctx); done <- err }() + }) nextID, err := identity.Ensure(c) if err != nil { t.Fatal(err) @@ -175,7 +175,7 @@ func TestAppDevicesV1ReplacementPairingDropsOldRoster(t *testing.T) { down.Store(true) _ = machineCall[appwire.DeviceRenameResult](t, w, "devices.rename", appwire.DeviceRenameParams{Name: "new cached desk"}) appDeviceEvent(t, rec, func(v appwire.Devices) bool { - return v.Mode == appwire.DevicesOn && len(v.Devices) == 2 && v.Devices[0].ID == fresh.Device.ID() && v.Devices[0].Name == "new cached desk" && v.Devices[1].ID == partner.Device.ID() && !v.Devices[0].Removed && !v.Devices[1].Removed && v.Devices[0].Online != nil && !*v.Devices[0].Online && v.Devices[1].Online != nil && !*v.Devices[1].Online + return v.Mode == appwire.DevicesOn && len(v.Devices) == 2 && v.Devices[0].ID == fresh.Device.ID() && v.Devices[0].Name == "new cached desk" && v.Devices[1].ID == partner.Device.ID() && !v.Devices[0].Removed && !v.Devices[1].Removed && v.Devices[0].Online == nil && v.Devices[1].Online == nil }) } diff --git a/cmd/codeaf/appdevices_test.go b/cmd/codeaf/appdevices_test.go index 8aea29a0e8..c363dd0001 100644 --- a/cmd/codeaf/appdevices_test.go +++ b/cmd/codeaf/appdevices_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "encoding/json" "net/http" "net/http/httptest" @@ -139,6 +140,17 @@ func (d *appDeviceDelayedList) wait(t *testing.T) { } } +// capture reserves this synthetic delay for the caller's read. The engine's +// startup publisher and shared watch may otherwise win the armed HTTP request. +func (d *appDeviceDelayedList) capture(t *testing.T, b *appBackend, start func()) { + t.Helper() + b.devices.op.Lock() + defer b.devices.op.Unlock() + d.armed.Store(true) + start() + d.wait(t) +} + // V1, V3 amendment: a completed removal survives an older concurrent roster // response from another sync client, even when the next refresh fails. func TestAppDevicesV1V3OlderRosterCannotUndoRemoval(t *testing.T) { @@ -160,7 +172,7 @@ func TestAppDevicesV1V3OlderRosterCannotUndoRemoval(t *testing.T) { t.Fatal(err) } } - _, rec, w := appDeviceBackend(t, a) + backend, rec, w := appDeviceBackend(t, a) if view := appDeviceList(t, w); len(view.Devices) != 2 || view.Devices[1].Removed { t.Fatalf("initial roster: %+v", view) } @@ -171,12 +183,12 @@ func TestAppDevicesV1V3OlderRosterCannotUndoRemoval(t *testing.T) { err error } done := make(chan response, 1) - delay.armed.Store(true) - go func() { - listing, err := sa.Dir.List(ctx) - done <- response{listing, err} - }() - delay.wait(t) + delay.capture(t, backend, func() { + go func() { + listing, err := sa.Dir.List(ctx) + done <- response{listing, err} + }() + }) removed := machineCall[appwire.DeviceRemoveResult](t, w, "devices.remove", appwire.DeviceRemoveParams{ID: sb.Device.ID()}) if removed.ID != sb.Device.ID() || removed.Line != pair.ChatsRevokedLine("computer") { t.Fatalf("removal receipt: %+v", removed) @@ -209,9 +221,17 @@ func TestAppDevicesV1V3OlderRosterCannotUndoRemoval(t *testing.T) { } // V1, V5: real watch traffic updates presence without any roster/presence read. -func TestAppDevicesV1V5ExistingWatchSuppliesPresence(t *testing.T) { +func TestAppDevicesV1V5ExistingWatchSuppliesPresenceP2(t *testing.T) { celltest.Available(t) - _, hits, _ := appDevicesRelay(t) + var watches atomic.Int64 + appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == directory.WatchPath { + watches.Add(1) + } + next.ServeHTTP(w, r) + }) + }) a, b := t.TempDir(), t.TempDir() id, err := identity.Ensure(a) if err != nil { @@ -226,13 +246,12 @@ func TestAppDevicesV1V5ExistingWatchSuppliesPresence(t *testing.T) { t.Fatal(err) } } - _, rec, w := appDeviceBackend(t, a) + backend, rec, w := appDeviceBackend(t, a) view := appDeviceList(t, w) - if view.Devices[1].Online == nil || *view.Devices[1].Online { - t.Fatalf("known offline presence: %+v", view) + if view.Devices[1].Online != nil { + t.Fatalf("P2 unobserved computer must start unknown: %+v", view) } - before := hits.Load() - // These sockets belong to other engine work, not to the Devices adapter. + // Other engine work shares the startup publisher's socket. follow := sa.Follow() if follow == nil { t.Fatal("the real directory has no follower") @@ -264,22 +283,13 @@ func TestAppDevicesV1V5ExistingWatchSuppliesPresence(t *testing.T) { v := data.(appwire.DevicesResult).Devices return len(v.Devices) == 2 && v.Devices[1].Online != nil && !*v.Devices[1].Online && !v.Devices[1].LastSeen.IsZero() }) - rec.mu.Lock() - after := len(rec.events) - rec.mu.Unlock() follow.Close() - seen := 0 - rec.waitFor(t, 5*time.Second, func(event string, data any) bool { - i := seen - seen++ - if i < after || event != appwire.EventDevicesChanged { - return false - } - v := data.(appwire.DevicesResult).Devices - return len(v.Devices) == 2 && v.Devices[1].Online == nil - }) - if hits.Load()-before != 2 { - t.Fatalf("watch events caused %d requests beyond the two upgrades", hits.Load()-before) + retained, err := backend.devices.snapshot() + if err != nil || retained.Devices[1].Online == nil || *retained.Devices[1].Online { + t.Fatalf("A1: closing another subscriber lost the engine's presence: %+v, %v", retained, err) + } + if watches.Load() != 2 { + t.Fatalf("A1: %d watch upgrades, want one shared local watch and one remote watch", watches.Load()) } } @@ -313,7 +323,19 @@ func TestAppDevicesV1OldRelayPresenceIsUnknown(t *testing.T) { // V1-V3: two paired homes use the real backend and registered wire methods. func TestAppDevicesV1V2V3PairedHomes(t *testing.T) { celltest.Available(t) - srv, hits, _ := appDevicesRelay(t) + var removedCert atomic.Value + var removedRequests, revokes atomic.Int64 + srv, _, _ := appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if cert, ok := removedCert.Load().(string); ok && r.Header.Get(reqsign.HeaderCert) == cert { + removedRequests.Add(1) + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/revoke") { + revokes.Add(1) + } + next.ServeHTTP(w, r) + }) + }) a, b := t.TempDir(), t.TempDir() id, err := identity.Ensure(a) if err != nil { @@ -323,6 +345,11 @@ func TestAppDevicesV1V2V3PairedHomes(t *testing.T) { t.Fatal(err) } sa, sb := appDeviceSync(t, a), appDeviceSync(t, b) + cert, err := json.Marshal(sb.Device.Cert) + if err != nil { + t.Fatal(err) + } + removedCert.Store(base64.RawURLEncoding.EncodeToString(cert)) for dir, name := range map[string]string{a: "desk", b: "laptop"} { if _, err := devname.Set(dir, name); err != nil { t.Fatal(err) @@ -364,11 +391,11 @@ func TestAppDevicesV1V2V3PairedHomes(t *testing.T) { if err != nil || len(rows) != 2 || rows[1].Name != "studio" { t.Fatalf("CLI roster: %+v, %v", rows, err) } - before := hits.Load() + before := revokes.Load() if _, err := wa.call("devices.remove", appwire.IDParams{ID: sa.Device.ID()}); err == nil || err.Code != appwire.CodeBadParams { t.Fatalf("remove self: %v", err) } - if hits.Load() != before { + if revokes.Load() != before { t.Fatal("self removal reached the relay") } if _, err := wa.call("devices.remove", appwire.IDParams{ID: "unknown"}); err == nil || err.Code != appwire.CodeNotFound { @@ -383,12 +410,12 @@ func TestAppDevicesV1V2V3PairedHomes(t *testing.T) { if len(view.Devices) != 2 || !view.Devices[1].Removed || view.Devices[1].RemoveLine != "" { t.Fatalf("removed device disappeared: %+v", view) } - if _, err := wb.call("devices.list", struct{}{}); err == nil || err.Code != appwire.CodeEngine || err.Message != chatlist.Removed { + if _, err := wb.call("devices.list", struct{}{}); err != nil && (err.Code != appwire.CodeEngine || err.Message != chatlist.Removed) { t.Fatalf("the removed computer's refusal: %v", err) } - before = hits.Load() + before = removedRequests.Load() unavailable := appDeviceList(t, wb) - if unavailable.Mode != appwire.DevicesUnavailable || unavailable.Line != chatlist.Removed || len(unavailable.Devices) != 1 || !unavailable.Devices[0].Removed || hits.Load() != before { + if unavailable.Mode != appwire.DevicesUnavailable || unavailable.Line != chatlist.Removed || len(unavailable.Devices) != 1 || !unavailable.Devices[0].Removed || removedRequests.Load() != before { t.Fatalf("known removal was not answered locally: %+v", unavailable) } srv.Close() @@ -429,10 +456,10 @@ func TestAppDevicesV4V5LocalModesMakeNoRelayRequests(t *testing.T) { if view.Mode != mode || len(view.Devices) != 1 || !view.Devices[0].Self || view.Devices[0].Online != nil || !view.Devices[0].LastSeen.IsZero() || view.Devices[0].RemoveLine != "" || view.Line == "" { t.Fatalf("local list: %+v", view) } - if mode == appwire.DevicesQuiet && (view.Line != chatlist.DevicesQuiet || !strings.Contains(view.Line, "codeaf pair") || !identity.Solo(dir)) { - t.Fatalf("fresh home is not quiet with terminal instructions: %+v", view) + if mode == appwire.DevicesQuiet && (view.Line != appDevicesQuiet || strings.Contains(view.Line, "codeaf pair") || !identity.Solo(dir)) { + t.Fatalf("fresh home is not quiet with the desktop's own line: %+v", view) } - if mode == appwire.DevicesOff && view.Line != chatlist.SyncOff { + if mode == appwire.DevicesOff && view.Line != appSyncOff { t.Fatalf("off copy: %q", view.Line) } if mode != appwire.DevicesUnavailable { @@ -456,17 +483,31 @@ func TestAppDevicesV4V5LocalModesMakeNoRelayRequests(t *testing.T) { } } -// V5: while Devices was never opened, engine-owned writes emit without a read. -func TestAppDevicesV5PassiveEventsDoNotReadRelay(t *testing.T) { +// A1/V5/F3: the engine's observer and shared follower do not duplicate writes. +func TestAppDevicesA1V5ObserverDoesNotDuplicateWrites(t *testing.T) { celltest.Available(t) - _, hits, _ := appDevicesRelay(t) + var puts atomic.Int64 + appDevicesRelay(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + puts.Add(1) + } + next.ServeHTTP(w, r) + }) + }) dir := t.TempDir() if _, err := identity.Ensure(dir); err != nil { t.Fatal(err) } s := appDeviceSync(t, dir) - before := hits.Load() _, rec, _ := appDeviceBackend(t, dir) + appDeviceEvent(t, rec, func(v appwire.Devices) bool { + return len(v.Devices) == 1 && v.Devices[0].Online != nil && *v.Devices[0].Online + }) + if _, err := s.Dir.List(t.Context()); err != nil { + t.Fatal(err) + } + before := puts.Load() if err := s.Rename(context.Background(), "elsewhere in the engine"); err != nil { t.Fatal(err) } @@ -479,8 +520,8 @@ func TestAppDevicesV5PassiveEventsDoNotReadRelay(t *testing.T) { t.Fatal(err) } appDeviceEvent(t, rec, func(v appwire.Devices) bool { return v.Devices[0].Name == "window name" }) - if hits.Load()-before != 2 { - t.Fatalf("passive events sent %d requests beyond the two writes", hits.Load()-before) + if puts.Load()-before != 2 { + t.Fatalf("A1: observer sent %d PUTs, want the two actual name changes", puts.Load()-before) } } @@ -493,6 +534,8 @@ func TestAppDevicesV2OfflineRenameAndUnreachableList(t *testing.T) { t.Fatal(err) } _, rec, w := appDeviceBackend(t, dir) + // Offline changes retain the complete roster learned while connected. + appDeviceList(t, w) srv.Close() if _, err := w.call("devices.list", struct{}{}); err == nil || err.Code != appwire.CodeUnreachable || err.Message != chatlist.Unreachable { t.Fatalf("unreachable list: %v", err) diff --git a/cmd/codeaf/applazy.go b/cmd/codeaf/applazy.go index 1ac0beb408..9d8f3ec58f 100644 --- a/cmd/codeaf/applazy.go +++ b/cmd/codeaf/applazy.go @@ -110,7 +110,7 @@ func (b *appBackend) needsLazyOpen(run *appCrewRun, read appCrewRead, row sessio func (b *appBackend) lazyCandidates(run *appCrewRun, read appCrewRead, first bool) map[string]session.SessionRow { if first { for _, row := range session.ReadWorld(session.PlacesRoot()).Sessions() { - if !row.DeletionPending { + if !row.DeletionPending && !appChatStaging(row.Dir) { b.remember(row) } } diff --git a/cmd/codeaf/appsessionactions_test.go b/cmd/codeaf/appsessionactions_test.go index a99a1c9cc6..fafcbd2760 100644 --- a/cmd/codeaf/appsessionactions_test.go +++ b/cmd/codeaf/appsessionactions_test.go @@ -215,13 +215,14 @@ func (h *actionHarness) agent() string { // actionWire is a client of the real appwire.Server. type actionWire struct { - t *testing.T - in *io.PipeWriter - mu sync.Mutex - next int - waiting map[int]chan actionFrame - hello []string - stop sync.Once + replyWithin time.Duration + t *testing.T + in *io.PipeWriter + mu sync.Mutex + next int + waiting map[int]chan actionFrame + hello []string + stop sync.Once } type actionFrame struct { @@ -292,10 +293,14 @@ func (w *actionWire) call(method string, params any) (json.RawMessage, *appwire. if _, err := io.WriteString(w.in, line); err != nil { w.t.Fatal(err) } + within := w.replyWithin + if within <= 0 { + within = 30 * time.Second + } select { case f := <-ch: return f.Result, f.Error - case <-time.After(30 * time.Second): + case <-time.After(within): w.t.Fatalf("%s: no answer", method) return nil, nil } diff --git a/cmd/codeaf/appstate.go b/cmd/codeaf/appstate.go index b5d367b640..a0e967a7fb 100644 --- a/cmd/codeaf/appstate.go +++ b/cmd/codeaf/appstate.go @@ -527,6 +527,22 @@ func (b *appBackend) settingsMoved() { } } +// listenSyncedSettings uses the restore transaction's notification rather than +// another file watcher or clock. One restore can change several settings. +func (b *appBackend) listenSyncedSettings() func() { + return config.ObserveSyncedPreferences(config.ProfileDir(), func(keys []string) { + for _, key := range keys { + switch key { + case config.KeyChatModel, config.KeyToolApprovalMode, config.KeySpendRail, + config.KeyDailyBudget, config.KeyCrewTaskCap, config.KeyEffort, + config.KeyTierHighModel, config.KeyTierMastermindModel, config.KeyModelFallbacks: + b.settingsMoved() + return + } + } + }) +} + // appDollars writes a figure the way the profile's money writers read one. func appDollars(usd float64) string { return strconv.FormatFloat(usd, 'f', -1, 64) diff --git a/cmd/codeaf/appsync.go b/cmd/codeaf/appsync.go index 5696207cbe..fc41951a34 100644 --- a/cmd/codeaf/appsync.go +++ b/cmd/codeaf/appsync.go @@ -1,17 +1,17 @@ package main import ( + "context" "crypto/sha256" "fmt" "path/filepath" + "reflect" "strings" "sync" "github.com/Agent-Field/codeaf/internal/appwire" "github.com/Agent-Field/codeaf/internal/cell" - "github.com/Agent-Field/codeaf/internal/chatlist" "github.com/Agent-Field/codeaf/internal/guard" - "github.com/Agent-Field/codeaf/internal/home" "github.com/Agent-Field/codeaf/internal/session" "github.com/Agent-Field/codeaf/internal/syncsetup" ) @@ -142,8 +142,7 @@ func (b *appBackend) located(a *appAgent, location *appwire.Location) { if old == nil && location == nil { return } - if old != nil && location != nil && old.Here == location.Here && old.Line == location.Line && - (old.Holder == nil && location.Holder == nil || old.Holder != nil && location.Holder != nil && *old.Holder == *location.Holder) { + if reflect.DeepEqual(old, location) { return } a.view.Location = location @@ -155,6 +154,7 @@ func (b *appBackend) located(a *appAgent, location *appwire.Location) { } func (b *appBackend) syncStarted(s *syncsetup.Sync) { + guard.Go("app/chats-started", func() { _, _ = b.ListChats() }) b.sync.mu.Lock() defer b.sync.mu.Unlock() if _, owed := s.PeekFirstRun(); owed { @@ -166,13 +166,13 @@ func (b *appBackend) syncStarted(s *syncsetup.Sync) { // syncStatusLocked is a local read. In particular it neither opens Sync nor // creates an identity, starts a drive or contacts a relay. func (b *appBackend) syncStatusLocked() appwire.Sync { - state := appwire.Sync{Mode: appwire.SyncQuiet} + state := appwire.Sync{Mode: appwire.SyncQuiet, Relay: appRelay(b.relayHome())} switch { - case syncsetup.Resolve(home.Dir()).Off: - state.Mode, state.Reason = appwire.SyncOff, chatlist.SyncOff + case state.Relay.Off: + state.Mode, state.Reason = appwire.SyncOff, appSyncOff case !cell.Available(): state.Mode, state.Reason = appwire.SyncUnavailable, cell.UnavailableReason() - case syncsetup.MayTalk(home.Dir()): + case syncsetup.MayTalk(b.relayHome()): state.Mode = appwire.SyncOn } if b.sync.owed != nil { @@ -195,7 +195,7 @@ func (b *appBackend) syncChangedLocked() appwire.Sync { state := b.syncStatusLocked() old := b.sync.last sameDisclosure := old.Disclosure == nil && state.Disclosure == nil || old.Disclosure != nil && state.Disclosure != nil && *old.Disclosure == *state.Disclosure - if old.Mode != state.Mode || old.Reason != state.Reason || !sameDisclosure { + if old.Mode != state.Mode || old.Reason != state.Reason || !sameDisclosure || !reflect.DeepEqual(old.Relay, state.Relay) { b.sync.last = state b.emit.Emit(appwire.EventSyncChanged, appwire.SyncChanged{Sync: state}) } @@ -228,6 +228,9 @@ func appLocationRefusal(location *appwire.Location) error { } func (b *appBackend) knownWorkRefusal(a *appAgent) error { + if err := b.pairingWorkRefusal(); err != nil { + return err + } var refusal error b.locked(func() { if a.windowMoved { @@ -239,6 +242,46 @@ func (b *appBackend) knownWorkRefusal(a *appAgent) error { return refusal } +// refreshWorkRefusal lets a person's next send recover a released source chat. +// Other work doors retain their cached refusal, so a known viewer does not pay +// a relay round trip for each task, queue or remote-action check. +func (b *appBackend) refreshWorkRefusal(a *appAgent) error { + refusal := b.knownWorkRefusal(a) + if refusal == nil { + return nil + } + if err := b.pairingWorkRefusal(); err != nil { + return err + } + var transcript string + var drives *driveBook + var moved bool + b.locked(func() { transcript, drives, moved = a.transcript, b.drives, a.windowMoved }) + dir, folder := session.FolderOf(transcript) + if moved || drives == nil || !folder { + return refusal + } + readDrive := func() *liveDrive { + drives.mu.Lock() + defer drives.mu.Unlock() + return drives.drive[filepath.Base(dir)] + } + live := readDrive() + if live == nil || live.started() == nil { + return refusal + } + drive := live.started() + ctx, cancel := context.WithTimeout(context.Background(), driveWorkWithin) + defer cancel() + if _, err := drive.ConfirmWork(ctx); err != nil { + return refusal + } + if err := drive.Gate(); err != nil { + return appEngineError(err) + } + return b.knownWorkRefusal(a) +} + // workAllowed reads cached ownership before handing new work to an open // conversation. A joined host also checks its session's cached fact remotely. // Callers hold a.gate, never b.mu across I/O. diff --git a/cmd/codeaf/appsync_test.go b/cmd/codeaf/appsync_test.go index 7002c6aa24..3e257fc91e 100644 --- a/cmd/codeaf/appsync_test.go +++ b/cmd/codeaf/appsync_test.go @@ -764,7 +764,7 @@ func TestC6AppSyncQuietOffAndUnavailableStayLocal(t *testing.T) { t.Fatal(err) } t.Setenv(syncsetup.URLVar, "off") - read(appwire.SyncOff, chatlist.SyncOff) + read(appwire.SyncOff, appSyncOff) if d, err := startDriveWith(cell.Cell{}, cellstore.Engine{}, func(error) {}, b.drives.notices); d != nil || err != nil { t.Fatalf("off drive = %v, %v", d, err) } diff --git a/cmd/codeaf/apptidy.go b/cmd/codeaf/apptidy.go index 92a1a18997..0be9586517 100644 --- a/cmd/codeaf/apptidy.go +++ b/cmd/codeaf/apptidy.go @@ -381,6 +381,7 @@ func (b *appBackend) dress(view *appwire.Agent) { } view.Line = agentLineLocked(a, view.State) }) + b.dressChatLocation(view) } // agentLineLocked is an agent's live one-liner for its state: the step it is diff --git a/cmd/codeaf/cell.go b/cmd/codeaf/cell.go index 53d9d1bc51..5cd61bcf0b 100644 --- a/cmd/codeaf/cell.go +++ b/cmd/codeaf/cell.go @@ -104,15 +104,19 @@ func openCellNamed(name, cwd string) (cell.Cell, error) { // openCellByID finds a cell by id where a chat keeps it: in the project bucket // its workspace names, or in the flat cells directory of cell.Create. func openCellByID(id string) (cell.Cell, error) { - buckets, _ := filepath.Glob(filepath.Join(home.Join("v3", "projects"), "*", id)) - copies, _ := filepath.Glob(filepath.Join(home.Join("v3", "projects"), "*", ".moving", id)) + return openCellByIDAt(home.Dir(), id) +} + +func openCellByIDAt(dir, id string) (cell.Cell, error) { + buckets, _ := filepath.Glob(filepath.Join(filepath.Join(dir, "v3", "projects"), "*", id)) + copies, _ := filepath.Glob(filepath.Join(filepath.Join(dir, "v3", "projects"), "*", ".moving", id)) buckets = append(buckets, copies...) - for _, dir := range append(buckets, filepath.Join(cell.Dir(home.Dir()), id)) { + for _, dir := range append(buckets, filepath.Join(cell.Dir(dir), id)) { if c, err := cell.OpenAt(dir, id); err == nil { return c, nil } } - return cell.Open(home.Dir(), id) + return cell.Open(dir, id) } func openCellAbove(dir string) (cell.Cell, error) { diff --git a/cmd/codeaf/chatv3.go b/cmd/codeaf/chatv3.go index 775488a1a7..465dc02a50 100644 --- a/cmd/codeaf/chatv3.go +++ b/cmd/codeaf/chatv3.go @@ -430,7 +430,7 @@ func openChatV3(name string, args []string, pickSession bool) error { // which is the safe direction and not the useful one ([beltWatch] states the // ordering problem this closes). launch.Subharnesses.Belt.watch(agent.ToolOnBelt) - if notice != "" { + if cfg.SessionFile != launch.SessionFile { // The session file moved under us, so everything downstream that names // it — /help, /new, the resumed line — has to name the new one. transcript, resumed = cfg.SessionFile, false @@ -1460,6 +1460,19 @@ func openV3Agent(cfg session.Config, workspace string, open func(session.Config) if handoff.Arriving(cfg.Place.Dir) { return nil, cfg, "", errors.New(chatArrivingSentence) } + model, modelNotice, err := continuedModel(cfg) + if err != nil { + var missing *continuedModelUnavailable + if !errors.As(err, &missing) { + return nil, cfg, "", err + } + // Taking and opening a chat do not require a usable model. Retain the + // incoming identity even if seating seals local bookkeeping first. + cfg.PrepareModel = missing.prepare + } + if model != "" { + cfg.Model = model + } cfg = v3Seated(v3Migrated(cfg)) // Restore the gate before construction, so restored work cannot start behind // the profile default. Return the launch config unchanged: a subsequent new @@ -1492,6 +1505,12 @@ func openV3Agent(cfg session.Config, workspace string, open func(session.Config) } agent, err := open(restored) if err == nil { + if model != "" { + if err := agent.ContinueWithModel(model, modelNotice); err != nil { + closeV3Agent(agent) + return nil, cfg, "", err + } + } if restoredPosture != "" { if err := agent.SetApprovalPosture(restoredPosture); err != nil { closeV3Agent(agent) @@ -1501,7 +1520,7 @@ func openV3Agent(cfg session.Config, workspace string, open func(session.Config) if savedEffort != "" { agent.SetConversationEffort(savedEffort) } - return agent, cfg, "", nil + return agent, cfg, modelNotice, nil } if !errors.Is(err, session.ErrSessionLocked) { return nil, cfg, "", err diff --git a/cmd/codeaf/chatv3_process.go b/cmd/codeaf/chatv3_process.go index 4a20b57a19..5a192adcc7 100644 --- a/cmd/codeaf/chatv3_process.go +++ b/cmd/codeaf/chatv3_process.go @@ -873,7 +873,7 @@ func (s *v3Seam) open(launch *v3Launch, cfg session.Config, resumed bool) (tui3. return tui3.Conversation{}, err } s.proc.track(agent) - if notice != "" { + if cfg.SessionFile != launch.SessionFile { // The session file moved under us, so this is a new conversation rather // than the one that was asked for, and the sentence says so. resumed = false diff --git a/cmd/codeaf/continued_model.go b/cmd/codeaf/continued_model.go new file mode 100644 index 0000000000..cb9a8cc337 --- /dev/null +++ b/cmd/codeaf/continued_model.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/session" +) + +// continuedModel identifies a copy before seating can seal local bookkeeping. +// Ordinary local launches retain their model rules. +func continuedModel(cfg session.Config) (string, string, error) { + dev, err := identity.LoadDevice(home.Dir()) + if err != nil { + return "", "", nil + } + c, err := cell.OpenAt(cfg.Place.Dir, cfg.Place.ID()) + if err != nil { + return "", "", nil + } + head, err := cellstore.Head(c) + from, adopted := cellstore.Adopted(c) + if err != nil || head == nil || !adopted && (head.Turn.Device == "" || head.Turn.Device == dev.Cert.Device) { + return "", "", nil + } + model, providerID := cfg.Model, "" + if meta, err := session.LoadMeta(c.Root); err == nil { + if state := meta.ModelIdentity; state != nil { + if state.Models != nil { + model = state.Models.Main + } + if state.Requested == model { + providerID = state.Provider + } + } + if meta.Model == model && meta.ModelProvider != "" { + providerID = meta.ModelProvider + } + } + sources := cfg.Sources.OrDefault(cfg.APIKey, cfg.BaseURL) + return continuedModelChoice(sources, model, providerID, from) +} + +// continuedModelUnavailable carries the original provider through local saves +// while the person sets up a usable default. +type continuedModelUnavailable struct{ model, provider, from string } + +func (e *continuedModelUnavailable) Error() string { + return fmt.Sprintf("This chat used %s, which isn't set up on this computer. Set up a model in Settings, then send again.", continuedModelLabel(e.model)) +} + +func (e *continuedModelUnavailable) prepare(model string, sources modelsource.Set) (string, string, error) { + provider := e.provider + if model != e.model { + provider = "" + } + return continuedModelChoice(sources, model, provider, e.from) +} + +func continuedModelChoice(sources modelsource.Set, model, providerID, from string) (string, string, error) { + if config.ModelUsable(sources, model, providerID) { + return "", "", nil + } + settings, err := config.LoadKeyless() + if err != nil { + return "", "", err + } + fallback := v3TalkModel("", settings) + if !config.ModelUsable(sources, fallback, "") { + return "", "", &continuedModelUnavailable{model: model, provider: providerID, from: from} + } + where := "" + if from != "" { + where = " on " + from + } + return fallback, fmt.Sprintf("This chat used %s%s. It isn't set up here, so it continues with %s.", model, where, fallback), nil +} + +func continuedModelLabel(model string) string { + for _, source := range modelsource.Vendored() { + if bare, ok := strings.CutPrefix(model, source.Written+"/"); ok { + return strings.TrimSuffix(bare, ":latest") + " (" + source.Name + ")" + } + } + return model +} diff --git a/cmd/codeaf/continued_model_test.go b/cmd/codeaf/continued_model_test.go new file mode 100644 index 0000000000..7c18865a55 --- /dev/null +++ b/cmd/codeaf/continued_model_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +func incomingModelConfig(t *testing.T) session.Config { + t.Helper() + celltest.RequireEngine(t) + t.Setenv(home.EnvVar, t.TempDir()) + t.Setenv(config.ProfileDirEnv, "") + t.Setenv(syncsetup.URLVar, "off") + for _, name := range config.ProviderKeyEnvNamesAt(home.Dir()) { + t.Setenv(name, "") + } + if _, err := identity.Device(home.Dir()); err != nil { + t.Fatal(err) + } + foreign, err := identity.Device(t.TempDir()) + if err != nil { + t.Fatal(err) + } + c, err := cell.Create(home.Dir(), cell.Options{Class: cell.HostBound}) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + if err := session.SaveMeta(c.Root, session.Meta{ID: c.ID, Workspace: work, Model: "ollama/llama3.2:latest"}); err != nil { + t.Fatal(err) + } + engine := cellstore.EngineFor(work) + engine.Identity = cellstore.Identity{Device: foreign.Cert.Device, Fence: 1} + if _, err := engine.Seal(t.Context(), c, cellstore.TurnInfo{}); err != nil { + t.Fatal(err) + } + t.Cleanup(cellstore.StopSpawned) + return session.Config{Workspace: work, Place: session.Place{Dir: c.Root, Workspace: work}, SessionFile: session.Place{Dir: c.Root}.Transcript(), Model: "ollama/llama3.2:latest", BaseURL: "http://127.0.0.1:1"} +} + +func TestChatT1UnavailableModelMakesNoNamingProviderCall(t *testing.T) { + cfg := incomingModelConfig(t) + var calls atomic.Int64 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + calls.Add(1) + } + http.Error(w, "an unavailable model must not be called", http.StatusBadRequest) + })) + defer endpoint.Close() + cfg.APIKey, cfg.BaseURL = "fixture", endpoint.URL + if err := config.WriteChatModel(home.Dir(), cfg.Model); err != nil { + t.Fatal(err) + } + agent, _, _, err := openV3Agent(cfg, cfg.Workspace, session.New) + if err != nil { + t.Fatal(err) + } + defer closeV3Agent(agent) + events, err := agent.Submit(t.Context(), "an unnamed incoming chat") + if err != nil { + t.Fatal(err) + } + for range events { + } + // Naming runs beside the turn. Let its scheduled goroutine reach the + // fixture before checking that the refusal prevented every provider call. + time.Sleep(100 * time.Millisecond) + if calls.Load() != 0 { + t.Fatalf("T1 unavailable model made %d naming/provider calls", calls.Load()) + } +} + +func TestChatC1T1NoUsableModelExplainsSetupOnFirstTurn(t *testing.T) { + cfg := incomingModelConfig(t) + work := cfg.Workspace + agent, _, _, err := openV3Agent(cfg, work, session.New) + if err != nil { + t.Fatal(err) + } + defer closeV3Agent(agent) + events, err := agent.Submit(t.Context(), "hello") + if err != nil { + t.Fatal(err) + } + var reason error + for event := range events { + if event.Kind == session.EventError { + reason = event.Err + } + } + if reason == nil || !strings.Contains(reason.Error(), "isn't set up on this computer") || !strings.Contains(reason.Error(), "Set up a model in Settings") { + t.Fatalf("C1/T1 no usable model did not explain setup: %v", reason) + } +} + +func TestChatC1ExplicitUsableModelKeepsLaunchPrecedence(t *testing.T) { + cfg := incomingModelConfig(t) + if err := config.WriteChatModel(home.Dir(), "profile/default"); err != nil { + t.Fatal(err) + } + cfg.Model, cfg.APIKey, cfg.BaseURL = "chosen/model", "fixture", "http://127.0.0.1:1" + agent, _, notice, err := openV3Agent(cfg, cfg.Workspace, session.New) + if err != nil { + t.Fatal(err) + } + defer closeV3Agent(agent) + if agent.Model() != "chosen/model" || notice != "" { + t.Fatalf("C1 explicit usable choice changed: %q, %q", agent.Model(), notice) + } +} diff --git a/cmd/codeaf/engine.go b/cmd/codeaf/engine.go index 60049e0bda..c13500edcf 100644 --- a/cmd/codeaf/engine.go +++ b/cmd/codeaf/engine.go @@ -1228,7 +1228,7 @@ func assembleEngine(hello remote.Hello, workspace, sessionFlag, launchDir string agent.SetReasoning(level) } transcript, resumed := launch.SessionFile, launch.Resumed - if notice != "" || hello.New { + if cfg.SessionFile != launch.SessionFile || hello.New { // The session file moved under us, so the welcome has to name the new // one — everything the surface prints about this conversation comes off // that frame. A minted conversation is the same fact said on purpose: it diff --git a/cmd/codeaf/pairroute.go b/cmd/codeaf/pairroute.go index be3d23a4f4..3632a4f713 100644 --- a/cmd/codeaf/pairroute.go +++ b/cmd/codeaf/pairroute.go @@ -8,6 +8,7 @@ package main import ( "net/http" "net/url" + "runtime" "strings" "time" @@ -28,13 +29,17 @@ const pairPollTimeout = pairbox.MaxWait + 10*time.Second // relay this computer syncs through. A relay named by flag is the one the other // device is told to pass, so it is `Shown`; the default needs no telling. func pairMailbox(relayFlag string) (pair.Mailbox, error) { - if syncsetup.Resolve(home.Dir()).Off { + return pairMailboxAt(home.Dir(), relayFlag) +} + +func pairMailboxAt(dir, relayFlag string) (pair.Mailbox, error) { + if syncsetup.Resolve(dir).Off { return pair.Mailbox{}, pair.ErrSyncOff } flag := strings.TrimSpace(relayFlag) base := flag if flag == "" { - relay := syncsetup.Resolve(home.Dir()) + relay := syncsetup.Resolve(dir) if relay.Off { return pair.Mailbox{}, pair.ErrSyncOff } @@ -63,14 +68,18 @@ func pairMailbox(relayFlag string) (pair.Mailbox, error) { // It also names the identities this one replaced by rotation, so a computer // still on one of them may follow instead of being refused. func pairGrant(route pair.Mailbox) (pair.Grant, error) { - id, err := identity.Ensure(home.Dir()) + return pairGrantAt(home.Dir(), route) +} + +func pairGrantAt(dir string, route pair.Mailbox) (pair.Grant, error) { + id, err := identity.Ensure(dir) if err != nil { return pair.Grant{}, err } - if err := identity.EndSolo(home.Dir()); err != nil { // offering the identity is starting a fleet + if err := identity.EndSolo(dir); err != nil { // offering the identity is starting a fleet return pair.Grant{}, err } - return pair.Grant{Identity: id, SyncURL: syncsetup.Named(route.URL), Replaces: identity.Predecessors(home.Dir())}, nil + return pair.Grant{Identity: id, SyncURL: syncsetup.Named(route.URL), Replaces: identity.Predecessors(dir)}, nil } // pairJoining is this computer as the device that types the code. @@ -83,6 +92,7 @@ func pairJoiningAt(dir string) func(replace bool) pair.Joining { Home: dir, ProfileDir: config.ProfileDir(), Label: devname.Name(dir), + Platform: runtime.GOOS, Replace: replace, SaveSyncURL: func(url string) error { return syncsetup.SaveRelayURL(dir, url) }, } diff --git a/cmd/codeaf/syncdrive_live.go b/cmd/codeaf/syncdrive_live.go index e9bc53973e..21246e594e 100644 --- a/cmd/codeaf/syncdrive_live.go +++ b/cmd/codeaf/syncdrive_live.go @@ -8,6 +8,7 @@ import ( "github.com/Agent-Field/codeaf/internal/cell" "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" "github.com/Agent-Field/codeaf/internal/guard" "github.com/Agent-Field/codeaf/internal/home" "github.com/Agent-Field/codeaf/internal/syncsetup" @@ -30,6 +31,8 @@ type liveDrive struct { ready chan struct{} attempted bool closed bool + retry *time.Timer + retryWait time.Duration // changes coalesces local health updates for the host's cached surface state. changes chan struct{} } @@ -80,11 +83,36 @@ func (l *liveDrive) install(d *syncsetup.Drive, err error, ready chan struct{}) l.drive, l.waiting, l.attempted = d, errors.Is(err, syncsetup.ErrNoIdentity) || errors.Is(err, syncsetup.ErrQuiet), true if d != nil { d.CatchUp() + l.retryWait = 0 + } else if !l.closed && cellsync.Unreachable(err) { + l.retryStartup() } l.renewalChanged() return l.closed } +// retryStartup owns the next attempt after a transport failure. It runs under +// mu; gates and seals remain local until the single startup attempt succeeds. +func (l *liveDrive) retryStartup() { + if l.retryWait == 0 { + l.retryWait = cellsync.DefaultInterval + } else { + l.retryWait = min(2*l.retryWait, cellsync.MaxBackoff) + } + resetAttempt := func() { + l.mu.Lock() + defer l.mu.Unlock() + l.retry = nil + if !l.closed { + l.attempted = false + } + } + l.retry = time.AfterFunc(l.retryWait, func() { + resetAttempt() + l.try() + }) +} + // openDrive is the started drive, or nil when none has started or this writer // has closed. It never starts one. func (l *liveDrive) openDrive() *syncsetup.Drive { @@ -180,6 +208,10 @@ func (l *liveDrive) Close(ctx context.Context) error { l.mu.Lock() defer l.mu.Unlock() l.closed = true + if l.retry != nil { + l.retry.Stop() + l.retry = nil + } return l.ready }() if ready != nil { diff --git a/cmd/codeaf/synctake.go b/cmd/codeaf/synctake.go index 6ac646e7af..fcfc9cd880 100644 --- a/cmd/codeaf/synctake.go +++ b/cmd/codeaf/synctake.go @@ -46,12 +46,7 @@ func wireSync(o *tui3.Options) { if o.Fleet == nil { o.Fleet = s } - take := takeover{s.Continuer(cellstore.EngineFor(""), syncsetup.TakeOptions{ - DeviceName: deviceName(), - RootFor: takeRootFor, - BucketFor: takeCopyBucket, - Notify: surfaceNotices.Say, - })} + take := takeover{chatContinuer(s, surfaceNotices.Say)} if o.Takeover == nil { o.Takeover = take } @@ -236,11 +231,13 @@ func surfaceTaken(got syncsetup.Continued, elapsed time.Duration) tui3.Taken { // already has, found where a chat keeps it, and otherwise a folder in the // bucket of chats that belong to no project, where a chat that arrives from // another machine is at home. -func takeRootFor(id string) string { - if c, err := openCellByID(id); err == nil { +func takeRootFor(id string) string { return takeRootForAt(home.Dir(), id) } + +func takeRootForAt(dir, id string) string { + if c, err := openCellByIDAt(dir, id); err == nil { return c.Root } - return filepath.Join(home.Join("v3", "projects", "copies"), id) + return filepath.Join(dir, "v3", "projects", "copies", id) } // takeCopyBucket files each copy under the folder it came from, apart from @@ -286,3 +283,15 @@ func (t takeover) SafeToContinue(ctx context.Context, id, current string, automa } return !dirty, err } + +// chatContinuer is the shared CLI and app continuation assembly. +func chatContinuer(s *syncsetup.Sync, notify func(string)) *syncsetup.Continuer { + return s.Continuer(cellstore.EngineFor(""), syncsetup.TakeOptions{ + DeviceName: devname.Name(s.Home), + RootFor: func(id string) string { return takeRootForAt(s.Home, id) }, + BucketFor: func(origin string) string { + return filepath.Join(s.Home, "v3", "projects", "copy-"+encodeWorkspace(origin)) + }, + Notify: notify, + }) +} diff --git a/docs/changes/unreleased/0-app-relay-doors.md b/docs/changes/unreleased/0-app-relay-doors.md new file mode 100644 index 0000000000..16716e8c55 --- /dev/null +++ b/docs/changes/unreleased/0-app-relay-doors.md @@ -0,0 +1,153 @@ +--- +kind: added +title: Desktop engines pair computers and continue chats from other computers +pr: 0 +surface: [engine, chat, docs] +invalidates: + - "Chat inventory could miss its first online rows when startup device publication connected the shared watch first; it now reconciles an already connected watch when subscribing." + - "A failed pairing-time computer record previously waited for a synced chat; paired engines now publish at startup and retry independently of chats." + - "A readable computer roster previously failed when its separate presence request failed; the roster now remains available with unknown presence." + - "Native relay activation previously flushed unsupported Windows directory handles; it now preserves regular-file syncs and uses platform-aware directory sync." + - "Windows raft-store paths previously became SQLite URI authorities; drive letters now stay in absolute file paths, allowing native store activation." + - "Recovered job checkpoints previously left stale process files running and reopened background chats; a running record whose ending the checkpoint already holds no longer counts as unfinished, and the next held recovery writes that ending." + - "Conversation model identities previously stayed in a local index, letting a take back reuse older model slots; the saved main and fallback now travel with the chat." + - "Preparing an unavailable incoming main model previously recorded a generic change and its specific reason; the change now records and publishes only the specific reason." + - "A missing local default previously refused a take after acquiring and installing the chat; taking now completes and the first send explains model setup without a provider call." + - "A restarted relay previously reported unobserved paired computers offline immediately; presence now remains unknown for one default presence window unless an observed connection actually lapses." + - "An open desktop view previously lost its writer to the idle sweep and read calls acquired a new fence; explicitly open views now renew the same hold until closed." + - "An expired own hold previously rejected newer locally saved work after a relay outage; unchanged ownership now renews and publishes that work." + - "A failed renewal of an expired own hold previously blocked local work; unreachable renewals now leave outage turns writable and saved for retry." + - "A taken chat previously sent an unavailable provider model to the receiver's router; it now checks provider readiness and explains its usable local default." + - "The desktop engine previously required terminal pairing; it now serves all four pairing modes with explicit questions and cancellation." + - "The app engine previously exposed only held-chat location; it now lists chats on other computers and conditionally continues them here." + - "Finished cell chats previously waited for another turn before desktop publication; paired app starts and pairing completion now offer them through the shared background publisher." + - "Desktop inventory previously inferred offline from an expired chat hold; offline wording now requires an explicit offline presence fact." + - "Desktop saved times previously came from relay publication; new publications carry the sealed head's turn time, and takeover text leaves dates and ages to the desktop." + - "Desktop inventory reads previously emitted chat changes; list calls now stay silent and the follower compares actual last-emitted content." + - "Synced model choices previously applied without a usable local provider; skipped choices now stay skipped until a newer choice arrives." + - "Opening an existing local hold previously acquired another fence; startup and idle ownership checks now renew an unchanged hold." + - "Presence refreshes previously ignored the reconnect grace period and could emit older last-seen times; queries now agree with watch events." + - "Excluding takeover staging folders from desktop admission did not release the store daemon's directory watcher; fetched copies now finish materialization in a one-shot process before installation." + - "Following an installed chat previously spawned the engine inside its workspace; empty or long Windows copy paths now use an existing local store directory and pass the workspace explicitly." + - "Taking a project chat back on Windows could fail restoring receipts beyond MAX_PATH; the engine now receives an extended-length composed-directory path." +--- +Paired engines publish their own computer record at startup, after pairing and +after rename, retrying failures with waits capped at sixty seconds. Unchanged +names and capabilities cause no repeated writes. Computer changes share the +existing directory follower and emit only when the displayed roster changes. +Pairing instructions name Pair a computer and Enter a code from the other +computer. Approval includes the other computer's platform. A successful pairing +does not warn about a deferred record write, and rename receipts distinguish a +published name from one waiting for the relay. + +Native relay unavailability logs its underlying local error through the relay +logger, once per cause per minute with bounded retained causes. Activation, +raft-store creation and transport generations preserve file syncs and atomic +replacement while avoiding unsupported directory flushes on Windows. HTTP +responses remain unchanged. Windows raft-store filenames also retain their +drive letters as absolute URI paths rather than invalid URI authorities. + +The additive app wire reports the resolved Relay address and source, retains the +last pairing operation, and sends device and sync changes after pairing. Joining +with existing chats asks before replacement and requests an engine restart when +the computer's shared keys change. Live work blocks joining. + +One shared inventory follower supplies versioned chat changes and removals; +presence-only changes reuse cached rows. A chat subscriber also reconciles the +shared watch's existing connected state, so startup device publication cannot +leave its first online rows silent. Local agent list and open responses +use cached locations without waiting for relay reads. Continue here uses the engine's observed +computer and fence and returns a local app agent. A stale choice reports the +engine's lost-race sentence. Older home-folder chats remain local. Kept branches +and remote crew membership are outside this inventory. + +Desktop pairing uses app sentences and labelled choices while keeping option +ids stable. An idle open chat says open on its computer, without claiming a +turn is running. Move failures use a person sentence with the technical cause +in error detail. After a failed installation, the released lease names its last +holder but does not make a missing local copy appear here. An unchanged source +chat can acquire a newer fence on its next send; fences are never rolled back. + +Synced preferences now notify desktop Settings once with its full current value. +Identical computer lists no longer repeat Devices events. Models whose catalog +disallows reasoning receive no reasoning parameter. An unlisted model's specific +thinking refusal gets one immediate repair without reasoning, remembered for the +process; a permanent capability refusal does not retain a retryable app turn. +Private takeover staging folders are excluded from app listing and automatic +recovery, preventing copied working presence from opening them before installation. + +Startup publication preserves its signed quiet release through the native relay's +presence wrapper, leaving no live lease or unauthorized release. Unknown computer +presence and unknown saved-turn times are omitted. A closed chat makes no claim +that its computer is offline; unchanged chat rows stay quiet across inventory reads. +Turn-time metadata is sent only when the relay advertises support, preserving +publication on older relays. Historical records are not backfilled at startup. + +Incoming model slots use the engine's resolved provider credentials and local +connections. An unavailable provider leaves the local choice and its edit clock +alone; the skipped revision is remembered locally, preventing sync flaps and +retroactive adoption after provider setup. Effort and limits still travel. + +An idle holder renews its existing fence through watch reconnects and resumes. +Continue here accepts a newer fence from the same holder only when the saved head +is unchanged; different holders and newer heads remain conflicts. Older relays +retain their exact-fence wire. Startup device events wait for the complete roster, +all device emitters compare their last value, and last-seen times never regress. +Presence queries honor the watch's reconnect grace period. Reconnecting without +changing a chat row emits no chat change. A completed turn after takeover is +published promptly while its holder stays open. +Incoming copies also finish materialization in a one-shot store process, releasing +its directory watcher before Windows renames the staging folder into place. The +engine follows the installed folder for later edits and saves. Failed installations +retain the existing lease release behavior. +Follow and later dirty checks pass the installed tree as an engine argument, +running from the local store so long Windows copy paths can be rebound. Missing +empty workspaces are created before binding; incoming chats use their receiver +workspace instead of an unavailable absolute path recorded on the sender. +Taking a project chat back on Windows also restores composed turn receipts whose +full paths exceed MAX_PATH, using the native extended-length path spelling. +Guarded materialization preserves caller-supplied non-daemon transports, including +their binaries and runners. Shared-daemon routes still finish in a one-shot +process before installation so no staging-tree watcher survives the move. + +Relay recovery renews an unchanged own hold even when locally saved work is newer +than the relay's head. Failed uploads retain the existing bounded, single-flight +retry loop; neither a failed renewal nor an expired own lease says this computer +continued the chat elsewhere. Another computer's successful continuation still +fences this window and preserves its pending work. +Incoming chats reuse the shared provider-readiness check before choosing a local +default, explain the change once, and apply the existing conversation model-switch +rules. A computer without a usable default gets a plain setup reason. +The saved model choice also identifies its provider before any call, so a newly +selected custom alias cannot fall through to a different computer's router. +Taking a chat without a usable default now returns its local agent and reconciles +the chat row's local id immediately. The first send gives a plain model setup +reason once. After a usable default is set, the next send uses it and explains +the change once. Actionable take reasons stay in the visible error message; +technical detail has no repeated engine prefixes. +Native and hosted relays omit unobserved computers for their first 75 seconds +of presence memory, while still reporting observed connection lapses. Restarting +the relay cannot briefly label a continuously running peer offline. + +An explicitly open desktop view keeps its hold through idle sweeps; heartbeats +renew the same fence without row churn. Conversations opened only for background +work still release after ten idle minutes and re-acquire when work needs them. +Hold receipts also +require a nonempty home, so a misconfigured sync cannot write relative lock files +into the source tree. The stale-drive refusal fixture now uses a temporary home. + +Desktop relay cancellation goroutines now use the shared panic guard. Inventory, +pairing, continuation, device publication and drive retry critical sections defer +their unlocks while retaining their existing lock order and work boundaries. + +A running process record whose ending the task checkpoint already holds no longer +counts as unfinished; that check only reads, so a chat held elsewhere is never +written to. Unfinished or ambiguous work still enters recovery, which durably settles +both the task checkpoint and process history. Idle release no longer triggers a +spurious conversation open, watch reconnect, presence update or settled-vault send. + +Taking a chat back replaces its remembered local model with the installed saved +choice. Main and fallback travel together; a usable main stays selected silently, +an unavailable main uses the receiver's usable default with one explanation, and +a missing usable default retains the existing setup sentence. Index rebuilding +preserves the saved choice instead of reinstating the journal's original model. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index 9eea63be48..7525ac307d 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -50,6 +50,7 @@ decides which combinations are accepted. | `agents.manual` | `{client_id, id, question}` | `TurnResult` | | `agents.move_to_closed` | `in.tidyIDs` | `Empty` | | `agents.move_to_task` | `{id, turn}` | `MoveResult` | +| `agents.take` | `in.TakeParams` | `TakeResult` | | `agents.open` | `in.IDParams` | `agentResult` | | `agents.queue` | `in.sayTo` | `queueResult` | | `agents.rename` | `{id, title}` | `agentResult` | @@ -186,6 +187,21 @@ decides which combinations are accepted. | `deliverables.remove` | `{agent, id}` | `Empty` | | `deliverables.snapshot` | `{agent, path, process, url}` | `Deliverable` | +## chats methods + +| method | params | result | +|---|---|---| +| `chats.list` | `{}` | `ChatsResult` | + +## pair methods + +| method | params | result | +|---|---|---| +| `pair.start` | `in.PairStartParams` | `PairResult` | +| `pair.get` | `in.PairGetParams` | `PairGetResult` | +| `pair.answer` | `in.PairAnswerParams` | `PairResult` | +| `pair.cancel` | `in.IDParams` | `PairResult` | + ## devices methods | method | params | result | @@ -587,6 +603,9 @@ decides which combinations are accepted. | `agent.changed` | `AgentChanged` | | `ask.closed` | `AskClosed` | | `ask.opened` | `AskOpened` | +| `chat.changed` | `ChatChanged` | +| `chat.removed` | `ChatRemoved` | +| `pair.changed` | `PairResult` | | `collection.changed` | `CollectionChanged` | | `compaction.ended` | `CompactionEnded` | | `crew.changed` | `CrewChanged` | @@ -697,3 +716,68 @@ programs. A client uses the available engine methods for its own actions. | `/update` | `/upgrade` | `/update`; `/update ` | | `/wall` | none | `/wall` | | `/workspace` | none | `/workspace`; `/workspace ` | + +## Relay, pairing and cross-computer chats + +`devices.pair` advertises all four pairing modes: `offer_code`, `use_code`, +`request_link` and `approve_link`. It is absent when the native engine is +unavailable. `pair.start` starts one operation; a concurrent start is `busy`. +Questions carry engine text and explicit options; copy their ids to +`pair.answer`. `pair.cancel` waits for cleanup. `pair.get` without an id returns +this process's current or last finished operation. Every snapshot has a lines +array, including when empty. Final states emit exactly once. Joining with local +chats first asks `replace_own_chats`; joining with live work is `busy`. A join +that changes this computer's shared keys finishes with `restart:true`: restart +the engine before sending work. Device and sync snapshots precede done. + +`sync.relay` advertises `Sync.relay`: resolved URL, source (`environment`, +`paired`, `default`, `none`) and `off`. The source follows CODEAF_SYNC_URL, the +saved pairing URL, then the hosted default. Off omits the URL. + +`chats.take` advertises inventory, events and conditional continuation even +when sync is off or unavailable; in those modes `chats.list` returns `[]`. +Never-paired homes also return `[]` without contacting a relay. The first +inventory read starts one shared Sync.Follow subscription; later version +changes reread and diff rows. Presence changes update cached online facts +without a directory read. No inventory polling timer runs. Kept branches are +outside this v1 inventory. Crew is omitted because relay rows carry no crew +membership; the local Agent keeps its existing crews. + +`chats.list` returns a fresh inventory without emitting `chat.changed` or +`chat.removed`. The follower compares with its last emitted rows, including +when a list has already read a newer version. Identical content at the same +version is never emitted again; changed presence can update that version. +An unset presence reading omits `online`, rather than implying `false`. +The `off` status describes an expired chat hold, independently of computer +presence. Only a known `online:false` permits an offline sentence. +`saved_at`, when known, is the sealed head's turn time. The directory stores +that fact separately from its `durable_at` publication clock; older records +without a known turn time omit it. The desktop formats it locally. The +`takeover_line` contains only holder or pending-work facts, with no date or age. +Directory clients send the optional turn time only after the relay advertises +`Codeaf-Saved-Turn-Times: 1`; older relays keep the compatible publication wire +and rows omit the unknown time. Existing records are not backfilled by startup. + +`local_agent` is the app's saved/open folder id, resolved independently from +`cell`. Remote-only chats have no local agent. Cell locations include the +observed computer, fence and optional saved `head` in opaque `seen`, plus `can_take` and the engine's +`takeover_line` when known. Older chats working in the home folder stay local. +Copy `cell` and `seen` verbatim to `agents.take`; never open an agent using the +remote cell id. It returns the adopted local Agent through the normal open path. +Stale observations answer `conflict`, `detail:{kind:"holder_changed"}` and the +engine's lost-race sentence; there is no automatic retry. Ownership changes +also update an idle open agent's location. Unknown online facts are absent. +A newer fence from the same holder is accepted when `seen.head` still matches; +a different holder or saved head conflicts. The relay advertises atomic head +checks with `Codeaf-Head-Takes: 1`; older relays retain exact-fence requests. +Watch reconnects retain fresh presence, last-seen times never regress, and an +unchanged row emits no chat event. Device events wait for the complete roster +and compare the full last-emitted value across every emitter. + +`agents.open` keeps the conversation open until explicit close, including while +its tab is hidden. Its idle relay hold renews through heartbeats with the same +fence, without release/acquire or `chat.changed` for unchanged content. Calls that +read the open agent keep using that conversation. Conversations opened only for +background work retain the ten-minute idle release, checked every 30 seconds; +new work reopens through existing admission. Running turns, unanswered questions, +established open crew holds and kept services retain their conversations. diff --git a/docs/ux-pairing-contract.md b/docs/ux-pairing-contract.md index a847540a86..eba4fff45a 100644 --- a/docs/ux-pairing-contract.md +++ b/docs/ux-pairing-contract.md @@ -182,8 +182,12 @@ Any paired device may approve or deny; the first decision wins. "dev_B": { "online": false, "last_seen": 1789999000000 } } } ``` -One entry per non-revoked device in the committed directory, combined with advisory socket -presence. Authorization and directory rows pass a fresh majority read; sampling current committed +One entry per non-revoked device whose advisory socket presence is known. After presence +memory starts or restarts, an unobserved device is omitted for one default presence window +(75 seconds), then reads offline if it has not returned. An observed socket's actual lapse +still reports offline during that window. Both native and hosted relays apply this rule; +stored `last_seen` alone is not evidence that a computer stopped during a relay restart. +Authorization and directory rows pass a fresh majority read; sampling current committed time can also write coordination state. Answers with `Codeaf-Dir-Version` like `/v1/dir/list`. Compatibility clients use this on app open or when their watch is down. A joined Home follows its local applied learner for display hints; taking or publishing still requires fresh authority. diff --git a/internal/appwire/chats.go b/internal/appwire/chats.go new file mode 100644 index 0000000000..cf6438fd73 --- /dev/null +++ b/internal/appwire/chats.go @@ -0,0 +1,98 @@ +package appwire + +import "time" + +const ( + FeatureChatsTake = "chats.take" + FeatureSyncRelay = "sync.relay" + EventChatChanged = "chat.changed" + EventChatRemoved = "chat.removed" +) + +type Seen struct { + Device string `json:"device"` + Fence uint64 `json:"fence"` + Head string `json:"head,omitempty"` +} +type ChatRow struct { + Cell string `json:"cell"` + Title string `json:"title"` + LocalAgent string `json:"local_agent,omitempty"` + Here bool `json:"here"` + Holder *LocationHolder `json:"holder,omitempty"` + Online *bool `json:"online,omitempty"` + SavedAt time.Time `json:"saved_at,omitzero"` + Status string `json:"status" enum:"here,running,idle,off"` + Crew string `json:"crew,omitempty"` + Seen Seen `json:"seen"` + CanTake bool `json:"can_take"` + Line string `json:"line,omitempty"` + TakeoverLine string `json:"takeover_line,omitempty"` +} +type ChatsResult struct { + Chats []ChatRow `json:"chats"` + Version uint64 `json:"version"` +} +type ChatChanged struct { + Chat ChatRow `json:"chat"` + Version uint64 `json:"version"` +} +type ChatRemoved struct { + Cell string `json:"cell"` + Version uint64 `json:"version"` +} +type TakeParams struct { + Cell string `json:"cell"` + Seen Seen `json:"seen"` +} +type TakeResult struct { + Agent Agent `json:"agent"` +} +type HolderChanged struct { + Kind string `json:"kind" enum:"holder_changed"` +} +type ChatsTaker interface { + ListChats() (ChatsResult, error) + TakeAgent(TakeParams) (Agent, error) +} +type Relay struct { + URL string `json:"url,omitempty"` + Source string `json:"source" enum:"environment,paired,default,none"` + Off bool `json:"off"` +} +type RelayReporter interface{ RelayKnown() bool } + +func withChatsFeatures(h Hello, e Engine) Hello { + if _, ok := e.(ChatsTaker); ok { + h = withFeature(h, FeatureChatsTake) + } + if d, ok := e.(RelayReporter); ok && d.RelayKnown() { + h = withFeature(h, FeatureSyncRelay) + } + return h +} +func init() { + methods["chats.list"] = typed(func(e Engine, _ struct{}) (*ChatsResult, error) { + d, ok := e.(ChatsTaker) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not list chats on other computers") + } + got, err := d.ListChats() + got.Chats = nonNil(got.Chats) + return &got, err + }) + methods["agents.take"] = typed(func(e Engine, p TakeParams) (*TakeResult, error) { + d, ok := e.(ChatsTaker) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not continue chats here") + } + if _, err := required("cell", p.Cell); err != nil { + return nil, err + } + if p.Seen.Device == "" { + return nil, Errorf(CodeBadParams, "seen is required") + } + got, err := d.TakeAgent(p) + return &TakeResult{Agent: got}, err + }) +} diff --git a/internal/appwire/events.go b/internal/appwire/events.go index f9a0b23105..5b74c25835 100644 --- a/internal/appwire/events.go +++ b/internal/appwire/events.go @@ -6,6 +6,9 @@ import "reflect" // other type is a contract break; the test recorders check every event they // see against this table, and the schema is generated from it. var eventShapes = map[string]reflect.Type{ + EventPairChanged: reflect.TypeFor[PairResult](), + EventChatChanged: reflect.TypeFor[ChatChanged](), + EventChatRemoved: reflect.TypeFor[ChatRemoved](), EventSyncChanged: reflect.TypeFor[SyncChanged](), EventHello: reflect.TypeFor[Hello](), EventAgentChanged: reflect.TypeFor[AgentChanged](), diff --git a/internal/appwire/pair.go b/internal/appwire/pair.go new file mode 100644 index 0000000000..df3a9d5f47 --- /dev/null +++ b/internal/appwire/pair.go @@ -0,0 +1,143 @@ +package appwire + +import "time" + +const ( + FeatureDevicesPair = "devices.pair" + EventPairChanged = "pair.changed" +) + +type PairStartParams struct { + Mode string `json:"mode" enum:"offer_code,use_code,request_link,approve_link"` + Code string `json:"code,omitempty"` + Link string `json:"link,omitempty"` + Name string `json:"name,omitempty"` +} +type PairGetParams struct { + ID string `json:"id,omitempty"` +} +type PairAnswerParams struct { + ID string `json:"id"` + Question string `json:"question"` + Option string `json:"option"` +} +type PairOption struct { + ID string `json:"id"` + Label string `json:"label"` +} +type PairQuestion struct { + ID string `json:"id"` + Kind string `json:"kind" enum:"approve,replace_own_chats"` + Title string `json:"title"` + Detail string `json:"detail,omitempty"` + Options []PairOption `json:"options"` +} +type PairError struct { + Code string `json:"code"` + Message string `json:"message"` +} +type Pairing struct { + ID string `json:"id"` + Mode string `json:"mode" enum:"offer_code,use_code,request_link,approve_link"` + State string `json:"state" enum:"preparing,waiting,question,done,failed,cancelled"` + Lines []string `json:"lines"` + Code string `json:"code,omitempty"` + Link string `json:"link,omitempty"` + Words string `json:"words,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitzero"` + Question *PairQuestion `json:"question,omitempty"` + Error *PairError `json:"error,omitempty"` + Restart bool `json:"restart,omitempty"` +} +type PairResult struct { + Pairing Pairing `json:"pairing"` +} +type PairGetResult struct { + Pairing *Pairing `json:"pairing,omitempty"` +} + +type Pairer interface { + StartPair(PairStartParams) (Pairing, error) + GetPair(string) (*Pairing, error) + AnswerPair(PairAnswerParams) (Pairing, error) + CancelPair(string) (Pairing, error) +} + +func withPairFeatures(h Hello, e Engine) Hello { + if d, ok := e.(interface{ PairingAvailable() bool }); ok && !d.PairingAvailable() { + return h + } + if _, ok := e.(Pairer); ok { + h = withFeature(h, FeatureDevicesPair) + } + return h +} +func init() { + methods["pair.start"] = typed(func(e Engine, p PairStartParams) (*PairResult, error) { + d, ok := e.(Pairer) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not pair computers") + } + switch p.Mode { + case "offer_code", "request_link": + case "use_code": + if _, err := required("code", p.Code); err != nil { + return nil, err + } + case "approve_link": + if _, err := required("link", p.Link); err != nil { + return nil, err + } + default: + return nil, Errorf(CodeBadParams, "unknown pairing mode") + } + got, err := d.StartPair(p) + return &PairResult{Pairing: pairingArrays(got)}, err + }) + methods["pair.get"] = typed(func(e Engine, p PairGetParams) (*PairGetResult, error) { + d, ok := e.(Pairer) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not pair computers") + } + got, err := d.GetPair(p.ID) + if got != nil { + copy := pairingArrays(*got) + got = © + } + return &PairGetResult{Pairing: got}, err + }) + methods["pair.answer"] = typed(func(e Engine, p PairAnswerParams) (*PairResult, error) { + d, ok := e.(Pairer) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not pair computers") + } + for field, value := range map[string]string{"id": p.ID, "question": p.Question, "option": p.Option} { + if _, err := required(field, value); err != nil { + return nil, err + } + } + got, err := d.AnswerPair(p) + return &PairResult{Pairing: pairingArrays(got)}, err + }) + methods["pair.cancel"] = typed(func(e Engine, p IDParams) (*PairResult, error) { + d, ok := e.(Pairer) + if !ok { + return nil, Errorf(CodeUnknownMethod, "this engine does not pair computers") + } + if _, err := required("id", p.ID); err != nil { + return nil, err + } + got, err := d.CancelPair(p.ID) + return &PairResult{Pairing: pairingArrays(got)}, err + }) +} + +func pairingArrays(p Pairing) Pairing { + p.Lines = nonNil(p.Lines) + if p.Question != nil { + q := *p.Question + q.Options = nonNil(q.Options) + p.Question = &q + } + return p +} diff --git a/internal/appwire/relay_test.go b/internal/appwire/relay_test.go new file mode 100644 index 0000000000..7cbb911071 --- /dev/null +++ b/internal/appwire/relay_test.go @@ -0,0 +1,46 @@ +package appwire + +import ( + "encoding/json" + "reflect" + "slices" + "testing" +) + +// E-P1/E-C1/E-T1/E-R1: the types, registrations and advertisements are additive. +func TestRelayAppContractShapesAndOldEngines(t *testing.T) { + for _, name := range []string{"pair.start", "pair.get", "pair.answer", "pair.cancel", "chats.list", "agents.take"} { + if !slices.Contains(Methods(), name) || Schema()["methods"].(map[string]any)[name] == nil { + t.Fatalf("missing %s", name) + } + if _, err := methods[name].call(&fakeEngine{}, json.RawMessage(`{}`)); codeOf(err) != CodeUnknownMethod { + t.Fatalf("old %s: %v", name, err) + } + } + h := withPairFeatures(withChatsFeatures(Hello{}, &fakeEngine{}), &fakeEngine{}) + for _, f := range []string{FeatureDevicesPair, FeatureChatsTake, FeatureSyncRelay} { + if slices.Contains(h.Features, f) { + t.Fatalf("old advertises %s", f) + } + } + for name, want := range map[string]reflect.Type{EventPairChanged: reflect.TypeFor[PairResult](), EventChatChanged: reflect.TypeFor[ChatChanged](), EventChatRemoved: reflect.TypeFor[ChatRemoved]()} { + if EventShape(name) != want { + t.Fatalf("%s", name) + } + } + raw, err := json.Marshal(ChatChanged{Chat: ChatRow{Cell: "chat", Title: "title", Status: "idle", Seen: Seen{Device: "device", Fence: 9007199254740993}}, Version: 9}) + if err != nil || !json.Valid(raw) { + t.Fatal(err) + } + // A uint64 must survive the Go wire exactly, including values above 2^53. + var decoded ChatChanged + if err := json.Unmarshal(raw, &decoded); err != nil || decoded.Chat.Seen.Fence != 9007199254740993 { + t.Fatalf("%s: %v", raw, err) + } + defs := Schema()["$defs"].(map[string]any) + for _, name := range []string{"Pairing", "PairQuestion", "ChatRow", "Seen", "Relay", "Location"} { + if defs[name] == nil { + t.Fatalf("missing schema %s", name) + } + } +} diff --git a/internal/appwire/server.go b/internal/appwire/server.go index f14ac79555..c1c865cb30 100644 --- a/internal/appwire/server.go +++ b/internal/appwire/server.go @@ -194,6 +194,8 @@ func (s *Server) Serve(ctx context.Context, in io.Reader, hello Hello, engine En hello = withProcessActionFeature(hello, engine) hello = withMemoryAppliesFeature(hello, engine) hello = withDevicesFeatures(hello, engine) + hello = withPairFeatures(hello, engine) + hello = withChatsFeatures(hello, engine) hello = withFactoryFeature(hello, engine) hello = withFactoryActorFeature(hello, engine) hello = withFactoryJobStepEditFeature(hello, engine) diff --git a/internal/appwire/sync.go b/internal/appwire/sync.go index 18db55c7fc..cef948739a 100644 --- a/internal/appwire/sync.go +++ b/internal/appwire/sync.go @@ -18,6 +18,7 @@ const ( // Sync is this computer's sync setting. Disclosure is present only after a // hosted drive starts and until a surface acknowledges the sentence. type Sync struct { + Relay *Relay `json:"relay,omitempty"` Mode string `json:"mode" enum:"quiet,on,off,unavailable"` Reason string `json:"reason,omitempty"` Disclosure *SyncDisclosure `json:"disclosure,omitempty"` @@ -36,12 +37,16 @@ type SyncChanged struct { Sync Sync `json:"sync"` } -// Location says why this window cannot take new work. Its absence leaves the -// ordinary admission rules in force; S0 never sends here:true. +// Location names the computer that can continue this chat. Its absence leaves +// the ordinary admission rules in force; chats.take also reports local ownership. type Location struct { - Here bool `json:"here"` - Holder *LocationHolder `json:"holder,omitempty"` - Line string `json:"line,omitempty"` + Cell string `json:"cell,omitempty"` + Seen *Seen `json:"seen,omitempty"` + CanTake bool `json:"can_take,omitempty"` + TakeoverLine string `json:"takeover_line,omitempty"` + Here bool `json:"here"` + Holder *LocationHolder `json:"holder,omitempty"` + Line string `json:"line,omitempty"` } type LocationHolder struct { diff --git a/internal/cellstore/binding.go b/internal/cellstore/binding.go index e7a45dadba..41213c68d1 100644 --- a/internal/cellstore/binding.go +++ b/internal/cellstore/binding.go @@ -22,8 +22,7 @@ func (e Engine) bindTarget(ctx context.Context, c cell.Cell, t Target) error { return err } if len(raw) > 0 && string(raw) != t.Tree { - t.CellDir = "" - if _, err := (Spawn{Binary: e.Binary, Run: e.Run}).Do(ctx, t, rebindOp{From: string(raw)}); err != nil { + if err := e.rebind(ctx, t, string(raw)); err != nil { return fmt.Errorf("follow chat folder: %w", err) } } diff --git a/internal/cellstore/dirty.go b/internal/cellstore/dirty.go index 2e9575a758..fce06ae326 100644 --- a/internal/cellstore/dirty.go +++ b/internal/cellstore/dirty.go @@ -15,11 +15,26 @@ import ( // planOp asks what restoring a snapshot would change, and changes nothing. type planOp struct { Snapshot string `json:"snapshot"` + Repo string `json:"-"` } func (planOp) Verb() string { return "plan" } -func (o planOp) Args() []string { return []string{"--json", "rewind", o.Snapshot, "--dry-run"} } +func (o planOp) Args() []string { + return []string{"--json", "rewind", o.Snapshot, "--dry-run", "--repo", o.Repo} +} + +// plan keeps the workspace in an argument so long Windows copy paths never +// become a spawned process's working directory, including on later take-backs. +func (e Engine) plan(ctx context.Context, c cell.Cell, snapshot string) ([]byte, error) { + target := e.target(c) + if err := os.MkdirAll(target.DataDir, 0700); err != nil { + return nil, err + } + tree := target.Tree + target.Tree = target.DataDir + return (Spawn{Binary: e.Binary, Run: e.Run}).Do(ctx, target, planOp{Snapshot: snapshot, Repo: tree}) +} // Dirty says whether the cell's tree holds work its newest sealed state does // not: the engine's own plan for restoring its workspace head lists exactly the @@ -52,8 +67,7 @@ func (e Engine) Dirty(ctx context.Context, c cell.Cell) (bool, error) { if head != nil && sizeOf(rel(c, cell.TranscriptPath)) != head.Receipt.Transcript.End { return true, nil } - target := e.target(c) - out, err := Spawn{Binary: e.Binary}.Do(ctx, target, planOp{Snapshot: "head"}) + out, err := e.plan(ctx, c, "head") if err != nil { return false, fmt.Errorf("dirty: %w", err) } @@ -78,15 +92,33 @@ func workChanged(plan []byte) (bool, error) { return false, nil } -// rebindOp tells the engine that the tree it restored at From now stands at the -// path the verb is run in. +// rebindOp tells the engine that the tree it restored at From now stands at Repo. type rebindOp struct { From string `json:"from"` + Repo string `json:"-"` } func (rebindOp) Verb() string { return "rebind" } -func (o rebindOp) Args() []string { return []string{"--json", "rebind", "--from", o.From} } +func (o rebindOp) Args() []string { + return []string{"--json", "rebind", "--from", o.From, "--repo", o.Repo} +} + +// rebind runs from the local store rather than a moved workspace. Empty trees +// may need creating, and Windows cannot start a process in a long copy path +// even when filesystem operations can open that directory. +func (e Engine) rebind(ctx context.Context, target Target, from string) error { + if err := os.MkdirAll(target.Tree, 0700); err != nil { + return fmt.Errorf("workspace %q: %w", target.Tree, err) + } + if err := os.MkdirAll(target.DataDir, 0700); err != nil { + return err + } + tree := target.Tree + target.Tree, target.CellDir = target.DataDir, "" + _, err := (Spawn{Binary: e.Binary, Run: e.Run}).Do(ctx, target, rebindOp{From: from, Repo: tree}) + return err +} // Follow registers the tree of to as the tree of from, which was restored and // then moved into place. The engine knows a workspace by its path, so without @@ -96,9 +128,8 @@ func (o rebindOp) Args() []string { return []string{"--json", "rebind", "--from" // takeover. func (e Engine) Follow(ctx context.Context, from, to cell.Cell) error { target := e.target(to) - target.CellDir = "" - if _, err := (Spawn{Binary: e.Binary}).Do(ctx, target, rebindOp{From: e.tree(from)}); err != nil { - return fmt.Errorf("follow: %w", err) + if err := e.rebind(ctx, target, e.tree(from)); err != nil { + return fmt.Errorf("follow %q via %q: %w", target.Tree, target.DataDir, err) } if err := os.MkdirAll(e.LocalDir(to), 0700); err != nil { return err diff --git a/internal/cellstore/engine.go b/internal/cellstore/engine.go index 4092d4b148..932d7abc7d 100644 --- a/internal/cellstore/engine.go +++ b/internal/cellstore/engine.go @@ -333,7 +333,7 @@ func (e Engine) cellDir(c cell.Cell) string { if e.workspace(c) == "" { return "" } - return stateDir(c) + return composedPath(stateDir(c)) } // cellDirArg names the directory the engine composes in as, and restores, the diff --git a/internal/cellstore/follow_test.go b/internal/cellstore/follow_test.go new file mode 100644 index 0000000000..39de0cd8d8 --- /dev/null +++ b/internal/cellstore/follow_test.go @@ -0,0 +1,52 @@ +package cellstore + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/cell" +) + +// W4 makes Windows's process-directory limit observable on every platform. +func TestFollowLongWorkspaceW4(t *testing.T) { + root := filepath.Join(t.TempDir(), strings.Repeat("a", 120), strings.Repeat("b", 120)) + to := cell.Cell{ID: "chat", Root: root} + from := cell.Cell{ID: to.ID, Root: root + ".taking"} + tree := canonicalTree(filepath.Join(to.Root, "work")) + if len(tree) < 260 { + t.Fatalf("W4 fixture needs a long workspace, got %d characters", len(tree)) + } + calls := 0 + e := Engine{Binary: "engine", DataRoot: t.TempDir(), WorkspaceOf: func(c cell.Cell) string { + return filepath.Join(c.Root, "work") + }} + e.Run = func(_ context.Context, dir string, _ []string, argv ...string) ([]byte, error) { + calls++ + if len(dir) >= 260 { + return nil, fmt.Errorf("Windows process directory is too long: %s", dir) + } + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + t.Fatalf("W4 process directory %q: %v", dir, err) + } + at := slices.Index(argv, "--repo") + if at < 0 || at+1 == len(argv) || argv[at+1] != tree { + t.Fatalf("W4 workspace argument: %q, want %q", argv, tree) + } + return []byte(`{"changes":[]}`), nil + } + if err := e.Follow(t.Context(), from, to); err != nil { + t.Fatal(err) + } + if dirty, err := e.Dirty(t.Context(), to); err != nil || dirty { + t.Fatalf("W4 dirty=%v: %v", dirty, err) + } + if calls != 2 { + t.Fatalf("W4 engine calls = %d, want rebind and dirty plan", calls) + } +} diff --git a/internal/cellstore/localwork.go b/internal/cellstore/localwork.go index 7ff97d4a38..c4a45cf8fb 100644 --- a/internal/cellstore/localwork.go +++ b/internal/cellstore/localwork.go @@ -55,7 +55,7 @@ func (e Engine) LocalWork(ctx context.Context, c cell.Cell) (bool, error) { } // The shared snapshot belongs to this chat. A workspace's latest head can // belong to another chat that opened in the same working folder. - out, err := (Spawn{Binary: e.Binary}).Do(ctx, e.target(c), planOp{Snapshot: base}) + out, err := e.plan(ctx, c, base) if err != nil { return false, err } diff --git a/internal/cellstore/log.go b/internal/cellstore/log.go index ee9050092e..38012baa34 100644 --- a/internal/cellstore/log.go +++ b/internal/cellstore/log.go @@ -25,6 +25,26 @@ type Entry struct { IsAdopted bool } +// SavedAt returns the seal time of the named head. A received head whose turn +// record is unavailable has no known time; copying it must not invent one. +func SavedAt(c cell.Cell, head string) (int64, error) { + last, err := lastTurn(rel(c, TurnsPath)) + if err != nil || last == nil { + return 0, err + } + if last.ID == head { + return last.SealedAtMs, nil + } + // Publication may be uploading an earlier head while another turn seals. + turns, err := Turns(c) + for i := len(turns) - 1; i >= 0; i-- { + if turns[i].ID == head { + return turns[i].SealedAtMs, err + } + } + return 0, err +} + // Turns reads the cell's whole chain, oldest first. A cell that was never // sealed has an empty chain. func Turns(c cell.Cell) ([]Turn, error) { diff --git a/internal/cellstore/planpath_unix.go b/internal/cellstore/planpath_unix.go index ef8214d376..4e4cf5cd7a 100644 --- a/internal/cellstore/planpath_unix.go +++ b/internal/cellstore/planpath_unix.go @@ -5,3 +5,5 @@ package cellstore import "path/filepath" func planRelative(root, physical string) (string, error) { return filepath.Rel(root, physical) } + +func composedPath(dir string) string { return dir } diff --git a/internal/cellstore/planpath_windows.go b/internal/cellstore/planpath_windows.go index 978183ba87..838d47b711 100644 --- a/internal/cellstore/planpath_windows.go +++ b/internal/cellstore/planpath_windows.go @@ -18,3 +18,17 @@ func planPath(path string) string { } return strings.TrimPrefix(path, `\\?\`) } + +// composedPath keeps receipt destinations usable by the engine's native +// replacement calls. A short .cell directory can have children beyond MAX_PATH. +// Unlike the workspace, furrow does not canonicalize this composed path itself. +func composedPath(dir string) string { + dir = canonicalTree(dir) + if strings.HasPrefix(dir, `\\?\`) { + return dir + } + if strings.HasPrefix(dir, `\\`) { + return `\\?\UNC\` + dir[2:] + } + return `\\?\` + dir +} diff --git a/internal/cellstore/syncops.go b/internal/cellstore/syncops.go index 3175b64bf0..7a53b6a650 100644 --- a/internal/cellstore/syncops.go +++ b/internal/cellstore/syncops.go @@ -346,7 +346,33 @@ func (e Engine) Sync(keys SyncKeys, ledger string) SyncEngine { if err := e.screenCapture(ctx, c); err != nil { return nil, err } - return e.do(ctx, c, e.cellDir(c), op) + // The daemon retains a directory watcher after materializing a tree. + // A take renames that tree next, which Windows refuses while watched. + // Join a one-shot process before returning; Follow binds the final root. + materializer := e + materializer.Transport = materializeTransport(e.transport()) + return materializer.do(ctx, c, e.cellDir(c), op) }, } } + +// materializeTransport releases daemon watchers before installation while +// preserving a caller's independent transport, binary and runner. +func materializeTransport(transport Transport) Transport { + switch t := transport.(type) { + case Daemon: + return Spawn{Binary: t.Binary, Socket: t.Socket} + case *Daemon: + return materializeTransport(*t) + case Fallback: + switch t.Primary.(type) { + case Daemon, *Daemon: + return materializeTransport(t.Secondary) + } + return Fallback{Primary: materializeTransport(t.Primary), Secondary: materializeTransport(t.Secondary)} + case *Fallback: + return materializeTransport(*t) + default: + return transport + } +} diff --git a/internal/cellstore/syncops_test.go b/internal/cellstore/syncops_test.go index 3fd4fdbe18..b943f291c0 100644 --- a/internal/cellstore/syncops_test.go +++ b/internal/cellstore/syncops_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "path/filepath" "reflect" "strings" "testing" @@ -190,6 +191,45 @@ func TestSyncEngineSpawnLeavesKeylessVerbsWithoutKeys(t *testing.T) { } } +func TestSyncMaterializePreservesInjectedSpawn(t *testing.T) { + for _, tc := range []struct { + name string + wrap func(Spawn) Transport + }{ + {"spawn", func(s Spawn) Transport { return s }}, + {"non-daemon fallback", func(s Spawn) Transport { + return Fallback{Primary: s, Secondary: &fixedTransport{err: errors.New("unexpected fallback")}} + }}, + {"daemon fallback", func(s Spawn) Transport { + return Fallback{Primary: Daemon{Binary: "unused-daemon"}, Secondary: s} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + c := cell.Cell{ID: "c1", Root: root} + var argv []string + var dir string + e := Engine{ + DataRoot: t.TempDir(), + Transport: tc.wrap(Spawn{Binary: "injected-engine", Run: func(_ context.Context, cwd string, _ []string, args ...string) ([]byte, error) { + dir, argv = cwd, args + return []byte(`{}`), nil + }}), + } + if err := e.Sync(SyncKeys{}, "ledger").Materialize(t.Context(), c, rid); err != nil { + t.Fatal(err) + } + want := []string{"injected-engine", "--json", "materialize", "--head", rid} + if len(argv) < len(want) || !reflect.DeepEqual(argv[:len(want)], want) || dir != c.Root { + t.Fatalf("injected spawn = %v in %q, want %v in %q", argv, dir, want, c.Root) + } + }) + } +} + func TestSyncEngineExportArgsSpellTheContract(t *testing.T) { op := exportOp{Head: "h", Outbox: "/o", Ledger: "l", MaxFrame: 5} want := []string{"--json", "export", "--head", "h", "--outbox", "/o", "--ledger", "l", "--max-frame", "5"} diff --git a/internal/cellsync/branch.go b/internal/cellsync/branch.go index a40f7b92d7..7c9696e2b0 100644 --- a/internal/cellsync/branch.go +++ b/internal/cellsync/branch.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sync" + "github.com/Agent-Field/codeaf/internal/cellstore" "github.com/Agent-Field/codeaf/internal/directory" ) @@ -33,6 +34,10 @@ type Brancher struct { // not. What an earlier refused publish already uploaded is reused, because // objects are content-addressed. func (b Brancher) Branch(ctx context.Context, from *Driving, head string, turns uint32, in PublishInfo) (string, error) { + savedAt, err := cellstore.SavedAt(from.Cell, head) + if err != nil { + return "", err + } _, frames, err := b.Publisher.uploadPlan(ctx, from.Cell, head) if err != nil { return "", err @@ -47,6 +52,7 @@ func (b Brancher) Branch(ctx context.Context, from *Driving, head string, turns } _, err = b.Dir.Create(ctx, id, directory.CellInit{ Head: head, Class: in.Class, Title: in.Title, Size: in.Size, Keys: in.Keys, + SavedAt: savedAt, ParentCell: from.ID(), OrphanTurns: turns, Frames: plan, }) if err != nil { diff --git a/internal/cellsync/ownership.go b/internal/cellsync/ownership.go index 8ef97b9d27..4f461ad09e 100644 --- a/internal/cellsync/ownership.go +++ b/internal/cellsync/ownership.go @@ -1,5 +1,12 @@ package cellsync +import ( + "context" + "time" + + "github.com/Agent-Field/codeaf/internal/directory" +) + // Lease names the directory record and fence this batcher publishes under. // After keeping orphaned turns, that record can be a branch rather than the // local cell's ID; admission must compare the same lease as publishing does. @@ -7,3 +14,33 @@ func (b *Batcher) Lease() (id string, fence uint64) { d, _, _ := b.view() return d.ID(), d.Fence } + +// Reacquire resumes an unchanged saved chat after its lease became free. The +// conditional acquisition cannot displace a winner that arrived after the read. +// Serializing with publishes keeps an old flush from replacing the new fence. +func (b *Batcher) Reacquire(ctx context.Context, seen directory.CellView) (bool, error) { + b.flushMu.Lock() + defer b.flushMu.Unlock() + d, pending, _ := b.view() + if b.isClosing() || pending != 0 || d.Head != seen.Cell.Head || seen.Cell.Lease.Expires > seen.Now { + return false, nil + } + v, err := b.Publisher.Dir.Acquire(ctx, d.ID(), directory.AcquireOpts{ + RequireConditional: true, + Expect: &directory.LeaseExpectation{Device: seen.Cell.Lease.Device, Fence: seen.Cell.Lease.Fence}, + }) + if err != nil { + return false, err + } + if v.Cell.Head != d.Head { + return false, b.Publisher.Dir.Release(ctx, d.ID(), v.Cell.Lease.Fence) + } + d.Fence, d.Frames = v.Cell.Lease.Fence, v.Cell.Frames + b.mu.Lock() + *b.Driving = d + b.stale, b.renewalEnded = false, false + b.renewalSince = time.Time{} + b.mu.Unlock() + b.renewalChanged() + return true, nil +} diff --git a/internal/cellsync/ownership_recovery_test.go b/internal/cellsync/ownership_recovery_test.go new file mode 100644 index 0000000000..f8684aeb33 --- /dev/null +++ b/internal/cellsync/ownership_recovery_test.go @@ -0,0 +1,60 @@ +package cellsync + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/directory" +) + +func TestReacquireReleasedLeaseS5(t *testing.T) { + for _, scenario := range []string{"released", "new winner", "new head", "pending work"} { + t.Run(scenario, func(t *testing.T) { + r := newRig(t) + r.publishFirst(map[string]string{"file": "saved"}) + b := r.batcher() + r.takeOver() + other := r.dir.For(devB) + v, err := other.Cell(t.Context(), cellID) + if err != nil { + t.Fatal(err) + } + if scenario == "new head" { + r.bPublishes(v.Cell.Head) + } + if err = other.Release(t.Context(), cellID, v.Cell.Lease.Fence); err != nil { + t.Fatal(err) + } + seen, err := other.Cell(t.Context(), cellID) + if err != nil { + t.Fatal(err) + } + if scenario == "new winner" { + if _, err = other.Acquire(t.Context(), cellID, directory.AcquireOpts{}); err != nil { + t.Fatal(err) + } + } + if scenario == "pending work" { + note(b, r.seal(map[string]string{"file": "not yet published"})) + } + resumed, err := b.Reacquire(t.Context(), seen) + if scenario != "released" { + if resumed { + t.Fatal("S5 resumed stale, raced or unpublished work") + } + return + } + if err != nil || !resumed { + t.Fatalf("S5 resume: %v, %v", resumed, err) + } + got := r.head(cellID) + if got.Lease.Device != devA || got.Lease.Fence != seen.Cell.Lease.Fence+1 { + t.Fatalf("S5 lease: %+v", got.Lease) + } + note(b, r.seal(map[string]string{"file": "after failed move"})) + b.flush(t.Context()) + if got := r.head(cellID); got.Head != r.drv.Head || b.Pending() != 0 { + t.Fatalf("S5 resumed publication: %+v", got) + } + }) + } +} diff --git a/internal/cellsync/publish.go b/internal/cellsync/publish.go index a33587db66..22b8d294d3 100644 --- a/internal/cellsync/publish.go +++ b/internal/cellsync/publish.go @@ -8,6 +8,7 @@ import ( "github.com/Agent-Field/codeaf/internal/blobstore" "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" "github.com/Agent-Field/codeaf/internal/directory" ) @@ -50,6 +51,7 @@ func (d Driving) ID() string { // PublishInfo is what a publish tells the directory about the cell. type PublishInfo struct { hints []directory.FrameHint + savedAt int64 Class, Title string Size uint64 Pending uint32 @@ -66,6 +68,11 @@ func (p *Publisher) Publish(ctx context.Context, d *Driving, head string, in Pub // publish is Publish that also reports what was uploaded. func (p *Publisher) publish(ctx context.Context, d *Driving, head string, in PublishInfo) (Export, error) { + var err error + in.savedAt, err = cellstore.SavedAt(d.Cell, head) + if err != nil { + return Export{}, err + } ex, frames, err := p.uploadPlan(ctx, d.Cell, head) if err != nil { return Export{}, err @@ -159,7 +166,8 @@ func (p *Publisher) advance(ctx context.Context, d Driving, head string, in Publ func (p *Publisher) move(ctx context.Context, d Driving, head string, in PublishInfo, frames []string) (uint64, error) { _, err := p.Dir.Publish(ctx, d.ID(), directory.Publish{ Fence: d.Fence, OldHead: d.Head, Head: head, - Size: in.Size, Class: in.Class, Title: in.Title, Pending: in.Pending, + SavedAt: in.savedAt, + Size: in.Size, Class: in.Class, Title: in.Title, Pending: in.Pending, Frames: frames, FrameHints: in.hints, }) return d.Fence, refusal(err) @@ -187,7 +195,8 @@ func (p *Publisher) resume(ctx context.Context, d Driving, head string, in Publi func (p *Publisher) create(ctx context.Context, d Driving, head string, in PublishInfo, frames []string) (uint64, error) { v, err := p.Dir.Create(ctx, d.ID(), directory.CellInit{ Head: head, Class: in.Class, Title: in.Title, Size: in.Size, Keys: in.Keys, - Frames: frames, FrameHints: in.hints, + SavedAt: in.savedAt, + Frames: frames, FrameHints: in.hints, }) return v.Cell.Lease.Fence, err } diff --git a/internal/cellsync/refusal.go b/internal/cellsync/refusal.go index 6b39638104..779c5501a6 100644 --- a/internal/cellsync/refusal.go +++ b/internal/cellsync/refusal.go @@ -1,11 +1,13 @@ package cellsync import ( + "context" "errors" "time" "github.com/Agent-Field/codeaf/internal/blobstore" "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -14,6 +16,13 @@ import ( // never trusted past an hour, so a wrong clock on the relay cannot silence sync. const MaxHalt = time.Hour +// Unreachable identifies transport failures that can recover without a person's +// intervention. They leave local work writable and use the bounded retry path. +func Unreachable(err error) bool { + return errors.Is(err, directory.ErrUnreachable) || errors.Is(err, blobstore.ErrUnreachable) || + errors.Is(err, wireauth.ErrUnavailable) || errors.Is(err, context.DeadlineExceeded) +} + // Refusal is one way the relay says no: the sentinel the wire names it by, the // one plain sentence the person reads, and how the client carries on. The table // below is the only place these three meet, so a new refusal is one row and no diff --git a/internal/cellsync/renewal.go b/internal/cellsync/renewal.go index 15bc5dd68b..dd8eb98f92 100644 --- a/internal/cellsync/renewal.go +++ b/internal/cellsync/renewal.go @@ -4,7 +4,6 @@ import ( "errors" "time" - "github.com/Agent-Field/codeaf/internal/blobstore" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -29,7 +28,7 @@ func (b *Batcher) renewalResult(seen Driving, err error) { case err == nil: changed = !b.renewalSince.IsZero() b.renewalSince = time.Time{} - case errors.Is(err, directory.ErrUnreachable), errors.Is(err, blobstore.ErrUnreachable), errors.Is(err, wireauth.ErrUnavailable): + case Unreachable(err): if seen.Fence != 0 && b.renewalSince.IsZero() { b.renewalSince, changed = b.now(), true } diff --git a/internal/chatlist/copy.go b/internal/chatlist/copy.go index c8cb1a8785..f4bacefd6c 100644 --- a/internal/chatlist/copy.go +++ b/internal/chatlist/copy.go @@ -9,6 +9,8 @@ import ( // The frozen sentences. Every surface takes its words // from here so a respelling happens once. const ( + // Available describes a free lease while work admission acquires a new fence. + Available = "This chat is available to continue here." OfferContinue = "continue here" // MoveHere is the verb for a chat another device is running right now, and // ContinueVerb for one it let go of; the devices row offers them by name. diff --git a/internal/chatlist/rows.go b/internal/chatlist/rows.go index 180895df88..2143e6d15d 100644 --- a/internal/chatlist/rows.go +++ b/internal/chatlist/rows.go @@ -32,6 +32,7 @@ type Row struct { Status Status DurableAgo time.Duration // Listing.Now - DurableAt, directory clock only DurableAt int64 // directory ms of the last durable turn: the fact DurableAgo is read from + SavedAt int64 // sealed head's time, or zero when unknown Pending, OrphanTurns uint32 // Fence binds the continuation card to the holder the person saw. Fence uint64 @@ -142,6 +143,7 @@ func rowOf(id string, c directory.Cell, l directory.Listing, self string, open O ReleasedAgo: time.Duration(l.Now-c.Lease.ReleasedAt) * time.Millisecond, DurableAgo: time.Duration(l.Now-c.DurableAt) * time.Millisecond, DurableAt: c.DurableAt, + SavedAt: c.SavedAt, Pending: c.Lease.Pending, OrphanTurns: c.OrphanTurns, } diff --git a/internal/clusterstate/records.go b/internal/clusterstate/records.go index 7c3a01c712..83b9c75f87 100644 --- a/internal/clusterstate/records.go +++ b/internal/clusterstate/records.go @@ -74,8 +74,9 @@ type Cell struct { // FrameHints is optional and absent by default; every cluster member must understand it before a device sends it. FrameHints []FrameHint `json:"frame_hints,omitempty"` V uint16 `json:"V"` - Head string `json:"head"` // last durable turn id (hex64) - DurableAt int64 `json:"durable_at"` // directory ms + Head string `json:"head"` // last durable turn id (hex64) + DurableAt int64 `json:"durable_at"` // directory ms + SavedAt int64 `json:"saved_at,omitempty"` // sealed head's time, independent of publication Class string `json:"class"` Size uint64 `json:"size"` ParentCell string `json:"parent_cell,omitempty"` @@ -106,6 +107,7 @@ type CellView struct { // CellInit creates a cell. The calling device holds the new lease at fence 1. type CellInit struct { + SavedAt int64 `json:"saved_at,omitempty"` FrameHints []FrameHint `json:"frame_hints,omitempty"` Head string `json:"head"` Class string `json:"class"` @@ -125,6 +127,7 @@ type Beat struct { // Publish moves a cell's durable head under a lease the caller holds. type Publish struct { + SavedAt int64 `json:"saved_at,omitempty"` FrameHints []FrameHint `json:"frame_hints,omitempty"` Fence uint64 `json:"fence"` OldHead string `json:"old_head"` @@ -149,4 +152,14 @@ type Rotation struct { type LeaseExpectation struct { Device string `json:"device"` Fence uint64 `json:"fence"` + Head string `json:"head,omitempty"` +} + +// Matches allows a newer fence only with evidence of the same holder and +// saved head. Older callers without a head retain the exact-fence contract. +func (e LeaseExpectation) Matches(c Cell) bool { + if c.Lease.Device != e.Device || e.Head != "" && c.Head != e.Head { + return false + } + return c.Lease.Fence == e.Fence || e.Head != "" && c.Lease.Fence > e.Fence } diff --git a/internal/clusterstate/rules.go b/internal/clusterstate/rules.go index 885d11a245..8165fdd44d 100644 --- a/internal/clusterstate/rules.go +++ b/internal/clusterstate/rules.go @@ -25,7 +25,7 @@ func Lifted(c Cell, vouchedUntil int64) Cell { func Acquire(c Cell, device string, now int64, force bool, expected ...*LeaseExpectation) (Cell, error) { if len(expected) > 0 && expected[0] != nil { seen := expected[0] - if c.Lease.Device != seen.Device || c.Lease.Fence != seen.Fence { + if !seen.Matches(c) { return c, ErrLeaseHeld } } @@ -64,6 +64,9 @@ func PublishTo(c Cell, device string, p Publish, now int64) (Cell, error) { c.Frames = p.Frames c.FrameHints = boundedFrameHints(p.FrameHints) c.Lease.Pending, c.DurableAt = p.Pending, now + if p.Head != p.OldHead || p.SavedAt > 0 { + c.SavedAt = p.SavedAt + } c.Lease.Expires = now + ttlMs c.Title = keepIfEmpty(p.Title, c.Title) return c, nil @@ -86,7 +89,7 @@ func ReleaseOf(c Cell, device string, fence uint64, at ...int64) (Cell, error) { // at fence 1. The caller has already checked that the id is free. func Created(in CellInit, device string, now int64) Cell { return Cell{ - V: 1, Head: in.Head, DurableAt: now, Class: in.Class, Size: in.Size, + V: 1, Head: in.Head, DurableAt: now, SavedAt: in.SavedAt, Class: in.Class, Size: in.Size, ParentCell: in.ParentCell, Title: in.Title, Keys: in.Keys, OrphanTurns: in.OrphanTurns, Frames: in.Frames, FrameHints: boundedFrameHints(in.FrameHints), Lease: Lease{Device: device, Fence: 1, Expires: now + ttlMs}, diff --git a/internal/config/modelusable.go b/internal/config/modelusable.go new file mode 100644 index 0000000000..2b186f00f4 --- /dev/null +++ b/internal/config/modelusable.go @@ -0,0 +1,32 @@ +package config + +import ( + "strings" + + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/roles" +) + +// ModelUsable checks resolved local provider readiness for an incoming model. +// providerID, when known, prevents a missing custom alias from falling through +// to the default router. No catalog or network request is needed. +func ModelUsable(sources modelsource.Set, model, providerID string) bool { + model, _ = roles.SplitEffort(model) + written := sources.Written() + for _, source := range modelsource.Vendored() { + // Cloud vendor names can also be OpenRouter author prefixes. + // Local runners and the sign-in transport cannot use that fallback. + if source.KeyOptional || source.ID == "codex" { + written = append(written, source.Written) + } + } + qualifier, _ := modelsource.Split(model, written) + service, _ := sources.For(model) + if providerID != "" && service.Source.ID != providerID && !(modelsource.IsCustomID(providerID) && modelsource.IsCustomID(service.Source.ID)) { + return false + } + if qualifier != "" && !strings.EqualFold(qualifier, service.Source.Written) { + return false + } + return service.HasCredentials() +} diff --git a/internal/config/preferences.go b/internal/config/preferences.go index 7856cfc34c..ee49ee0765 100644 --- a/internal/config/preferences.go +++ b/internal/config/preferences.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "os" + "sync" "time" ) @@ -53,7 +54,8 @@ func ApplyPreferences(dir string, changes map[string]string) error { // ApplyPreferencesIf preserves a setting changed after the sync reader took // its snapshot. The next round captures that newer local edit (C30.3). func ApplyPreferencesIf(dir string, changes, expected map[string]string) error { - return editProfile(dir, "preferences", func(held map[string]json.RawMessage) (profileChange, error) { + var applied []string + err := editProfile(dir, "preferences", func(held map[string]json.RawMessage) (profileChange, error) { change := profileChange{set: map[string]json.RawMessage{}} for k, v := range changes { if expected != nil && string(held[k]) != expected[k] { @@ -75,8 +77,54 @@ func ApplyPreferencesIf(dir string, changes, expected map[string]string) error { change.set[k] = json.RawMessage(v) } } + for k := range change.set { + applied = append(applied, k) + } + applied = append(applied, change.removed...) return change, nil }) + if err == nil && len(applied) > 0 { + preferenceListeners.Lock() + var listeners []func([]string) + for _, listener := range preferenceListeners.byPath[BudgetConfigPath(dir)] { + listeners = append(listeners, listener) + } + preferenceListeners.Unlock() + // Readers may take the profile lock, so notify after the transaction ends. + for _, listener := range listeners { + listener(applied) + } + } + return err +} + +var preferenceListeners = struct { + sync.Mutex + next uint64 + byPath map[string]map[uint64]func([]string) +}{byPath: map[string]map[uint64]func([]string){}} + +// ObserveSyncedPreferences hears successful preference restores in this process. +// Local setting writers keep their existing surface events. Closing stops future +// notifications; a notification already copied for delivery may finish. +func ObserveSyncedPreferences(dir string, changed func([]string)) func() { + path := BudgetConfigPath(dir) + preferenceListeners.Lock() + preferenceListeners.next++ + id := preferenceListeners.next + if preferenceListeners.byPath[path] == nil { + preferenceListeners.byPath[path] = map[uint64]func([]string){} + } + preferenceListeners.byPath[path][id] = changed + preferenceListeners.Unlock() + return func() { + preferenceListeners.Lock() + defer preferenceListeners.Unlock() + delete(preferenceListeners.byPath[path], id) + if len(preferenceListeners.byPath[path]) == 0 { + delete(preferenceListeners.byPath, path) + } + } } // PreferenceTime reads the setting's edit clock, including a removal. Older diff --git a/internal/config/preferences_sync_test.go b/internal/config/preferences_sync_test.go new file mode 100644 index 0000000000..6f550b75d2 --- /dev/null +++ b/internal/config/preferences_sync_test.go @@ -0,0 +1,44 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSyncedPreferenceNotificationsS1(t *testing.T) { + a, b := t.TempDir(), t.TempDir() + var received [][]string + stop := ObserveSyncedPreferences(a, func(keys []string) { + // The notification must run after the profile transaction has unlocked. + if err := WriteDefaultEffort(b, DefaultEffortAt(a)); err != nil { + t.Error(err) + } + received = append(received, keys) + }) + defer stop() + apply := func(dir string, changes, expected map[string]string) { + t.Helper() + if err := ApplyPreferencesIf(dir, changes, expected); err != nil { + t.Fatal(err) + } + } + apply(a, map[string]string{KeyEffort: `"high"`}, nil) + apply(a, map[string]string{KeyEffort: `"high"`}, nil) + apply(a, map[string]string{KeyEffort: `"low"`}, map[string]string{KeyEffort: `"medium"`}) + apply(b, map[string]string{KeyEffort: `"low"`}, nil) + if !reflect.DeepEqual(received, [][]string{{KeyEffort}}) { + t.Fatalf("S1 changed/no-op/race/profile scope: %v", received) + } + if err := ApplyPreferences(a, map[string]string{KeyEffort: `invalid`}); err == nil { + t.Fatal("S1 malformed restore succeeded") + } + apply(a, map[string]string{KeyEffort: ""}, nil) + if len(received) != 2 { + t.Fatalf("S1 removal notifications: %v", received) + } + stop() + apply(a, map[string]string{KeyEffort: `"low"`}, nil) + if len(received) != 2 { + t.Fatal("S1 closed listener notified") + } +} diff --git a/internal/directory/conditional_take_test.go b/internal/directory/conditional_take_test.go index 0474de12fb..e6041ee6ce 100644 --- a/internal/directory/conditional_take_test.go +++ b/internal/directory/conditional_take_test.go @@ -38,6 +38,40 @@ func TestC7OldServiceRefusesTakesAcrossClients(t *testing.T) { } } +func TestHTTPF2HeadGuardRequiresAtomicSupport(t *testing.T) { + for _, supported := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy", true: "current"}[supported], func(t *testing.T) { + takes := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(directory.ConditionalTakeHeader, "1") + if supported { + w.Header().Set(directory.HeadTakeHeader, "1") + } + if strings.HasSuffix(r.URL.Path, "/acquire") { + takes++ + var body directory.AcquireOpts + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + if body.Expect == nil || body.Expect.Head != "saved-head" { + t.Error("F2 head guard was stripped") + } + } + _ = json.NewEncoder(w).Encode(directory.CellView{}) + })) + defer srv.Close() + client := directory.NewHTTP(srv.URL, fakeSign("id_one", devA), srv.Client()) + _, err := client.Acquire(t.Context(), cellX, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: devA, Fence: 1, Head: "saved-head"}}) + if supported && (err != nil || takes != 1) { + t.Fatalf("F2 supported take failed: %v, %d requests", err, takes) + } + if !supported && (!errors.Is(err, directory.ErrConditionalTake) || takes != 0) { + t.Fatalf("F2 unsupported atomic guard was weakened: %v, %d requests", err, takes) + } + }) + } +} + // TestC7OldServiceStillAllowsAnOrdinaryResume proves C7's compatibility // boundary: a local non-forced resume cannot displace a live other holder. func TestC7OldServiceStillAllowsAnOrdinaryResume(t *testing.T) { diff --git a/internal/directory/handler.go b/internal/directory/handler.go index dfe7cc3a26..5c90d8c2b6 100644 --- a/internal/directory/handler.go +++ b/internal/directory/handler.go @@ -62,6 +62,8 @@ type handler struct { func (h *handler) serve(rt route) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(SavedTurnTimeHeader, "1") + w.Header().Set(HeadTakeHeader, "1") c, err := h.admit(w, r) if err != nil { refuse(w, err) diff --git a/internal/directory/http.go b/internal/directory/http.go index c2ab3ffa3d..2bf1932264 100644 --- a/internal/directory/http.go +++ b/internal/directory/http.go @@ -45,6 +45,7 @@ func (h *HTTP) List(ctx context.Context) (l Listing, err error) { hdr, err := h.doHeader(ctx, http.MethodGet, dirBase+"/list", nil, &l) if err == nil { l.Version, _ = strconv.ParseUint(hdr.Get(VersionHeader), 10, 64) + headTakeServices.Store(h.serviceKey(), hdr.Get(HeadTakeHeader) == "1") } return l, err } @@ -53,6 +54,7 @@ func (h *HTTP) Cell(ctx context.Context, id string) (v CellView, err error) { hdr, err := h.doHeader(ctx, http.MethodGet, cellPath(id, ""), nil, &v) if err == nil { conditionalServices.Store(h.serviceKey(), hdr.Get(ConditionalTakeHeader) == "1") + headTakeServices.Store(h.serviceKey(), hdr.Get(HeadTakeHeader) == "1") } return v, err } @@ -70,6 +72,9 @@ func (h *HTTP) SetVault(ctx context.Context, old, next string) error { } func (h *HTTP) Create(ctx context.Context, id string, in CellInit) (v CellView, err error) { + if in.SavedAt != 0 && !h.savedTurnTimes(ctx, id) { + in.SavedAt = 0 + } return v, h.do(ctx, http.MethodPost, cellPath(id, ""), in, &v) } @@ -79,9 +84,31 @@ func (h *HTTP) Acquire(ctx context.Context, id string, o AcquireOpts) (v CellVie return v, err } } + if o.Expect != nil && o.Expect.Head != "" { + if supported, _ := headTakeServices.Load(h.serviceKey()); supported != true { + if _, err := h.Cell(ctx, id); err != nil { + return v, err + } + if supported, _ = headTakeServices.Load(h.serviceKey()); supported != true { + return v, ErrConditionalTake + } + } + } return v, h.do(ctx, http.MethodPost, cellPath(id, "acquire"), o, &v) } +// HeadTakeHeader advertises atomic saved-head checks on conditional takes. +const HeadTakeHeader = "Codeaf-Head-Takes" + +var headTakeServices sync.Map + +// HeadTakes reports the service's last advertised atomic head-check support. +// Callers omit head observations on older relays, preserving their take wire. +func (h *HTTP) HeadTakes() bool { + known, _ := headTakeServices.Load(h.serviceKey()) + return known == true +} + // ConditionalTakeHeader is the service's promise that a take compares the // expected holder and fence atomically. Older services omit it. const ConditionalTakeHeader = "codeaf-conditional-takes" @@ -93,6 +120,23 @@ const ConditionalTakeHeader = "codeaf-conditional-takes" // codeaf runs moves chats at the next attempt instead of after a restart. var conditionalServices sync.Map +// SavedTurnTimeHeader advertises optional sealed-head timestamps. Older +// relays strictly decode signed bodies, so clients omit the field without it. +const SavedTurnTimeHeader = "Codeaf-Saved-Turn-Times" + +var savedTimeServices sync.Map + +func (h *HTTP) savedTurnTimes(ctx context.Context, id string) bool { + if known, ok := savedTimeServices.Load(h.serviceKey()); ok { + return known == true + } + // Ordinary startup and drive reads already teach the capability. A direct + // first publish probes once; inability to learn it keeps the older wire. + _, _ = h.Cell(ctx, id) + known, _ := savedTimeServices.Load(h.serviceKey()) + return known == true +} + func (h *HTTP) serviceKey() string { return strings.TrimRight(h.base, "/") } func (h *HTTP) requireConditionalTake(ctx context.Context, id string) error { @@ -113,6 +157,9 @@ func (h *HTTP) Heartbeat(ctx context.Context, id string, b Beat) (v CellView, er } func (h *HTTP) Publish(ctx context.Context, id string, p Publish) (v CellView, err error) { + if p.SavedAt != 0 && !h.savedTurnTimes(ctx, id) { + p.SavedAt = 0 + } return v, h.do(ctx, http.MethodPost, cellPath(id, "publish"), p, &v) } @@ -167,6 +214,9 @@ func (h *HTTP) doHeader(ctx context.Context, method, path string, in, out any) ( return nil, fmt.Errorf("%w: %w", ErrUnreachable, err) } defer resp.Body.Close() + if resp.StatusCode < http.StatusInternalServerError { + savedTimeServices.Store(h.serviceKey(), resp.Header.Get(SavedTurnTimeHeader) == "1") + } if resp.StatusCode >= 300 { return resp.Header, refusal(resp) } diff --git a/internal/directory/http_test.go b/internal/directory/http_test.go index 8addd3a909..f631092557 100644 --- a/internal/directory/http_test.go +++ b/internal/directory/http_test.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "errors" + "io" "net/http" "net/http/httptest" "strings" "sync" + "sync/atomic" "testing" "time" @@ -86,6 +88,59 @@ func TestHTTPConformance(t *testing.T) { }) } +func TestHTTPChatR2OlderRelayStillPublishes(t *testing.T) { + clock := directorytest.NewFakeClock() + next := directory.Handler(fakeAuth, newNamespaces(clock).open) + var supported atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + // Older relays strictly decode signed create/publish bodies. + if !supported.Load() && bytes.Contains(raw, []byte(`"saved_at"`)) { + t.Error("R2 old relay received an unsupported saved_at field") + http.Error(w, `{"err":"bad_request"}`, http.StatusBadRequest) + return + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + reply := httptest.NewRecorder() + next.ServeHTTP(reply, r) + for key, values := range reply.Header() { + if supported.Load() || !strings.EqualFold(key, directory.SavedTurnTimeHeader) { + w.Header()[key] = values + } + } + w.WriteHeader(reply.Code) + _, _ = w.Write(reply.Body.Bytes()) + })) + defer srv.Close() + client := directory.NewHTTP(srv.URL, fakeSign("id_one", devA), srv.Client()) + v, err := client.Create(t.Context(), cellX, directory.CellInit{Head: head1, SavedAt: 700_000}) + if err != nil { + t.Fatal(err) + } + if v.Cell.SavedAt != 0 { + t.Fatal("R2 old relay invented a sealed-turn time") + } + if _, err := client.Publish(t.Context(), cellX, directory.Publish{Fence: v.Cell.Lease.Fence, OldHead: head1, Head: head2, SavedAt: 800_000}); err != nil { + t.Fatal(err) + } + if err := client.Release(t.Context(), cellX, v.Cell.Lease.Fence); err != nil { + t.Fatal(err) + } + // A subsequent response learns an upgraded service without restarting. + supported.Store(true) + if _, err := client.Cell(t.Context(), cellX); err != nil { + t.Fatal(err) + } + v, err = client.Publish(t.Context(), cellX, directory.Publish{Fence: v.Cell.Lease.Fence, OldHead: head2, Head: head1, SavedAt: 900_000}) + if err != nil || v.Cell.SavedAt != 900_000 { + t.Fatalf("R2 upgraded relay lost turn time: %+v, %v", v, err) + } +} + func TestHandlerTakesDeviceFromSignature(t *testing.T) { g := newRig(t, directorytest.NewFakeClock(), fakeAuth) ctx := context.Background() diff --git a/internal/directory/presence.go b/internal/directory/presence.go index 84a5693913..17091ebc66 100644 --- a/internal/directory/presence.go +++ b/internal/directory/presence.go @@ -14,7 +14,7 @@ type DevicePresence struct { } // PresenceView is the answer to GET /v1/dir/presence (docs/ux-pairing-contract.md, section 3.5): -// one entry per device that is not revoked. It is the fallback for a client +// one entry per device whose presence is known and that is not revoked. It is the fallback for a client // whose watch socket is down, and what a surface reads at open. type PresenceView struct { Now int64 `json:"now"` @@ -45,12 +45,12 @@ func presenceOf(l Listing, online map[string]bool) PresenceView { func (c *memoryClient) Presence(ctx context.Context) (PresenceView, error) { l, err := c.List(ctx) - return presenceOf(l, c.m.feed.OnlineSet()), err + return c.m.feed.WithPresence(presenceOf(l, nil)), err } func (c *sqliteClient) Presence(ctx context.Context) (PresenceView, error) { l, err := c.List(ctx) - return presenceOf(l, c.s.feed.OnlineSet()), err + return c.s.feed.WithPresence(presenceOf(l, nil)), err } // Presence asks the relay who is online. ErrTooOld-style fallbacks are the caller's: a relay without the route answers not found. @@ -68,7 +68,7 @@ func (m *Memory) seen(device string, at int64) { m.mu.Lock() defer m.mu.Unlock() if d, ok := m.devices[device]; ok { - d.LastSeen = at + d.LastSeen = max(d.LastSeen, at) set(m, m.devices, device, d) } } @@ -79,7 +79,7 @@ func (s *SQLite) seen(device string, at int64) { if get(tx, "devices", device, &d) != nil { return nil // not a device of this identity: nothing to stamp } - d.LastSeen = at + d.LastSeen = max(d.LastSeen, at) return put(tx, s.feed, "devices", device, d, at) }) } diff --git a/internal/directory/presence_reconnect_test.go b/internal/directory/presence_reconnect_test.go new file mode 100644 index 0000000000..2ed491238d --- /dev/null +++ b/internal/directory/presence_reconnect_test.go @@ -0,0 +1,36 @@ +package directory + +import ( + "testing" + "time" +) + +func TestPresenceP1QueriesRespectReconnectDebounce(t *testing.T) { + m := NewMemory(func() time.Time { return time.UnixMilli(5000) }) + ts := &timers{} + m.feed.SetAfter(ts.after) + a, b := m.For("dev_a"), m.For("dev_b") + if err := a.PutDevice(t.Context(), "dev_a", Device{V: 1}); err != nil { + t.Fatal(err) + } + sub, err := m.feed.Subscribe("dev_a", nil) + if err != nil { + t.Fatal(err) + } + sub.Close() + view, err := b.Presence(t.Context()) + if err != nil { + t.Fatal(err) + } + if !view.Devices["dev_a"].Online { + t.Fatal("P1 presence query marked a freshly disconnected computer offline during reconnect debounce") + } + ts.fire() + view, err = b.Presence(t.Context()) + if err != nil { + t.Fatal(err) + } + if view.Devices["dev_a"].Online { + t.Fatal("P1 computer stayed online after presence lapsed") + } +} diff --git a/internal/directory/presence_restart_test.go b/internal/directory/presence_restart_test.go new file mode 100644 index 0000000000..5228ddff9a --- /dev/null +++ b/internal/directory/presence_restart_test.go @@ -0,0 +1,69 @@ +package directory + +import ( + "fmt" + "testing" + "time" +) + +func BenchmarkPresenceRoster(b *testing.B) { + clock := &tick{t: time.UnixMilli(1_000_000)} + dir := NewMemory(clock.Now) + defer dir.Close() + client := dir.For("reader") + for i := range 16 { + if err := client.PutDevice(b.Context(), fmt.Sprintf("dev_%d", i), Device{V: 1}); err != nil { + b.Fatal(err) + } + } + // Compare the same complete roster before and after startup recovery. + clock.Advance(2 * time.Minute) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, err := client.Presence(b.Context()); err != nil { + b.Fatal(err) + } + } +} + +// P2: a fresh presence feed has no evidence about the stored roster yet. +// Both stores use this same boundary after their presence memory restarts. +func TestPresenceP2FreshMemoryIsUnknownUntilObservedOrWindowPasses(t *testing.T) { + for _, b := range backends(t) { + t.Run(b.name, func(t *testing.T) { + a := b.dir.For("dev_a") + for _, id := range []string{"dev_a", "dev_b"} { + if err := a.PutDevice(t.Context(), id, Device{V: 1}); err != nil { + t.Fatal(err) + } + } + view, err := a.Presence(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, known := view.Devices["dev_b"]; known { + t.Fatalf("P2 unobserved computer was reported offline after restart: %+v", view) + } + sub, err := b.feed.SubscribeBeat("dev_b", nil, 10*time.Second) + if err != nil { + t.Fatal(err) + } + defer sub.Close() + view, err = a.Presence(t.Context()) + if err != nil || !view.Devices["dev_b"].Online { + t.Fatalf("P2 observed computer is not online: %+v, %v", view, err) + } + b.clock.Advance(Window(10*time.Second) + time.Millisecond) + view, err = a.Presence(t.Context()) + if p, known := view.Devices["dev_b"]; err != nil || !known || p.Online { + t.Fatalf("P2 actually lapsed computer is not offline: %+v, %v", view, err) + } + b.clock.Advance(Window(DefaultBeat)) + view, err = a.Presence(t.Context()) + if p, known := view.Devices["dev_a"]; err != nil || !known || p.Online { + t.Fatalf("P2 never returning computer is not offline after a window: %+v, %v", view, err) + } + }) + } +} diff --git a/internal/directory/watchpresence.go b/internal/directory/watchpresence.go index 43573bd8d5..a7f16b48e4 100644 --- a/internal/directory/watchpresence.go +++ b/internal/directory/watchpresence.go @@ -31,7 +31,8 @@ type After func(d time.Duration, fn func()) (stop func() bool) // whose sockets simply fell silent is announced offline by the sweep at the // moment its last one lapsed, and its sockets are told to go. type presence struct { - clock func() time.Time + clock func() time.Time + started time.Time // after is the timers the debounce and the sweep run on. after After @@ -57,7 +58,8 @@ type seenAt struct { func newPresence(clock func() time.Time) *presence { return &presence{ - clock: clock, + clock: clock, + started: clock(), after: func(d time.Duration, fn func()) func() bool { return time.AfterFunc(d, fn).Stop }, @@ -257,6 +259,12 @@ func (p *presence) onlineSet() map[string]bool { return p.liveDevices() } +func (p *presence) presentSet() map[string]bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.believed() +} + // announce tells every event socket except those of `except` about e. func (p *presence) announce(e dirwatch.Event, except string) { p.mu.Lock() @@ -310,6 +318,33 @@ func (f *Feed) Online(device string) bool { return f.pres.online(device) } // OnlineSet is every device that holds a watch socket now. The caller owns the map. func (f *Feed) OnlineSet() map[string]bool { return f.pres.onlineSet() } +// PresentSet agrees with presence events during the reconnect debounce. A +// roster refresh cannot announce offline before the watch would announce it. +func (f *Feed) PresentSet() map[string]bool { return f.pres.presentSet() } + +// WithPresence applies this feed's evidence to a roster answer. A fresh relay +// has not observed the stored computers yet; absence becomes offline only +// after a default presence window, or after an observed socket actually lapses. +func (f *Feed) WithPresence(v PresenceView) PresenceView { + p := f.pres + p.mu.Lock() + defer p.mu.Unlock() + online := p.believed() + young := p.clock().Sub(p.started) < Window(DefaultBeat) + for id, seen := range v.Devices { + if young && !online[id] && len(p.socks[id]) == 0 && !p.gone[id] { + delete(v.Devices, id) + continue + } + seen.Online = online[id] + if seen.Online { + seen.LastSeen = v.Now + } + v.Devices[id] = seen + } + return v +} + // announceJoin tells the event sockets of every device but the new one that it // joined. A backend calls it, through SetOnJoin, in the turn that approves the // device; the name is the sealed name as stored. diff --git a/internal/fsreplace/syncdir.go b/internal/fsreplace/syncdir.go new file mode 100644 index 0000000000..6bf32f3cfb --- /dev/null +++ b/internal/fsreplace/syncdir.go @@ -0,0 +1,31 @@ +package fsreplace + +import ( + "errors" + "fmt" + "os" + "runtime" +) + +// SyncDir persists directory entries where the operating system supports it. +// Windows does not support FlushFileBuffers on the read-only directory handles +// returned by os.Open. Regular file contents still require their own Sync +// before Rename; a directory flush cannot substitute for that boundary. +func SyncDir(path string) (err error) { + dir, err := os.Open(path) + if err != nil { + return err + } + defer func() { err = errors.Join(err, dir.Close()) }() + info, err := dir.Stat() + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("sync directory %s: not a directory", path) + } + if runtime.GOOS == "windows" { + return nil + } + return dir.Sync() +} diff --git a/internal/fsreplace/syncdir_test.go b/internal/fsreplace/syncdir_test.go new file mode 100644 index 0000000000..1cc60f1673 --- /dev/null +++ b/internal/fsreplace/syncdir_test.go @@ -0,0 +1,25 @@ +package fsreplace + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestA4SyncDirectoryKeepsErrorsAndSupportsNativeStorage(t *testing.T) { + dir := t.TempDir() + if err := SyncDir(dir); err != nil { + t.Fatalf("A4: sync existing directory: %v", err) + } + if err := SyncDir(filepath.Join(dir, "absent")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("A4: missing directory error lost: %v", err) + } + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, []byte("contents"), 0600); err != nil { + t.Fatal(err) + } + if err := SyncDir(file); err == nil { + t.Fatal("A4: a file was accepted as a directory") + } +} diff --git a/internal/handoff/contract_regression_test.go b/internal/handoff/contract_regression_test.go index bcc970962b..8fcb080e64 100644 --- a/internal/handoff/contract_regression_test.go +++ b/internal/handoff/contract_regression_test.go @@ -46,3 +46,35 @@ func TestC7CardCannotDisplaceAHolderThatArrivedBeforeConfirmation(t *testing.T) t.Fatalf("stale card changed holder: %v, %+v", err, w.cellRec(chatID).Lease) } } + +func TestTakeF2SameHolderSameHeadFenceChange(t *testing.T) { + for _, change := range []string{"fence", "head", "holder"} { + t.Run(change, func(t *testing.T) { + w := newWorld(t) + a, b := w.device(devA), w.device(devB) + a.start(map[string]string{"a.txt": "one"}) + old := w.cellRec(chatID) + seen := &directory.LeaseExpectation{Device: old.Lease.Device, Fence: old.Lease.Fence, Head: old.Head} + switch change { + case "fence": + _, err := a.dir.Acquire(context.Background(), chatID, directory.AcquireOpts{}) + if err != nil { + t.Fatal(err) + } + case "head": + a.work(map[string]string{"a.txt": "two"}) + case "holder": + if _, err := w.device("dev_c").taker().Take(context.Background(), chatID); err != nil { + t.Fatal(err) + } + } + _, err := b.taker().TakeSeen(context.Background(), chatID, seen) + if change == "fence" && err != nil { + t.Fatalf("F2 harmless fence bump refused: %v", err) + } + if change != "fence" && !errors.Is(err, directory.ErrLeaseHeld) { + t.Fatalf("F2 %s must conflict: %v", change, err) + } + }) + } +} diff --git a/internal/handoff/handoff.go b/internal/handoff/handoff.go index 148ee0ef6a..48a833cce1 100644 --- a/internal/handoff/handoff.go +++ b/internal/handoff/handoff.go @@ -110,7 +110,7 @@ func (t Taker) TakeSeen(ctx context.Context, id string, seen *directory.LeaseExp if err != nil { return Taken{}, fmt.Errorf("handoff: read chat %s: %w", id, err) } - if seen != nil && (view.Cell.Lease.Device != seen.Device || view.Cell.Lease.Fence != seen.Fence) { + if seen != nil && !seen.Matches(view.Cell) { return Taken{}, directory.ErrLeaseHeld } c := cell.Cell{ID: id, Root: t.RootFor(id)} @@ -118,7 +118,11 @@ func (t Taker) TakeSeen(ctx context.Context, id string, seen *directory.LeaseExp if err != nil { return Taken{}, err } - head, fence, plan, err := t.claim(ctx, c, view.Cell.Head, displacing(view)) + how := displacing(view) + if seen != nil && seen.Head != "" { + how.Expect.Head = seen.Head + } + head, fence, plan, err := t.claim(ctx, c, view.Cell.Head, how) if err != nil { return Taken{}, err } diff --git a/internal/handoff/native_fetch_test.go b/internal/handoff/native_fetch_test.go new file mode 100644 index 0000000000..c6c24eaa74 --- /dev/null +++ b/internal/handoff/native_fetch_test.go @@ -0,0 +1,66 @@ +package handoff + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + "github.com/Agent-Field/codeaf/internal/home" +) + +// L5 exercises the real fetch transport before the directory move, including +// the native engine's retained handles. No provider or external relay is used. +func TestTakeNativeFetchL5(t *testing.T) { + celltest.RequireEngine(t) + t.Setenv(home.EnvVar, t.TempDir()) + defer cellstore.StopSpawned() + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + t.Setenv("CODEAF_ENGINE_DAEMON", "") + source, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source.Root, "hello.txt"), []byte("hello from A"), 0600); err != nil { + t.Fatal(err) + } + a := cellstore.Engine{DataRoot: t.TempDir()} + sealed, err := a.Seal(ctx, source, cellstore.TurnInfo{}) + if err != nil { + t.Fatal(err) + } + keys := cellstore.SyncKeys{CellKey: make([]byte, 32), Dedup: make([]byte, 32)} + objects := blobstore.NewMemory() + sender := a.Sync(keys, "00000000000000e3") + pub := cellsync.Publisher{Engine: sender, Store: objects} + if _, err := pub.Upload(ctx, source, sealed.Turn.ID); err != nil { + t.Fatal(err) + } + b := cellstore.Engine{DataRoot: t.TempDir()} + receiver := b.Sync(keys, "00000000000000e3") + root := filepath.Join(t.TempDir(), source.ID) + staged := cell.Cell{ID: source.ID, Root: stagingOf(root)} + if err := os.MkdirAll(staged.Root, 0700); err != nil { + t.Fatal(err) + } + fetch := cellsync.Fetcher{Engine: receiver, Store: objects, Inbox: receiver.Inbox} + if err := fetch.Fetch(ctx, staged, sealed.Turn.ID); err != nil { + t.Fatal(err) + } + if err := swap(staged.Root, root); err != nil { + t.Fatalf("L5 rename after real fetch: %v", err) + } + if err := b.Follow(ctx, staged, cell.Cell{ID: source.ID, Root: root}); err != nil { + t.Fatal(err) + } + if raw, err := os.ReadFile(filepath.Join(root, "hello.txt")); err != nil || string(raw) != "hello from A" { + t.Fatalf("L5 restored content %q: %v", raw, err) + } +} diff --git a/internal/handoff/shared_daemon_linux_test.go b/internal/handoff/shared_daemon_linux_test.go new file mode 100644 index 0000000000..4e18a876ab --- /dev/null +++ b/internal/handoff/shared_daemon_linux_test.go @@ -0,0 +1,64 @@ +package handoff + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +// Linux permits renaming watched directories. Inspect the live handles too so +// W1 detects the Windows obstruction before a native Windows run is available. +func assertTakeStageReleased(t *testing.T, root string) { + t.Helper() + inodes := map[string]string{} + if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + info, err := d.Info() + if err != nil { + return err + } + inodes[fmt.Sprintf("ino:%x", info.Sys().(*syscall.Stat_t).Ino)] = path + } + return nil + }); err != nil { + t.Fatal(err) + } + processes, err := filepath.Glob("/proc/[0-9]*/exe") + if err != nil { + t.Fatal(err) + } + for _, exe := range processes { + binary, _ := os.Readlink(exe) + if !strings.Contains(filepath.Base(binary), "furrow") { + continue + } + proc := filepath.Dir(exe) + fds, _ := filepath.Glob(filepath.Join(proc, "fd", "*")) + fds = append(fds, filepath.Join(proc, "cwd")) + for _, fd := range fds { + link, _ := os.Readlink(fd) + if link == root || strings.HasPrefix(link, root+string(filepath.Separator)) { + t.Errorf("W1 engine retains %s -> %s", fd, link) + } + if link != "anon_inode:inotify" { + continue + } + raw, err := os.ReadFile(filepath.Join(proc, "fdinfo", filepath.Base(fd))) + if err != nil { + t.Fatal(err) + } + for _, field := range strings.Fields(string(raw)) { + if path := inodes[field]; path != "" { + t.Errorf("W1 engine retains directory watch on %s (%s)", path, fd) + } + } + } + } +} diff --git a/internal/handoff/shared_daemon_other_test.go b/internal/handoff/shared_daemon_other_test.go new file mode 100644 index 0000000000..2da91e7337 --- /dev/null +++ b/internal/handoff/shared_daemon_other_test.go @@ -0,0 +1,8 @@ +//go:build !linux + +package handoff + +import "testing" + +// The directory rename itself is the native Windows handle check. +func assertTakeStageReleased(t *testing.T, root string) {} diff --git a/internal/handoff/shared_daemon_test.go b/internal/handoff/shared_daemon_test.go new file mode 100644 index 0000000000..8f131d3208 --- /dev/null +++ b/internal/handoff/shared_daemon_test.go @@ -0,0 +1,175 @@ +package handoff + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + "github.com/Agent-Field/codeaf/internal/home" +) + +// W1/W2 exercise a composed app chat: the daemon stays alive while its private +// staging folder moves, and the installed tree must still support dirty/seal. +func TestTakeSharedDaemonW1W2(t *testing.T) { + takeSharedDaemon(t, false, false) +} + +// W5 also covers an empty workspace absent after installation. The sender's +// recorded path is not a working directory on the receiver. +func TestTakeEmptyWorkspaceW5(t *testing.T) { + takeSharedDaemon(t, true, false) +} + +// W6 restores B's work and receipt into A's original composed layout. The +// workspace is short, but receipt names under the separate .cell exceed MAX_PATH. +func TestTakeBackComposedW6(t *testing.T) { + takeSharedDaemon(t, false, true) +} + +func takeSharedDaemon(t *testing.T, empty, takeBack bool) { + t.Helper() + celltest.RequireEngine(t) + t.Setenv(home.EnvVar, t.TempDir()) + t.Setenv("CODEAF_ENGINE_DAEMON", "1") + defer cellstore.StopSpawned() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancel() + parent := t.TempDir() + if takeBack { + // Keep the composed root short enough for a process and its temporary + // files while making the final 64-character receipt name exceed 260. + parent = filepath.Join(parent, strings.Repeat("c", max(1, 172-len(parent)-1))) + } + source, err := cell.CreateIn(parent, cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + if !empty { + if err := os.WriteFile(filepath.Join(work, "hello.txt"), []byte("hello from A"), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(source.Root, cell.TranscriptPath), []byte("{\"type\":\"session\",\"cwd\":\"/Users/sender/unavailable-project\"}\n"), 0600); err != nil { + t.Fatal(err) + } + a := cellstore.Engine{DataRoot: t.TempDir(), Workspace: work} + sealed, err := a.Seal(ctx, source, cellstore.TurnInfo{}) + if err != nil { + t.Fatal(err) + } + keys := cellstore.SyncKeys{CellKey: make([]byte, 32), Dedup: make([]byte, 32)} + objects := blobstore.NewMemory() + pub := cellsync.Publisher{Engine: a.Sync(keys, "00000000000000e6"), Store: objects} + if _, err := pub.Upload(ctx, source, sealed.Turn.ID); err != nil { + t.Fatal(err) + } + b := cellstore.Engine{DataRoot: t.TempDir(), WorkspaceOf: func(c cell.Cell) string { + return filepath.Join(c.Root, "work") + }} + root := filepath.Join(t.TempDir(), source.ID) + stage := cell.Cell{ID: source.ID, Root: stagingOf(root)} + if err := os.MkdirAll(filepath.Join(stage.Root, "work"), 0700); err != nil { + t.Fatal(err) + } + receiver := b.Sync(keys, "00000000000000e6") + fetch := cellsync.Fetcher{Engine: receiver, Store: objects, Inbox: receiver.Inbox} + if err := fetch.Fetch(ctx, stage, sealed.Turn.ID); err != nil { + t.Fatal(err) + } + assertTakeStageReleased(t, stage.Root) + if empty { + // Directory entries are not chat content. Exercise installation without + // relying on the materializer preserving an empty working directory. + if err := os.Remove(filepath.Join(stage.Root, "work")); err != nil { + t.Fatal(err) + } + // The app files an arriving copy under its origin after fetching. Its + // installed path can exceed Windows's process-directory limit. + root = filepath.Join(t.TempDir(), strings.Repeat("a", 100), strings.Repeat("b", 100), source.ID) + if err := os.MkdirAll(filepath.Dir(root), 0700); err != nil { + t.Fatal(err) + } + } + if err := swap(stage.Root, root); err != nil { + t.Fatalf("W1 rename after shared-daemon fetch: %v", err) + } + installed := cell.Cell{ID: source.ID, Root: root} + if err := b.Follow(ctx, stage, installed); err != nil { + t.Fatalf("W2 final binding: %v", err) + } + if empty { + tree := filepath.Join(root, "work") + info, err := os.Stat(tree) + if err != nil || !info.IsDir() { + t.Fatalf("W5 followed workspace %q: %v", tree, err) + } + bound, err := os.ReadFile(filepath.Join(b.LocalDir(installed), "engine-tree")) + resolved, resolveErr := filepath.EvalSymlinks(tree) + if err != nil || resolveErr != nil || string(bound) != resolved { + t.Fatalf("W5 binding %q, want local tree %q: %v", bound, tree, err) + } + if dirty, err := b.Dirty(ctx, installed); err != nil || dirty { + t.Fatalf("W5 installed head: dirty=%v err=%v", dirty, err) + } + return + } + path := filepath.Join(root, "work", "hello.txt") + if raw, err := os.ReadFile(path); err != nil || string(raw) != "hello from A" { + t.Fatalf("W1 restored content %q: %v", raw, err) + } + if dirty, err := b.Dirty(ctx, installed); err != nil || dirty { + t.Fatalf("W2 installed head: dirty=%v err=%v", dirty, err) + } + if err := os.WriteFile(path, []byte("hello from B"), 0600); err != nil { + t.Fatal(err) + } + if dirty, err := b.Dirty(ctx, installed); err != nil || !dirty { + t.Fatalf("W2 edited installed tree: dirty=%v err=%v", dirty, err) + } + installed, err = cell.OpenAt(root, source.ID) + if err != nil { + t.Fatal(err) + } + returned, err := b.Seal(ctx, installed, cellstore.TurnInfo{}) + if err != nil { + t.Fatalf("W2 seal at final root: %v", err) + } + if err := cellstore.Await(ctx, installed); err != nil { + t.Fatalf("W2 save at final root: %v", err) + } + if !takeBack { + return + } + sender := b.Sync(keys, "00000000000000e6") + if _, err := (&cellsync.Publisher{Engine: sender, Store: objects}).Upload(ctx, installed, returned.Turn.ID); err != nil { + t.Fatal(err) + } + receipt := filepath.Join(cell.StateDir, "receipts", returned.Turn.Receipt+".json") + receiptAtA := filepath.Join(source.Root, receipt) + if len(receiptAtA) <= 260 { + t.Fatalf("W6 needs a long receipt path, got %d characters", len(receiptAtA)) + } + back := a.Sync(keys, "00000000000000e6") + if err := (&cellsync.Fetcher{Engine: back, Store: objects, Inbox: back.Inbox}).Fetch(ctx, source, returned.Turn.ID); err != nil { + t.Fatalf("W6 take-back into original composed directory: %v", err) + } + if raw, err := os.ReadFile(filepath.Join(work, "hello.txt")); err != nil || string(raw) != "hello from B" { + t.Fatalf("W6 restored work %q: %v", raw, err) + } + want, err := os.ReadFile(filepath.Join(installed.Root, receipt)) + if err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(receiptAtA); err != nil || string(got) != string(want) { + t.Fatalf("W6 restored receipt %q: %v", receiptAtA, err) + } +} diff --git a/internal/keys/merge.go b/internal/keys/merge.go index fe42aae28a..e9a7765bf6 100644 --- a/internal/keys/merge.go +++ b/internal/keys/merge.go @@ -69,5 +69,8 @@ func supersedes(a record, ad *vaultDoc, b record, bd *vaultDoc) bool { return false } sa, sb := a.stampOf(ad), b.stampOf(bd) + if sa == sb && a.live() && b.live() && a.Entry == b.Entry && a.Metadata != "" && b.Metadata == "" { + return true + } return sa > sb || (sa == sb && a.Deleted && b.live()) } diff --git a/internal/keys/vault.go b/internal/keys/vault.go index db6644ec0a..91045d38e6 100644 --- a/internal/keys/vault.go +++ b/internal/keys/vault.go @@ -35,8 +35,9 @@ type Entry struct { // one small record per deleted secret. type record struct { Entry - Deleted bool `json:"deleted,omitempty"` - Updated int64 `json:"updated,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Updated int64 `json:"updated,omitempty"` + Metadata string `json:"metadata,omitempty"` } // stampOf is the time a slot was last written, falling back to its document's. @@ -114,6 +115,12 @@ func (v *Vault) Put(id string, e Entry) error { return v.PutAt(id, e, time.Now() // time of the edit, so recording an old edit late never beats a newer one that // another machine has already sent. func (v *Vault) PutAt(id string, e Entry, at time.Time) error { + return v.PutMetadataAt(id, e, at, "") +} + +// PutMetadataAt keeps a carrier's optional identity facts beside its value. +// Older readers still receive the same value and ignore the extra metadata. +func (v *Vault) PutMetadataAt(id string, e Entry, at time.Time, metadata string) error { return v.update(func(d *vaultDoc) error { stamp := at.UnixMilli() if previous, exists := d.Secrets[id]; exists { @@ -121,11 +128,32 @@ func (v *Vault) PutAt(id string, e Entry, at time.Time) error { } // A new slot has its own edit clock. Falling back to the document's // clock would make another setting's recent capture age this one. - d.Secrets[id] = record{Entry: e, Updated: stamp} + d.Secrets[id] = record{Entry: e, Updated: stamp, Metadata: metadata} return nil }) } +// Annotate supplies missing carrier facts without making the choice newer. +// The expected value prevents an upgrade from describing a different choice +// that arrived from another computer while the profile was being read. +func (v *Vault) Annotate(id string, expected Entry, metadata string) error { + v.mu.Lock() + defer v.mu.Unlock() + d, err := v.read() + if err != nil { + return err + } + r, ok := d.Secrets[id] + if !ok || !r.live() || r.Entry != expected || r.Metadata != "" || metadata == "" { + return nil + } + r.Updated = r.stampOf(d) + r.Metadata = metadata + d.Secrets[id] = r + // Preserve the document clock too: older slots may still inherit it. + return v.write(d) +} + // Get returns the secret under id. func (v *Vault) Get(id string) (Entry, error) { d, err := v.read() @@ -180,6 +208,28 @@ func (v *Vault) Entries(project string) ([]Entry, error) { return out, nil } +// Revision identifies a slot's choice and the carrier facts saved with it. +type Revision struct { + Stamp int64 + Metadata string +} + +// Revisions returns the revisions of a scope's live slots in one vault read. +// Receivers can remember a rejected revision without changing the shared slot. +func (v *Vault) Revisions(scope string) (map[string]Revision, error) { + d, err := v.read() + if err != nil { + return nil, err + } + out := map[string]Revision{} + for _, r := range d.Secrets { + if r.live() && r.Scope == scope { + out[r.Name] = Revision{Stamp: r.stampOf(d), Metadata: r.Metadata} + } + } + return out, nil +} + // IDs lists the ids that start with prefix, in order, the removed ones too: a // reader that keeps one slot per path needs the paths that were deleted as much // as the live ones, since a tombstone is what tells it to delete the file. diff --git a/internal/manual/chat/desktop-sync.md b/internal/manual/chat/desktop-sync.md index 9b6b74ec01..098be13576 100644 --- a/internal/manual/chat/desktop-sync.md +++ b/internal/manual/chat/desktop-sync.md @@ -65,3 +65,161 @@ When the app opens a chat with unfinished command records, each command is named in a visible notice: ` — stopped when codeaf was killed — not run again. Ask me to run it again if you still want it.` The warning does not run the command again or claim that it succeeded. + + +## Relay in desktop Settings and Devices — address, default, off and pairing + +With a compatible engine, Settings → Devices shows Relay and where its address +came from: set here, from pairing, default or off. An address set here wins over +one saved by pairing. Clearing it uses the paired address, or the hosted default +when none was saved. Off stops sync. Saving a desktop relay setting restarts its +engine; working agents require confirmation first. Cancelling keeps the setting. +A computer that has never paired remains quiet even with a relay address set. + +## Pair a computer or enter a pairing code in Computers + +Computers offers Pair a computer. Choose to show a code, or choose +Enter a code from the other computer to type one. Setup also offers joining. +A link can request or approve pairing instead. +Compare the engine's three words or check number on both computers, then answer +its explicit question. Nothing is approved by pressing Enter without a choice. +Choose Pair or Don't pair. The code's instruction says: `On your other computer, +choose Pair a computer, then Enter a code from the other computer, and type +. This code lasts 10 minutes.` The approval names the other computer and +its platform, for example `Pair with Abir (Windows)?`, including when both +computers share a name. +Cancel takes back the waiting code or ends the request. A wrong code shows the +engine's sentence and does not add a computer. + +A computer that already has chats asks before replacing them. Choose the option +Not now to cancel without changing them, or Replace this computer's chats to +continue. Finish any live work before +joining another computer. After joining changes what this computer shares, the +desktop restarts its engine and reads setup, devices and sync again. +These actions appear only when the engine advertises pairing support. + +## When does a paired computer appear and when is its new name shared + +Every paired desktop engine shares its own computer name and capabilities at +startup and after pairing, even with no chats. A failed write retries +automatically, starting after five seconds and doubling to a maximum of sixty +seconds between attempts. Success ends the retry. Reconnecting checks the +record again; an unchanged record is not rewritten. Sync off and a home that +has never paired remain quiet. + +Renaming shares the new name immediately and says `This computer is now called +.` If the relay cannot accept it yet, the name stays saved here and the +line adds `Your other computers will see it once this computer is back online.` +The engine retries without another rename or a chat. + +Computers lists this computer and the relay's roster even while this computer's +own record is still waiting to be shared. A separate presence failure leaves +online status unknown; a readable roster is still shown. Changes arrive through +engine events, and unchanged lists do not repeat them. + +## Chats on your other computers and Continue here in the desktop + +Chats from other computers appear under On your other computers, with the +computer's name. An online dot appears only when the engine knows its presence. +Finished movable chats saved before pairing are shared after pairing and when +the paired app engine starts. This does not open the chat or start a new turn. +An open, idle chat says `open on `; this describes where it is open and +does not mean a reply is running. Closing a chat does not make its computer +offline. ` offline` appears only when that computer is known to be offline; +unknown presence has no online dot or offline claim. The saved time is the time +of the newest shared saved turn, shown in your local time, rather than the time +it was copied to the relay. An unknown saved time is left out. The engine's +takeover sentence adds holder or pending-work facts without repeating a date or age. +After a relay starts or restarts, computers it has not heard from have unknown +presence for up to 75 seconds. They have no offline line during that window. +An observed computer whose connection actually lapses can still be marked offline. +Select a chat to preview its title, saved time and engine sentence. Continue +here moves it through the engine, then opens the returned local chat. Its +history and working copies use the same continuation path as the terminal. + +An already local chat that moved away keeps its place and becomes read-only. +Continue here is available when the engine says it can move. Continuing here +stops new work on the other computer. If another computer moved it after the +preview was read, the engine says `another device continued this chat first`; +refresh the row and choose again. There is no automatic retry. Unsent drafts +and attachments remain yours. Changes arrive as engine events without desktop +polling. An older chat that works in the home folder stays on that computer. + +Continue here still works after a reconnect when the same computer holds the +same saved turn. A different holder or a newer saved turn needs a fresh choice. +Completed turns are shared while the chat stays open, including after a move. + +## Does an idle desktop chat keep its hold on the relay + +A chat opened for viewing keeps its conversation and relay hold until it is +explicitly closed, including while its tab is hidden. Heartbeats renew the same +hold without releasing it or moving its fence. Reading the chat does not cause +ownership changes or repeat unchanged chat rows. + +A conversation opened only for background work is let go after ten idle minutes; +the sweep checks every 30 seconds. It stays released until work needs it again. +Sending, queueing or retrying opens it through the normal ownership check. +Steering still needs a running turn. A task or job can also open it. Running +turns, unanswered questions, held crew members and kept services retain their +conversations. If another computer continues a chat, the existing Continue here +and kept-work rules still apply. + +Stopped processes, finished tasks and settled jobs do not reopen a background +chat at startup or after release. Recovery repairs a stale running process +record when the matching task checkpoint already knows it ended. Unfinished +work still recovers normally. Releasing the last background chat does not restart +sync or repeat settled preferences. + +## Relay outages, local turns and automatic recovery + +While the relay is unreachable, an open chat continues to run and save on this +computer. Unpublished turns retry automatically with waits capped at 60 seconds, without +another send, quit or restart. A lapsed hold that still belongs to this computer +is renewed when the relay returns; its saved work is then shared. The chat stays +writable and never names this computer as having continued it elsewhere. + +If another computer continued the chat meanwhile, this window becomes read-only +and names that computer. Work saved here during the outage is kept through the +usual kept-work path. Use Continue here to bring the current shared chat back. + +## Continue here when this computer cannot use the chat's model + +A chat saved on another computer can name a provider that is not set up here. +When it opens here, codeaf checks the same local provider readiness used for +shared model settings. An unavailable model continues with this computer's usable +default. One line in the chat explains which model was unavailable and which +model is used here. The existing conversation model-switch rules save that choice; +continuing back on a computer without that provider checks its own default again. +Every continuation uses the saved chat's latest main and fallback choices, +including when this computer already has an older copy. A usable saved model +stays selected without a change notice. An unavailable main model gets one +specific explanation; a separate fallback change may have its own line. +Continue here completes even when the default is also unavailable: the chat is +here, and its row opens the returned local agent. Its first send fails locally +with a setup sentence, such as `This chat used llama3.2 (Ollama), which isn't set up on this computer. Set up a model in Settings, then send again.` +That reason appears once in the agent view, and no provider call is made for +the turn. Set a usable default in Settings, then send again. The next send uses +that default and explains the change once; taking the chat again is unnecessary. + +## If Continue here fails while moving a chat + +If moving the chat fails, the app says +`This chat could not be moved to this computer.` Technical causes are kept in +the error details. A plain model setup reason is carried in the visible message. +If installation failed after acquisition, the chat is released. +The computer without the installed chat still offers Continue here and names the +last holder without claiming it is open there. The original open chat says +`This chat is available to continue here.` Its next send can acquire the free +lease when its saved head still matches the relay. +If another computer saved newer work, use Continue here to restore that work. + +## Shared settings after pairing — model, effort and spending limits + +When shared preferences arrive from another paired computer, desktop Settings +updates through the engine's settings event. One restore reports the full current +settings once, including changes to the default model, effort and spending limits. +An unchanged restore sends no settings event. Repeated roster notifications also +send no Devices event when the displayed computer list has not changed. +A model choice is applied here only when its provider is set up on this computer; +otherwise this computer keeps its own model. Setting up that provider later does +not apply an older skipped choice; only a newer choice on the other computer travels. diff --git a/internal/manual/chat/models-and-cost.md b/internal/manual/chat/models-and-cost.md index a71202e5fc..cfa9960503 100644 --- a/internal/manual/chat/models-and-cost.md +++ b/internal/manual/chat/models-and-cost.md @@ -1672,6 +1672,10 @@ answer: it cannot think harder than its own ceiling. remembered, so it costs one rejected request for that model and never a failed reply. - A model whose catalog row says it takes no reasoning knob at all is sent nothing about thinking. +- If an unlisted endpoint says its model does not support thinking or reasoning, + codeaf retries the request once without the knob and remembers that capability + for the model until this process exits. That capability refusal is permanent; + it does not schedule the same failed turn again later. - On a small context window the budget shrinks to the room the conversation leaves beside an answer. When less than 1,024 tokens of thinking would fit, the budget is dropped and the rung travels as `high`, so `xhigh` on a 32,000-token window is never refused for the diff --git a/internal/manual/chat/naming-a-device.md b/internal/manual/chat/naming-a-device.md index 6de3fef020..48a41b8539 100644 --- a/internal/manual/chat/naming-a-device.md +++ b/internal/manual/chat/naming-a-device.md @@ -20,6 +20,9 @@ Your other devices will call this computer "tower". Press Enter to keep that, or Press Enter to keep the host name, or type a name and press Enter. To say it without being asked, type `codeaf pair --name "atlas"`. It works with a typed code too: `codeaf pair 42-715-302 --name "atlas"`. The device that approves shows that name when it asks you to approve. +Current computers include the platform during approval, such as `Abir (Windows)`, +so two computers named Abir can be distinguished. + ## How do I rename a device later — codeaf devices rename and the n key Type this on the device you mean: @@ -40,16 +43,18 @@ Edit it and press enter to save, ctrl+u to clear it, esc to leave it as it was. After a rename it says: ``` -This computer is now called atlas on all your devices. +This computer is now called atlas. ``` Your other devices show the new name the next time they list devices or draw a home row. If the sync service cannot be reached at that moment, the device keeps the name and says: ``` -This computer is now called atlas. Your other devices will show it after this one next connects. +This computer is now called atlas. Your other computers will see it once this computer is back online. ``` -You do not type it again. A device that is not paired with anything yet says `This computer is now called atlas.` +You do not type it again. A running paired desktop engine retries automatically, +even without a chat. A device that is not paired with anything yet says +`This computer is now called atlas.` ## What can a device name be — length, spaces, control characters and duplicates diff --git a/internal/manual/chat/relay-hosted-or-your-own.md b/internal/manual/chat/relay-hosted-or-your-own.md index 9fc2b5ef0e..e31833eaba 100644 --- a/internal/manual/chat/relay-hosted-or-your-own.md +++ b/internal/manual/chat/relay-hosted-or-your-own.md @@ -48,6 +48,14 @@ Stop it with SIGTERM or ctrl+c: requests under way finish first. **The relay has There is no Docker image for the relay. The page `docs/relay.md` in the repository says how to run, back up and upgrade it. +## Why does my native relay answer leadership unavailable or HTTP 503 + +The relay could not activate its durable group or complete the requested work. +Its existing log includes the underlying local error, limited to one line per +cause per minute. Check that log for a storage, permission or consensus error; +the HTTP response keeps its usual wording. `--quiet` disables these log lines +along with the other relay logging. + ## Who can use my own relay Moving chats and pairing are available by default on Linux, macOS and native Windows on every build with a compatible engine. Pair both computers first. diff --git a/internal/manual/chat/use-this-on-another-computer.md b/internal/manual/chat/use-this-on-another-computer.md index 1ebbb1082a..d31d0db987 100644 --- a/internal/manual/chat/use-this-on-another-computer.md +++ b/internal/manual/chat/use-this-on-another-computer.md @@ -413,6 +413,8 @@ Moving chats and pairing are available by default on Linux, macOS and native Win Older chats move through independent copies after pairing. Their original files and layout remain untouched for older builds. Nothing is deleted automatically to save disk. Native Windows moves chats with the native engine, just like Linux and macOS. +Continue here installs the saved chat while other chats stay open. You can send +from the installed copy and continue it back on your other computer. Install Git for Windows to run commands; codeaf uses Git Bash, never the WSL launcher. A build with no compatible engine keeps moving chats absent and says `moving chats needs a newer furrow engine than this build carries`. @@ -424,6 +426,10 @@ or mixed line endings even when the receiving Git uses another checkout setting. links; links that cannot be created are reported on the same line. Hard-linked names currently arrive on Windows as separate files with identical bytes. The chat works in its folder on this computer, with old absolute workspace paths remapped there. +An empty workspace is created here too. Continuing a chat follows its installed +folder even when the copy's full path exceeds Windows' process-directory limit. +Taking a project chat back also restores its saved turn receipts when their +paths exceed Windows' usual file-path limit. The encrypted vault and withheld secrets live under your codeaf home in your user profile. Windows protects that profile with its account permissions; Unix file mode bits are recorded for later moves, but do not provide Windows access control. diff --git a/internal/pair/chatpair_test.go b/internal/pair/chatpair_test.go index b0f2c65fe0..9bc6b5cc81 100644 --- a/internal/pair/chatpair_test.go +++ b/internal/pair/chatpair_test.go @@ -35,3 +35,23 @@ func TestPairingSharesTheChatsAndTheWordsAgree(t *testing.T) { t.Fatalf("B holds %v (%v), want A's identity", held.ID(), err) } } + +func TestA3CodePairingApprovalNamesTheOtherPlatform(t *testing.T) { + r := newChatRig(t) + a, b := newScreen(), newScreen() + a.answer = func(_ context.Context, label, _ string) bool { + if label != "Abir (Windows)" { + t.Errorf("A3: approval identifies %q, want Abir (Windows)", label) + } + return true + } + done := r.offer(t.Context(), a) + code := a.nextCode(t) + _, err := Join(t.Context(), r.route, Joining{Home: r.homeB, Label: "Abir", Platform: "windows"}, code.Shown(), b) + if err != nil { + t.Fatal(err) + } + if end := within(t, done); end.err != nil { + t.Fatal(end.err) + } +} diff --git a/internal/pair/join.go b/internal/pair/join.go index c497e0c3da..905e05bc83 100644 --- a/internal/pair/join.go +++ b/internal/pair/join.go @@ -31,6 +31,8 @@ type Joining struct { Home string // Label is what this device calls itself on the other device's screen. Label string + // Platform disambiguates computers with the same name during approval. + Platform string // Replace lets a computer that already has different chats give them up. Replace bool // MergeState keeps memory and preferences when a startup-only home joins. @@ -80,7 +82,7 @@ func joinRun(ctx context.Context, route Mailbox, j Joining, code JoinCode, ui Jo link := &boxLink{ctx: ctx, box: route.Box, plate: code.Plate, mine: pairbox.SideB, theirs: pairbox.SideA, key: key, gone: map[stage]error{awaitAnswer: ErrStopped, awaitVerdict: ErrCodeDidNotWork}, sendGone: NothingWaitingUnder(code.Plate)} - offer, err := joinOffer(j.Label) + offer, err := joinOffer(j.Label, j.Platform) if err != nil { return Grant{}, err } @@ -108,12 +110,16 @@ func phraseJoin(err error, route Mailbox) error { // joinOffer is what this device sends in message 3: its name, then random bytes // that make the message differ from any other session's. -func joinOffer(label string) ([]byte, error) { +func joinOffer(label string, platform ...string) ([]byte, error) { nonce := make([]byte, nonceSize) if _, err := rand.Read(nonce); err != nil { return nil, err } - return append([]byte(readableLabel(label)), nonce...), nil + name := readableLabel(label) + if len(platform) != 0 && platform[0] != "" { + name = ComputerLabel(name, platform[0]) + } + return append([]byte(name), nonce...), nil } // adopt takes the grant into this home, or says why not. A computer that diff --git a/internal/pair/lines.go b/internal/pair/lines.go index c4d184f68d..909f89184f 100644 --- a/internal/pair/lines.go +++ b/internal/pair/lines.go @@ -119,6 +119,14 @@ var platformWords = map[string]string{ "ios": "iPhone or iPad", "android": "Android", } +// ComputerLabel retains the name and adds a familiar platform when known. +func ComputerLabel(name, platform string) string { + if word, ok := platformWords[platform]; ok { + return fmt.Sprintf("%s (%s)", name, word) + } + return name +} + // WantsToJoinLine tells the approving person which device is asking. A system // that has no familiar name is left out rather than shown as a raw word. func WantsToJoinLine(name, platform string) string { @@ -163,14 +171,14 @@ func RenamedAloneLine(name string) string { // RenamedLine is what a computer in a fleet says after it was named and the // sync service has the new name, so every other device shows it. func RenamedLine(name string) string { - return fmt.Sprintf("This computer is now called %s on all your devices.", name) + return RenamedAloneLine(name) } // RenamedHereLine is what it says when the new name is kept here but the sync // service could not be reached: the next time this computer connects it tells // the others, so nothing needs to be typed again. func RenamedHereLine(name string) string { - return fmt.Sprintf("This computer is now called %s. Your other devices will show it after this one next connects.", name) + return fmt.Sprintf("This computer is now called %s. Your other computers will see it once this computer is back online.", name) } // NamePrompt is the question a person is asked while a computer joins: the name diff --git a/internal/pair/offer.go b/internal/pair/offer.go index 6c868aca15..4087ec251a 100644 --- a/internal/pair/offer.go +++ b/internal/pair/offer.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "math/big" + "strings" "sync/atomic" "time" @@ -144,7 +145,15 @@ func joinerLabel(offer []byte) string { if len(offer) < nonceSize { return readableLabel("") } - return readableLabel(string(offer[:len(offer)-nonceSize])) + name := string(offer[:len(offer)-nonceSize]) + // Keep the established text offer readable by older engines. A platform + // suffix is outside the name's limit, so a full-length name retains it. + for platform, word := range platformWords { + if plain, ok := strings.CutSuffix(name, " ("+word+")"); ok { + return ComputerLabel(readableLabel(plain), platform) + } + } + return readableLabel(name) } // nonceSize is the random tail of a joining device's offer. It makes the offer diff --git a/internal/provider/calllog.go b/internal/provider/calllog.go index ed120d0e6f..1d4eb4c987 100644 --- a/internal/provider/calllog.go +++ b/internal/provider/calllog.go @@ -42,6 +42,7 @@ const ( learnedCacheControlRefused = "cache_control_refused" learnedReasoningBudgetRefused = "reasoning_budget_refused" learnedReasoningReplayRefused = "reasoning_replay_refused" + learnedReasoningUnsupported = "reasoning_unsupported" ) type callTagKey struct{} diff --git a/internal/provider/client.go b/internal/provider/client.go index 013e73cf12..19b497a2eb 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -634,6 +634,12 @@ func (c *Client) sendRecovered(ctx context.Context, request *ai.Request, knobs c } model := c.modelFor(request) peek, readErr := io.ReadAll(io.LimitReader(response.Body, maxErrorPeek)) + // The reasoning repair has already had its one attempt. A persistent + // capability refusal cannot be cured by widening endpoint preferences. + if readErr == nil && response.StatusCode == http.StatusBadRequest && refusesReasoning(peek) { + response.Body = rewound(peek, response.Body) + return response, nil + } // WHETHER THE BASE UNDERSTANDS THE `provider` FIELD AT ALL IS ASKED BEFORE // ANYTHING ELSE (#433), because until it is answered nothing below this line // is reading the right refusal. It is a QUESTION here and an answer only @@ -890,7 +896,7 @@ func (c *Client) resend(ctx context.Context, request *ai.Request, knobs callKnob // adapter knows how to answer. It is asked before the error body is touched, so // a 400 that could not be ours costs no extra read. func (c *Client) repairable(model string, knobs callKnobs) bool { - return c.resolveEffort(model, knobs.effort) == EffortOff || + return c.resolveEffort(model, knobs.effort) != EffortNone || c.resolveReasoningBudget(model, knobs.effort) > 0 || c.dialectFor(model) == cacheDialectBreakpoints || (len(knobs.reasoning) > 0 && !reasoningReplayRefused(model)) @@ -903,6 +909,10 @@ func (c *Client) repairable(model string, knobs callKnobs) bool { // first match winning, because a single 400 can name more than one field. func (c *Client) learn(model string, knobs callKnobs, payload []byte) []string { var learned []string + if c.resolveEffort(model, knobs.effort) != EffortNone && refusesReasoning(payload) { + quirks.record(func(q *quirksStore) map[string]time.Time { return q.noReasoning }, model, time.Now()) + learned = append(learned, learnedReasoningUnsupported) + } if c.resolveEffort(model, knobs.effort) == EffortOff && refusesDisabledReasoning(payload) { noteReasoningMandatory(model) learned = append(learned, learnedReasoningMandatory) @@ -2624,6 +2634,8 @@ func adaptiveCompletionTimeout(maxTokens int, configuredFloor time.Duration) tim // somebody and the refusal are both in the JSON it sent: `error.metadata` // carries `provider_name` and `raw`, and this client threw them away. type APIError struct { + // ReasoningUnsupported is a permanent model capability refusal. + ReasoningUnsupported bool // Status is the refusal's HTTP status, or the normalized gateway failure // for an in-band error that supplied no status of its own. Status int @@ -2763,6 +2775,9 @@ func (e *APIError) OurRequest() bool { if e == nil { return false } + if e.ReasoningUnsupported { + return true + } // AN EMPTY ACCOUNT IS OURS TO STOP WHOEVER WAS ASKED, so this is read // before the two "nobody was asked" answers below. A payment refusal is a // fact about the account rather than about one upstream or one list, and @@ -2927,6 +2942,9 @@ func apiError(status int, payload []byte) error { // built and is therefore true of every APIError this build makes, including // the ones a stream raises in-band. failure.Overflow = overflowRefusal(status, failure.Code, failure.Message, failure.Raw) + if status == http.StatusBadRequest && refusesReasoning(payload) { + failure.ReasoningUnsupported = true + } readContextLimit(failure) return failure } diff --git a/internal/provider/dispatch.go b/internal/provider/dispatch.go index 3f2b58656a..5344abe45a 100644 --- a/internal/provider/dispatch.go +++ b/internal/provider/dispatch.go @@ -846,8 +846,9 @@ func (c *Client) send(ctx context.Context, request *ai.Request, knobs callKnobs, } // A context refusal needs a smaller request, even when the router names // the machine that refused it. Return it to the conversation before an - // endpoint walk can resend the same oversized bytes. - if failure, ok := RefusalFrom(apiError(response.StatusCode, peek)); ok && failure.Overflow { + // endpoint walk can resend the same oversized bytes. A model capability + // refusal also belongs to its one repair rather than an endpoint walk. + if failure, ok := RefusalFrom(apiError(response.StatusCode, peek)); ok && (failure.Overflow || failure.ReasoningUnsupported) { response.Body = rewound(peek, response.Body) sharedLimiter.release(false, 0) return response, nil diff --git a/internal/provider/quirks.go b/internal/provider/quirks.go index 474814e213..fe316b0940 100644 --- a/internal/provider/quirks.go +++ b/internal/provider/quirks.go @@ -66,6 +66,8 @@ type quirksStore struct { // fields. It is per model because one incompatible endpoint must not erase // continuity for every reasoning model routed through this process. noReasoningReplay map[string]time.Time + // noReasoning is process-local evidence that the model accepts no knob. + noReasoning map[string]time.Time // answerCut is the sixth learned fact and the only one that is a NUMBER // rather than a date: how many completion tokens a model spent before its // structured answer was cut off, keyed by model and by the lane that asked. @@ -99,6 +101,7 @@ type quirksStore struct { } var quirks = &quirksStore{ + noReasoning: map[string]time.Time{}, mandatory: map[string]time.Time{}, disableIgnored: map[string]time.Time{}, noCacheControl: map[string]time.Time{}, @@ -612,6 +615,7 @@ func (q *quirksStore) forget() { q.noCacheControl = map[string]time.Time{} q.noReasoningBudget = map[string]time.Time{} q.noReasoningReplay = map[string]time.Time{} + q.noReasoning = map[string]time.Time{} q.answerCut = map[string]int{} q.servedWindow = map[string]int{} q.contextLimits = map[string]ContextLimit{} diff --git a/internal/provider/quirks_test.go b/internal/provider/quirks_test.go index e7ffdfd7c3..fad39c4fb6 100644 --- a/internal/provider/quirks_test.go +++ b/internal/provider/quirks_test.go @@ -224,6 +224,7 @@ func quirksAt(t *testing.T, models ...string) string { delete(quirks.noCacheControl, normalizeModel(model)) delete(quirks.noReasoningBudget, normalizeModel(model)) delete(quirks.noReasoningReplay, normalizeModel(model)) + delete(quirks.noReasoning, normalizeModel(model)) } quirks.path = "" }) diff --git a/internal/provider/reasoning_sync_test.go b/internal/provider/reasoning_sync_test.go new file mode 100644 index 0000000000..ee84affd72 --- /dev/null +++ b/internal/provider/reasoning_sync_test.go @@ -0,0 +1,81 @@ +package provider + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/Agent-Field/codeaf/internal/effort" +) + +func TestReasoningRefusalS2(t *testing.T) { + for _, message := range []string{`\"llama3.2:latest\" does not support thinking`, `This model does not support reasoning`} { + t.Run(message, func(t *testing.T) { + quirksAt(t, "ollama/llama3.2:latest") + recorded := &capture{} + cfg := Config{APIKey: "test-key", BaseURL: "http://provider.test", Model: "ollama/llama3.2:latest"} + cfg.HTTPClient = handlerClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recorded.record(r) + w.Header().Set("Content-Type", "application/json") + if recorded.body(recorded.count() - 1)["reasoning"] != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"` + message + `","code":400}}`)) + return + } + _, _ = w.Write([]byte(`{"choices":[{"finish_reason":"stop","message":{"role":"assistant","content":"ok"}}]}`)) + })) + ctx := WithConfiguredEffortRung(context.Background(), effort.High) + for i := range 2 { + client, err := NewClient(cfg) + if err != nil { + t.Fatal(err) + } + if _, err = client.CompleteWithMessages(ctx, userMessages("hello")); err != nil { + t.Fatalf("S2 call %d: %v", i, err) + } + if got := recorded.count(); got != i+2 { + t.Fatalf("S2 sent %d requests, want %d", got, i+2) + } + } + if recorded.body(0)["reasoning"] == nil || recorded.body(1)["reasoning"] != nil || recorded.body(2)["reasoning"] != nil { + t.Fatal("S2 must remove reasoning from repair and future clients") + } + }) + } +} + +func TestCatalogDisablesReasoningS2(t *testing.T) { + client, recorded := newTestClient(t, Config{SupportsParameter: func(string, string) (bool, bool) { return false, true }}) + ctx := WithConfiguredEffortRung(context.Background(), effort.High) + if _, err := client.CompleteWithMessages(ctx, userMessages("hello")); err != nil { + t.Fatal(err) + } + if recorded.count() != 1 || recorded.body(0)["reasoning"] != nil { + t.Fatal("S2 catalog refusal must prevent reasoning before sending") + } +} + +func TestReasoningRefusalIsPermanentS2(t *testing.T) { + quirksAt(t, "refusal/always") + recorded := &capture{} + client, err := NewClient(Config{ + APIKey: "test-key", BaseURL: "http://provider.test", Model: "refusal/always", + HTTPClient: handlerClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recorded.record(r) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"This model does not support thinking","metadata":{"provider_name":"local"}}}`)) + })), + }) + if err != nil { + t.Fatal(err) + } + _, err = client.CompleteWithMessages(WithConfiguredEffortRung(context.Background(), effort.High), userMessages("hello")) + var failure *APIError + if !errors.As(err, &failure) || !failure.ReasoningUnsupported || !failure.OurRequest() { + t.Fatalf("S2 refusal lost its permanent classification: %v", err) + } + if recorded.count() != 2 { + t.Fatalf("S2 sent %d requests, want one refusal and one repair", recorded.count()) + } +} diff --git a/internal/provider/wire.go b/internal/provider/wire.go index 517b9baa1a..483a753393 100644 --- a/internal/provider/wire.go +++ b/internal/provider/wire.go @@ -175,6 +175,28 @@ func ReasoningBudgetRefused(model string) bool { return reasoningBudgetRefused(m func reasoningBudgetRefused(model string) bool { return quirks.knowsNoReasoningBudget(model) } +// refusesReasoning recognizes a model's refusal of the entire reasoning knob. +// Budget and disabled-reasoning refusals have narrower, independent repairs. +func refusesReasoning(payload []byte) bool { + text := strings.ToLower(string(payload)) + for _, refusal := range []string{ + "does not support thinking", "does not support reasoning", + "thinking is not supported", "reasoning is not supported", + "thinking is unsupported", "reasoning is unsupported", + } { + if strings.Contains(text, refusal) { + return true + } + } + return false +} + +// ReasoningUnsupported identifies the permanent capability refusal after the +// adapter's one repair, including errors carried over the app's remote wire. +func ReasoningUnsupported(status int, message string) bool { + return status == 400 && refusesReasoning([]byte(message)) +} + // refusesReasoningBudget reads a 400 body for the second complaint this adapter // can repair by itself: the thinking budget is not a field this endpoint takes. // @@ -480,6 +502,14 @@ func (c *Client) effortAsked(requested effortRequest) effortRequest { // resolveEffort decides whether the knob may travel, and in what shape. func (c *Client) resolveEffort(model string, requested effortRequest) Effort { + if quirks.recorded(func(q *quirksStore) map[string]time.Time { return q.noReasoning }, model) { + return EffortNone + } + if c.config.SupportsParameter != nil { + if supported, known := c.config.SupportsParameter(model, "reasoning"); known && !supported { + return EffortNone + } + } requested = c.effortAsked(requested) effort := c.requestedEffort(model, requested) // A model that reasons unconditionally answers the disable with a 400, and diff --git a/internal/raftstore/sqlite.go b/internal/raftstore/sqlite.go index 5a831a4753..cee972d037 100644 --- a/internal/raftstore/sqlite.go +++ b/internal/raftstore/sqlite.go @@ -10,11 +10,14 @@ import ( "encoding/binary" "encoding/json" "errors" + "fmt" "net/url" "os" "path/filepath" + "strings" "sync" + "github.com/Agent-Field/codeaf/internal/fsreplace" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" @@ -141,13 +144,13 @@ func Open(path string, identity Identity) (*SQLite, error) { var info os.FileInfo info, err = os.Lstat(abs) if err == nil && (!info.Mode().IsRegular() || info.Size() == 0) { - err = ErrCorrupt + err = fmt.Errorf("%w: %s (regular=%t, bytes=%d)", ErrCorrupt, abs, info.Mode().IsRegular(), info.Size()) } } if err != nil { return nil, err } - u := url.URL{Scheme: "file", Path: abs} + u := sqliteURI(abs) q := url.Values{"_txlock": {"immediate"}, "_pragma": {"busy_timeout(10000)", "journal_mode(WAL)", "synchronous(FULL)"}, "mode": {"rw"}} u.RawQuery = q.Encode() db, err := sql.Open("sqlite", u.String()) @@ -167,7 +170,7 @@ func Open(path string, identity Identity) (*SQLite, error) { var mode string var synchronous int if err = db.QueryRowContext(ctx, `PRAGMA journal_mode`).Scan(&mode); err != nil { - return nil, ErrCorrupt + return nil, errors.Join(ErrCorrupt, err) } if err = db.QueryRowContext(ctx, `PRAGMA synchronous`).Scan(&synchronous); err != nil || mode != "wal" || synchronous != 2 { return nil, ErrCorrupt @@ -197,18 +200,9 @@ func Open(path string, identity Identity) (*SQLite, error) { } // SQLite syncs the WAL before COMMIT returns. Sync the containing // directory too so first creation survives losing power immediately. - dir, err := os.Open(filepath.Dir(abs)) - if err != nil { - return nil, err - } - err = dir.Sync() - closeErr := dir.Close() - if err != nil { + if err := fsreplace.SyncDir(filepath.Dir(abs)); err != nil { return nil, err } - if closeErr != nil { - return nil, closeErr - } } if _, err = s.Recover(ctx); err != nil { return nil, err @@ -217,6 +211,17 @@ func Open(path string, identity Identity) (*SQLite, error) { return s, nil } +// sqliteURI keeps a Windows drive in the absolute path, rather than turning +// it into a URI authority such as file://C:. SQLite accepts /C:/ as a drive +// path, with forward slashes even when the native filename uses backslashes. +func sqliteURI(abs string) url.URL { + path := filepath.ToSlash(abs) + if len(abs) >= 3 && (abs[0] >= 'A' && abs[0] <= 'Z' || abs[0] >= 'a' && abs[0] <= 'z') && abs[1] == ':' && (abs[2] == '\\' || abs[2] == '/') { + path = "/" + strings.ReplaceAll(abs, `\`, "/") + } + return url.URL{Scheme: "file", Path: path} +} + func (s *SQLite) Close() error { s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/raftstore/windows_uri_test.go b/internal/raftstore/windows_uri_test.go new file mode 100644 index 0000000000..e59e7675e7 --- /dev/null +++ b/internal/raftstore/windows_uri_test.go @@ -0,0 +1,22 @@ +package raftstore + +import "testing" + +func TestA4WindowsStoreUsesAbsoluteFileURI(t *testing.T) { + for _, path := range []string{`C:\Users\Abir\relay store\raft\id_x\1.db`, `C:/Users/Abir/relay store/raft/id_x/1.db`} { + u := sqliteURI(path) + if got, want := u.String(), "file:///C:/Users/Abir/relay%20store/raft/id_x/1.db"; got != want { + t.Errorf("A4: %q becomes %q, want %q", path, got, want) + } + if u.Host != "" { + t.Errorf("A4: drive letter became URI authority: %q", u.Host) + } + } +} + +func TestA4UnixColonDirectoryRemainsAnAbsolutePath(t *testing.T) { + u := sqliteURI("/:/relay/1.db") + if got := u.String(); got != "file:///:/relay/1.db" { + t.Fatalf("A4: Unix colon directory was treated as a drive: %q", got) + } +} diff --git a/internal/relayserve/ingress.go b/internal/relayserve/ingress.go index 88af19e443..b848ce18fa 100644 --- a/internal/relayserve/ingress.go +++ b/internal/relayserve/ingress.go @@ -1,6 +1,7 @@ package relayserve import ( + "fmt" "io" "net/http" @@ -27,21 +28,28 @@ func (n *namespaces) clusterAPI(w http.ResponseWriter, r *http.Request) { } d, err := n.activate(r.Context(), cs.IdentityID(request.IdentityKey), request.IdentityKey) if err != nil { + n.unavailable("cluster activation", err) http.Error(w, cs.LeadershipUnavailable, 503) return } clock, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: d.Cluster, Actor: d.effects.Actor, Session: "ingress", Sequence: 1, Now: n.now().UnixMilli(), Change: cs.Mutation{DeviceSeen: &cs.DeviceSeen{Device: actor}}}, d.effects.Key) if err != nil { + n.unavailable("ingress clock", err) http.Error(w, cs.LeadershipUnavailable, 503) return } result, err := d.host.ClusterAPI(r.Context(), request, clock) if err != nil { + n.unavailable("cluster operation", err) http.Error(w, "the selected work is unavailable; refresh before trying again", 503) return } raw, err = cs.Encode(result) if err != nil || len(raw) > cs.MaxEvaluationBytes { + if err == nil { + err = fmt.Errorf("cluster result exceeds %d bytes: %d", cs.MaxEvaluationBytes, len(raw)) + } + n.unavailable("cluster encoding", err) http.Error(w, cs.LeadershipUnavailable, 503) return } diff --git a/internal/relayserve/namespaces.go b/internal/relayserve/namespaces.go index 5be92ce58c..7764725849 100644 --- a/internal/relayserve/namespaces.go +++ b/internal/relayserve/namespaces.go @@ -43,6 +43,9 @@ type namespaces struct { groups map[string]*nativeNamespace mu sync.Mutex dirs map[string]*directory.SQLite + logf func(string, ...any) + faultMu sync.Mutex + faults map[string]time.Time } func newNamespaces(root string, now func() time.Time, grace directory.GraceBounds, watchers int) *namespaces { @@ -52,6 +55,12 @@ func newNamespaces(root string, now func() time.Time, grace directory.GraceBound return &namespaces{root: root, now: now, grace: grace, watchers: watchers, dirs: map[string]*directory.SQLite{}, groups: map[string]*nativeNamespace{}} } +func (n *namespaces) group(id string) *nativeNamespace { + n.mu.Lock() + defer n.mu.Unlock() + return n.groups[id] +} + // directory resolves only the irrevocably published consensus owner. Source // access stays inside activation and never becomes a fallback request writer. func (n *namespaces) directory(id string) (directory.Directory, error) { diff --git a/internal/relayserve/native.go b/internal/relayserve/native.go index eeef9eb25c..b0738fbd59 100644 --- a/internal/relayserve/native.go +++ b/internal/relayserve/native.go @@ -19,6 +19,7 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/fsreplace" "github.com/Agent-Field/codeaf/internal/raftdirectory" "github.com/Agent-Field/codeaf/internal/raftstore" "github.com/Agent-Field/codeaf/internal/rafttransport" @@ -61,9 +62,7 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat } n.activation.Lock() defer n.activation.Unlock() - n.mu.Lock() - d := n.groups[id] - n.mu.Unlock() + d := n.group(id) if d != nil { return d, nil } @@ -126,17 +125,12 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat if e != nil { return nil, e } - if e = os.Rename(tmp, provisionPath); e != nil { + if e = fsreplace.Rename(tmp, provisionPath); e != nil { return nil, e } // Newly created ancestors must survive before the source UUID freezes. for _, dir := range []string{path, filepath.Dir(path), n.root, filepath.Dir(n.root)} { - parent, e := os.Open(dir) - if e != nil { - return nil, e - } - e = errors.Join(parent.Sync(), parent.Close()) - if e != nil { + if e = fsreplace.SyncDir(dir); e != nil { return nil, e } } @@ -244,9 +238,9 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat return nil, err } n.mu.Lock() + defer n.mu.Unlock() d.armMembers() n.groups[id] = d - n.mu.Unlock() return d, nil } func (d *nativeNamespace) project(s *raftstore.SQLite, a raftstore.Application) error { @@ -407,12 +401,14 @@ func (n *namespaces) authenticate(r *http.Request, body []byte) (string, string, if errors.Is(err, wireauth.ErrGone) { return "", "", err } + n.unavailable("directory activation", err) return "", "", errors.Join(wireauth.ErrUnavailable, err) } if err = d.authority(r.Context(), actor, false); err != nil { if errors.Is(err, wireauth.ErrRevoked) { return "", "", err } + n.unavailable("directory authority", err) return "", "", errors.Join(wireauth.ErrUnavailable, err) } *r = *r.WithContext(context.WithValue(r.Context(), blobActorKey{}, actor)) @@ -430,20 +426,18 @@ type nativeClient struct { owner *nativeNamespace } +// Preserve the signed quiet-release mutation through the presence wrapper. +// Falling back to Release changes the request that the consensus adapter checks. +func (c *nativeClient) ReleaseQuiet(ctx context.Context, id string, fence uint64) error { + return directory.ReleaseQuiet(ctx, c.Client, id, fence) +} + func (c *nativeClient) Presence(ctx context.Context) (directory.PresenceView, error) { p, e := c.Client.Presence(ctx) if e != nil { return p, e } - online := c.owner.feed.OnlineSet() - for id, v := range p.Devices { - v.Online = online[id] - if v.Online { - v.LastSeen = p.Now - } - p.Devices[id] = v - } - return p, nil + return c.owner.feed.WithPresence(p), nil } func (c *nativeClient) ApproveRequest(ctx context.Context, code string, a directory.Approval) error { if err := c.owner.effects.Admit(ctx, c.owner.now().UnixMilli(), c.owner.links, code); err != nil { @@ -522,12 +516,8 @@ func advanceNativeTransport(path string) (uint64, error) { if err != nil { return 0, err } - if err = os.Rename(f.Name(), path); err != nil { - return 0, err - } - dir, err := os.Open(filepath.Dir(path)) - if err != nil { + if err = fsreplace.Rename(f.Name(), path); err != nil { return 0, err } - return generation, errors.Join(dir.Sync(), dir.Close()) + return generation, fsreplace.SyncDir(filepath.Dir(path)) } diff --git a/internal/relayserve/personal.go b/internal/relayserve/personal.go index ddc573cfe4..1aa06d94b6 100644 --- a/internal/relayserve/personal.go +++ b/internal/relayserve/personal.go @@ -38,10 +38,9 @@ func (n *namespaces) enrollPersonal(w http.ResponseWriter, r *http.Request) { http.Error(w, "invalid enrollment", 403) return } - n.mu.Lock() - d := n.groups[id] - n.mu.Unlock() + d := n.group(id) if d == nil { + n.unavailable("personal enrollment", errors.New("authenticated namespace is absent")) http.Error(w, "cluster is unavailable", 503) return } @@ -81,9 +80,7 @@ func (n *namespaces) exchangePersonal(w http.ResponseWriter, r *http.Request) { http.Error(w, "invalid peer request", 400) return } - n.mu.Lock() - d := n.groups[request.Scope.Identity] - n.mu.Unlock() + d := n.group(request.Scope.Identity) if d == nil && checkedID(request.Scope.Identity) == nil { // A returning peer may activate an EXISTING durable namespace, but an // unregistered key cannot create storage or invent a cluster root. diff --git a/internal/relayserve/presence_restart_test.go b/internal/relayserve/presence_restart_test.go new file mode 100644 index 0000000000..0caeadad56 --- /dev/null +++ b/internal/relayserve/presence_restart_test.go @@ -0,0 +1,45 @@ +package relayserve_test + +import ( + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" +) + +func TestNativePresenceP2RestartKeepsUnseenComputerUnknown(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + b := newDevice(t, id, t.TempDir()) + for _, d := range []device{a, b} { + if err := r.dir(d).PutDevice(bg, d.dev.ID(), directory.Device{V: 1}); err != nil { + t.Fatal(err) + } + } + r.restart() + view, err := r.dir(a).Presence(bg) + if err != nil { + t.Fatal(err) + } + if _, known := view.Devices[b.dev.ID()]; known { + t.Fatalf("P2 native restart invented offline presence: %+v", view) + } + conn, _, err := directorytest.DialWatch(bg, r.srv.URL, r.signer(b)) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + if _, _, err = conn.Read(bg); err != nil { + t.Fatal(err) + } + view, err = r.dir(a).Presence(bg) + if err != nil || !view.Devices[b.dev.ID()].Online { + t.Fatalf("P2 native relay did not observe returned computer: %+v, %v", view, err) + } + r.clock.Advance(directory.Window(directory.DefaultBeat) + time.Second) + view, err = r.dir(a).Presence(bg) + if peer, known := view.Devices[b.dev.ID()]; err != nil || !known || peer.Online { + t.Fatalf("P2 native relay did not report actual lapse: %+v, %v", view, err) + } +} diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index c675938503..1fdb1a1b53 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -118,6 +118,7 @@ func New(cfg Config) *Service { } links := directory.NewLinks(now, directory.DefaultLinkLimits) svc.ns = newNamespaces(cfg.Store, now, cfg.Grace, cfg.MaxWatchers) + svc.ns.logf = cfg.Logf svc.ns.after = cfg.After svc.ns.links = links mux.HandleFunc(clusterstate.ClusterAPIPath, svc.ns.clusterAPI) diff --git a/internal/relayserve/unavailable.go b/internal/relayserve/unavailable.go new file mode 100644 index 0000000000..bb0da5a493 --- /dev/null +++ b/internal/relayserve/unavailable.go @@ -0,0 +1,35 @@ +package relayserve + +import ( + "time" +) + +// unavailable keeps the local cause that a generic 503 cannot explain. Each +// cause has its own one-minute budget, and a noisy failing store cannot grow +// the relay's diagnostic memory without bound. +func (n *namespaces) unavailable(phase string, err error) { + if n.logf == nil || err == nil { + return + } + key := err.Error() + n.faultMu.Lock() + defer n.faultMu.Unlock() + now := n.now() + if last, ok := n.faults[key]; ok && now.Sub(last) < time.Minute { + return + } + if n.faults == nil { + n.faults = map[string]time.Time{} + } + if len(n.faults) >= 64 { + var oldest string + for cause, at := range n.faults { + if oldest == "" || at.Before(n.faults[oldest]) { + oldest = cause + } + } + delete(n.faults, oldest) + } + n.faults[key] = now + n.logf("relay unavailable phase=%s cause=%q", phase, err.Error()) +} diff --git a/internal/relayserve/unavailable_test.go b/internal/relayserve/unavailable_test.go new file mode 100644 index 0000000000..a6c456815a --- /dev/null +++ b/internal/relayserve/unavailable_test.go @@ -0,0 +1,30 @@ +package relayserve + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/directory" +) + +func TestA4UnavailableLogsEachCauseWithBoundedRate(t *testing.T) { + now := time.Unix(100, 0) + n := newNamespaces(t.TempDir(), func() time.Time { return now }, directory.GraceBounds{}, 0) + var lines []string + n.logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } + for range 20 { + n.unavailable("activate", errors.New("sync raft/1.db: access denied")) + } + n.unavailable("activate", errors.New("unreadable provision")) + if len(lines) != 2 || !strings.Contains(lines[0], "access denied") || !strings.Contains(lines[1], "unreadable provision") { + t.Fatalf("A4: repeated or missing causes: %q", lines) + } + now = now.Add(time.Minute) + n.unavailable("activate", errors.New("sync raft/1.db: access denied")) + if len(lines) != 3 { + t.Fatalf("A4: rate limit did not expire: %q", lines) + } +} diff --git a/internal/session/agent.go b/internal/session/agent.go index 0353d4721d..8c1809ba78 100644 --- a/internal/session/agent.go +++ b/internal/session/agent.go @@ -2066,13 +2066,17 @@ func (a *Agent) startTurnLocked(ctx context.Context, user userMessage, watcher * // the transcript on the line above; a shell opening additionally waits // for an ordinary reply to explain its output. Naming is started under // this lock so two Submits racing to be first cannot buy two names. - if user.bash == "" { + if user.bash == "" && a.config.PrepareModel == nil { a.startTitleLocked() } // A DESKTOP CONVERSATION IS NAMED FROM WHAT THE PERSON OPENED WITH, and // the shell opening is one of the things that decides it too // (intentname.go). - a.startIntentNameLocked(user) + // A continued chat may still need model setup. Its turn checks readiness + // before starting either naming errand, which also calls a provider. + if a.config.PrepareModel == nil { + a.startIntentNameLocked(user) + } } // THEIR NEXT WORDS ARE WHAT CHANGED. A generation Interrupt minted waits // here for the sentence that follows Esc, and that sentence is the one diff --git a/internal/session/app_processes.go b/internal/session/app_processes.go index 422bb587ce..4c281455aa 100644 --- a/internal/session/app_processes.go +++ b/internal/session/app_processes.go @@ -397,6 +397,68 @@ func (a *Agent) finishProcessRecords() { } } +// reconcileProcessRecords carries known job endings into their process files. +// The caller holds the journal claim. Unfinished checkpoints and reused job +// numbers cannot turn an unknown process into an invented ending. +func reconcileProcessRecords(file string, document taskDocument) error { + processRecordMu.Lock() + defer processRecordMu.Unlock() + s := processStore{dir: filepath.Dir(file)} + root, err := s.openRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer root.Close() + head, found, err := readHead(root) + if err != nil || !found { + return err + } + for _, n := range knownJobEndings(document) { + old, found, err := readRecord(root, n.ID) + if err != nil { + return err + } + if !found || old.Over() || !sameJobRun(old, n) { + continue + } + // The process record owns its command, name, kind and workspace. Only + // the ending comes from the checkpoint that observed this same run. + old.State, old.Ended = n.State, n.Ended + if n.Elapsed > 0 { + old.Elapsed = n.Elapsed + } + old.ExitCode, old.ExitCodeUnknown = n.ExitCode, n.ExitCodeUnknown + if _, err := s.put(root, &head, old, true); err != nil { + return err + } + } + return nil +} + +// knownJobEndings is each settled, unparked job run of a checkpoint, by job +// number: the endings a checkpoint observed for itself. +func knownJobEndings(document taskDocument) map[int]JobNotice { + ended := map[int]JobNotice{} + for _, record := range document.Runs { + if record.Kind != TaskKindJob || !record.State.settled() || record.Park != nil { + continue + } + if n, ok := jobNoticeFromRow(runRowNotice(record)); ok { + ended[n.ID] = n + } + } + return ended +} + +// sameJobRun says a process record and a checkpoint ending describe the same +// run, so a reused job number never borrows another run's ending. +func sameJobRun(record, ending JobNotice) bool { + return !record.Started.IsZero() && !ending.Started.IsZero() && record.Started.Equal(ending.Started) && record.ToolCallID == ending.ToolCallID +} + // mergeKeptJobs fills gaps left by engines that kept jobs only in the roster. // The process shelf owns any job it already knows, including a reused number. func mergeKeptJobs(rows, kept []JobNotice) []JobNotice { diff --git a/internal/session/conversation_models.go b/internal/session/conversation_models.go index 709b3480c2..4dc9de4761 100644 --- a/internal/session/conversation_models.go +++ b/internal/session/conversation_models.go @@ -43,6 +43,36 @@ func (a *Agent) ConversationModelPair() (ConversationModels, bool) { // SetConversationModels changes future requests only. In particular it never // calls hearModelLocked, whose terminal behavior may cancel an unproductive stream. func (a *Agent) SetConversationModels(pair ConversationModels) ([]ModelSwitch, error) { + return a.setConversationModels(pair, "") +} + +// ContinueWithModel applies a receiver's usable model through the conversation's +// existing main/fallback rules and records its specific explanation once. +func (a *Agent) ContinueWithModel(model, notice string) error { + if pair, ok := a.ConversationModelPair(); ok { + pair.Main = model + if pair.Fallback == model { + pair.Fallback = "" + } + _, err := a.setConversationModels(pair, notice) + return err + } + a.SetModel(model) + if notice != "" { + record := func() ModelSwitch { + a.mu.Lock() + defer a.mu.Unlock() + a.continuedModelNoticed = model + return a.recordModelSwitchLocked(notice) + }() + a.publishModelSwitch(record) + } + return nil +} + +// A continued chat can supply the reason for its main-model change. Keeping +// that reason on the same write path avoids recording a second generic notice. +func (a *Agent) setConversationModels(pair ConversationModels, mainNotice string) ([]ModelSwitch, error) { if a == nil { return nil, errors.New("conversation is unavailable") } @@ -90,7 +120,12 @@ func (a *Agent) SetConversationModels(pair ConversationModels) ([]ModelSwitch, e } notices := []ModelSwitch{} if previous.Main != pair.Main { - notices = append(notices, a.recordModelSwitchLocked("Main model changed from "+previous.Main+" to "+pair.Main+"."+suffix)) + if mainNotice == "" { + mainNotice = "Main model changed from " + previous.Main + " to " + pair.Main + "." + suffix + } else { + a.continuedModelNoticed = pair.Main + } + notices = append(notices, a.recordModelSwitchLocked(mainNotice)) } if previous.Fallback != pair.Fallback && (hadPair || previous.Main != pair.Main) { from, to := previous.Fallback, pair.Fallback diff --git a/internal/session/digest.go b/internal/session/digest.go index b39ee1972f..a509431119 100644 --- a/internal/session/digest.go +++ b/internal/session/digest.go @@ -143,7 +143,12 @@ func (d Digest) Spend() (usd float64, tokens int) { // transcript cannot know (effort, approval, referred places, working copies, // archive marks) are m's and pass through untouched. func (d Digest) Derive(m Meta) Meta { - m.ID, m.Model = d.ID, d.Model + m.ID = d.ID + // The journal header names the original model, not later conversation + // choices. A sealed identity carries the current main/fallback rules. + if m.ModelIdentity == nil { + m.Model = d.Model + } if d.Workspace != "" { // a sealed journal names none: the summary already knows where it is here m.Workspace = d.Workspace } diff --git a/internal/session/loop.go b/internal/session/loop.go index 4ce3eaf986..e6edc3be74 100644 --- a/internal/session/loop.go +++ b/internal/session/loop.go @@ -337,6 +337,18 @@ func (a *Agent) turnCue(user userMessage) string { // a follow-up (agent.go) — an interrupted or faulted turn must not be the thing // that starts the next one. func (a *Agent) runTurn(ctx context.Context, hub *eventHub, user userMessage) bool { + if user.bash == "" { + resolved, ready := a.prepareModel(hub) + if !ready { + return false + } + if resolved && !user.empty() && !user.resumed { + a.mu.Lock() + a.startTitleLocked() + a.startIntentNameLocked(user) + a.mu.Unlock() + } + } // A newly admitted stage turn may do more work after an earlier report. // Withdraw that candidate before this turn can submit a newer one; runTurn // is serialized, so a later valid final report cannot be invalidated here. diff --git a/internal/session/metatruth.go b/internal/session/metatruth.go index 6080c0d6c6..0b27943b9f 100644 --- a/internal/session/metatruth.go +++ b/internal/session/metatruth.go @@ -1,11 +1,11 @@ package session -// metatruth.go splits a session's meta in two. The summary (title, model, -// spend, times) is a citation of the transcript and stays in meta.json, outside -// the sealed tree, where internal/cellindex rebuilds it. The truth is what no -// transcript can restate (the dials, the places, the unlanded trees, what was -// put away) and lives in .cell/session.json, so it is sealed with the -// workspace and travels with it. +// metatruth.go splits a session's meta in two. The summary (title, spend, +// times) is a citation of the transcript and stays in meta.json, outside the +// sealed tree, where internal/cellindex rebuilds it. The truth is what no +// transcript can restate (the model choice, the dials, the places, the unlanded +// trees, what was put away) and lives in .cell/session.json, so it is sealed +// with the workspace and travels with it. // // Trees records which worktrees exist; the worktree directories themselves are // workspace material with absolute paths, and they stay where they are. @@ -29,6 +29,9 @@ type SessionTruth struct { V uint16 `json:"V"` ChatURI string `json:"chat_uri,omitempty"` WorkspacePolicy *WorkspacePolicy `json:"workspace_policy,omitempty"` + Model string `json:"model,omitempty"` + ModelProvider string `json:"model_provider,omitempty"` + ModelIdentity *ModelIdentity `json:"model_identity,omitempty"` LaunchDir string `json:"launch_dir,omitempty"` Owned bool `json:"owned,omitempty"` Effort string `json:"effort,omitempty"` @@ -42,12 +45,13 @@ type SessionTruth struct { const truthVersion = 1 func (m Meta) truth() SessionTruth { - return SessionTruth{V: truthVersion, ChatURI: m.ChatURI, WorkspacePolicy: m.WorkspacePolicy, LaunchDir: m.LaunchDir, Owned: m.Owned, Effort: m.Effort, Approval: m.Approval, Places: m.Places, Trees: m.Trees, Archived: m.Archived, ArchivedTasks: m.ArchivedTasks} + return SessionTruth{V: truthVersion, ChatURI: m.ChatURI, WorkspacePolicy: m.WorkspacePolicy, Model: m.Model, ModelProvider: m.ModelProvider, ModelIdentity: m.ModelIdentity, LaunchDir: m.LaunchDir, Owned: m.Owned, Effort: m.Effort, Approval: m.Approval, Places: m.Places, Trees: m.Trees, Archived: m.Archived, ArchivedTasks: m.ArchivedTasks} } func (m *Meta) setTruth(t SessionTruth) { m.ChatURI = t.ChatURI m.WorkspacePolicy = t.WorkspacePolicy + m.Model, m.ModelProvider, m.ModelIdentity = t.Model, t.ModelProvider, t.ModelIdentity m.LaunchDir, m.Owned, m.Effort, m.Approval = t.LaunchDir, t.Owned, t.Effort, t.Approval m.Places, m.Trees, m.Archived, m.ArchivedTasks = t.Places, t.Trees, t.Archived, t.ArchivedTasks } @@ -73,6 +77,11 @@ func overlayTruth(dir string, meta Meta) (Meta, error) { if err != nil { return Meta{}, err } + // Older cells kept their model only in the local summary. Preserve that + // choice until its next save writes the additive sealed fields. + if t.Model == "" && t.ModelIdentity == nil { + t.Model, t.ModelProvider, t.ModelIdentity = meta.Model, meta.ModelProvider, meta.ModelIdentity + } meta.setTruth(mapTruth(t, codecOf(dir, meta).resolve)) return meta, nil } diff --git a/internal/session/model_identity.go b/internal/session/model_identity.go index a8f6be3168..7bf1abcde8 100644 --- a/internal/session/model_identity.go +++ b/internal/session/model_identity.go @@ -84,7 +84,9 @@ func (a *Agent) startConversationModelRequest(model, provider string, pair Conve previous := a.modelIdentity.Requested var notice *ModelSwitch isFallback := pair.Main != "" && model == pair.Fallback - if pair.Main != "" && (isFallback && (a.fallbackNoticeOwed || previous != model) || previous != "" && previous != model) { + continued := !isFallback && model == a.continuedModelNoticed + a.continuedModelNoticed = "" + if pair.Main != "" && !continued && (isFallback && (a.fallbackNoticeOwed || previous != model) || previous != "" && previous != model) { text := "Using main model " + model + "." if model == pair.Fallback { text = "Main model " + pair.Main + " failed. Using fallback model " + model + "." diff --git a/internal/session/modelmeta_test.go b/internal/session/modelmeta_test.go index 3448809e90..ff2b93a550 100644 --- a/internal/session/modelmeta_test.go +++ b/internal/session/modelmeta_test.go @@ -1,11 +1,40 @@ package session import ( + "encoding/json" + "os" + "path/filepath" "sync" "testing" "time" + + "github.com/Agent-Field/codeaf/internal/modelsource" ) +func TestModelMetaC1RemembersProviderBeforeTheFirstCall(t *testing.T) { + dir := t.TempDir() + agent, _ := newTestAgent(t, &scriptedCompleter{}, func(cfg *Config) { + cfg.Place = Place{Dir: dir} + cfg.Sources = modelsource.NewSet( + modelsource.Connected{Source: modelsource.Source{ID: "openrouter", Written: "openrouter"}, Key: "router"}, + modelsource.Connected{Source: modelsource.Source{ID: "custom-local", Written: "local-models"}, Key: "local"}, + ) + }) + agent.SetModel("local-models/never-called") + agent.SettleWrites() + raw, err := os.ReadFile(filepath.Join(dir, "meta.json")) + if err != nil { + t.Fatal(err) + } + var saved map[string]any + if err := json.Unmarshal(raw, &saved); err != nil { + t.Fatal(err) + } + if saved["model"] != "local-models/never-called" || saved["modelProvider"] != "custom-local" { + t.Fatalf("C1 saved model/provider = %v/%v", saved["model"], saved["modelProvider"]) + } +} + // A different window must see a chosen model even before another turn runs. func TestModelChoicePublishesSavedConfigurationWithoutAnotherTurn(t *testing.T) { dir := t.TempDir() diff --git a/internal/session/place.go b/internal/session/place.go index 52e0142b90..0df84cfcec 100644 --- a/internal/session/place.go +++ b/internal/session/place.go @@ -269,6 +269,9 @@ type Meta struct { Owned bool `json:"owned,omitempty"` // Model is the conversation's model at last save, for the picker row. Model string `json:"model,omitempty"` + // ModelProvider identifies the selected model's service even before its + // first call, when execution history cannot identify a custom alias. + ModelProvider string `json:"modelProvider,omitempty"` // Build names the codeaf that most recently wrote this identity. Build string `json:"build,omitempty"` // Effort is the rung on the effort ladder this conversation was set to — diff --git a/internal/session/placemeta.go b/internal/session/placemeta.go index 1a0fdccf57..6a725d43e7 100644 --- a/internal/session/placemeta.go +++ b/internal/session/placemeta.go @@ -70,6 +70,7 @@ func (a *Agent) stampUserLocked(text string) { meta.AutomaticModels = snapshot.AutomaticModels meta.ModelIdentity = snapshot.ModelIdentity meta.Model, meta.Effort, meta.Approval = snapshot.Model, snapshot.Effort, snapshot.Approval + meta.ModelProvider = snapshot.ModelProvider meta.Places, meta.Trees = snapshot.Places, snapshot.Trees meta.LastUserAt = at if strings.TrimSpace(meta.Title) == "" { @@ -289,6 +290,7 @@ func (a *Agent) updateMeta(dir string, snapshot Meta, patch func(*Meta)) { meta.AutomaticModels = snapshot.AutomaticModels meta.ModelIdentity = snapshot.ModelIdentity meta.Model, meta.Effort, meta.Approval = snapshot.Model, snapshot.Effort, snapshot.Approval + meta.ModelProvider = snapshot.ModelProvider } if meta.Places == nil { meta.Places = snapshot.Places @@ -328,6 +330,7 @@ func (a *Agent) fillMetaLocked(meta Meta) Meta { } if model := strings.TrimSpace(a.model); model != "" { meta.Model = model + meta.ModelProvider = a.config.serviceFor(model).Source.ID } // The rung is written whatever it is, absence included, because absence is // a value a person can choose their way back to: a session dialled to max @@ -391,6 +394,7 @@ func (a *Agent) stampMeta() { meta.AutomaticModels = snapshot.AutomaticModels meta.ModelIdentity = snapshot.ModelIdentity meta.Model, meta.Effort, meta.Approval = snapshot.Model, snapshot.Effort, snapshot.Approval + meta.ModelProvider = snapshot.ModelProvider meta.Places, meta.Trees = snapshot.Places, snapshot.Trees }) } diff --git a/internal/session/prepare_model.go b/internal/session/prepare_model.go new file mode 100644 index 0000000000..db09f79975 --- /dev/null +++ b/internal/session/prepare_model.go @@ -0,0 +1,33 @@ +package session + +// prepareModel runs on the serialized turn road, before auxiliary or main +// provider calls. The callback reads local readiness only. +func (a *Agent) prepareModel(hub *eventHub) (resolved, ready bool) { + a.mu.Lock() + prepare, model := a.config.PrepareModel, a.model + if prepare == nil { + a.mu.Unlock() + return false, true + } + sources := a.config.Sources.OrDefault(a.config.APIKey, a.config.BaseURL) + a.mu.Unlock() + replacement, notice, err := prepare(model, sources) + if err == nil && replacement != "" { + err = a.ContinueWithModel(replacement, notice) + notice = "" + } + if err != nil { + hub.send(Event{Kind: EventError, Err: err}) + return false, false + } + a.mu.Lock() + a.config.PrepareModel = nil + if notice != "" { + n := a.recordModelSwitchLocked(notice) + a.mu.Unlock() + a.publishModelSwitch(n) + } else { + a.mu.Unlock() + } + return true, true +} diff --git a/internal/session/prepare_model_e13_test.go b/internal/session/prepare_model_e13_test.go new file mode 100644 index 0000000000..58c2223830 --- /dev/null +++ b/internal/session/prepare_model_e13_test.go @@ -0,0 +1,52 @@ +package session + +import ( + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/modelsource" +) + +func TestPrepareModelN1OneSpecificNoticeAndFallbackRules(t *testing.T) { + for _, fallback := range []string{"", "local/default"} { + t.Run("fallback="+fallback, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "transcript.jsonl") + const notice = "This chat used remote/model on B. It isn't set up here, so it continues with local/default." + if err := SaveMeta(filepath.Dir(file), Meta{ID: "n1-chat", Model: "remote/model", ModelIdentity: &ModelIdentity{ + Selected: "remote/model", Requested: "remote/model", Last: "remote/model", + Models: &ConversationModels{Main: "remote/model", Fallback: fallback}, + }}); err != nil { + t.Fatal(err) + } + a, _ := newTestAgent(t, &scriptedCompleter{}, func(c *Config) { + c.SessionFile = file + c.Place = Place{Dir: filepath.Dir(file)} + c.PrepareModel = func(string, modelsource.Set) (string, string, error) { + return "local/default", notice, nil + } + }) + events, err := a.Submit(t.Context(), "continue here") + if err != nil { + t.Fatal(err) + } + var published []ModelSwitch + for event := range events { + if event.ModelSwitch != nil { + published = append(published, *event.ModelSwitch) + } + } + a.SettleWrites() + kept := ReadTranscript(file).ModelSwitches + want := 1 + if fallback != "" { + want++ + } + if len(kept) != want || len(published) != want || kept[0].Text != notice || published[0].Text != notice { + t.Fatalf("N1 notices: kept=%+v published=%+v", kept, published) + } + if pair, ok := a.ConversationModelPair(); !ok || pair.Main != "local/default" || pair.Fallback != "" { + t.Fatalf("N1 model rules: %+v", pair) + } + }) + } +} diff --git a/internal/session/process_recovery_e13_test.go b/internal/session/process_recovery_e13_test.go new file mode 100644 index 0000000000..d7f3432be8 --- /dev/null +++ b/internal/session/process_recovery_e13_test.go @@ -0,0 +1,116 @@ +package session + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The checkpoint knows this instant command ended, while its process record +// still carries the running snapshot left by the old engine. Deciding that +// needs no live conversation and writes nothing: the chat may be held by +// another computer, and its sealed files are not this check's to change. +func TestProcessRecoveryE13B1SettledCheckpointNeedsNoRecovery(t *testing.T) { + for _, state := range []string{"done", "failed"} { + t.Run(state, func(t *testing.T) { + file := seedE13ProcessRecovery(t, state) + record := filepath.Join(filepath.Dir(file), "processes", "1.json") + before, err := os.ReadFile(record) + if err != nil { + t.Fatal(err) + } + for range 2 { + if NeedsProcessRecovery(file) { + t.Fatal("B1 settled job demanded a live conversation") + } + } + after, err := os.ReadFile(record) + if err != nil || !bytes.Equal(before, after) { + t.Fatalf("B1 the check changed the process record: %v", err) + } + }) + } +} + +// The next conversation that holds the chat makes the known ending durable. +func TestProcessRecoveryE13B1HeldRecoveryRepairsRunningRecord(t *testing.T) { + file := seedE13ProcessRecovery(t, "done") + a, _ := newTestAgent(t, &scriptedCompleter{}, func(c *Config) { + c.SessionFile = file + c.Place = Place{Dir: filepath.Dir(file)} + }) + n, found, err := ReadProcessRecord(file, 1) + if err != nil || !found || n.State == JobRunning || n.Ended.IsZero() || n.Command != "say hi" { + t.Fatalf("B1 ending was not durable: %+v, %v", n, err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + if NeedsProcessRecovery(file) { + t.Fatal("B1 a repaired record demanded recovery again") + } +} + +func TestProcessRecoveryE13B2UnfinishedJobStillNeedsRecovery(t *testing.T) { + file := seedE13ProcessRecovery(t, "running") + if !NeedsProcessRecovery(file) { + t.Fatal("B2 unfinished job skipped recovery") + } +} + +func TestProcessRecoveryE13B2RecoveryDurablySettlesBothStores(t *testing.T) { + file := seedE13ProcessRecovery(t, "running") + a, _ := newTestAgent(t, &scriptedCompleter{}, func(c *Config) { + c.SessionFile = file + c.Place = Place{Dir: filepath.Dir(file)} + }) + n, found, err := ReadProcessRecord(file, 1) + if err != nil || !found || n.State != JobStopped { + t.Fatalf("B2 recovered ending was not durable: %+v, %v", n, err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + if NeedsProcessRecovery(file) { + t.Fatal("B1/B2 completed recovery demanded another live conversation") + } +} + +func TestProcessRecoveryE13B2AnotherRunsEndingIsNotBorrowed(t *testing.T) { + file := seedE13ProcessRecovery(t, "done") + n, _, _ := ReadProcessRecord(file, 1) + n.Started = n.Started.AddDate(0, 0, 1) + raw, err := json.Marshal(n) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(file), "processes", "1.json"), raw, 0600); err != nil { + t.Fatal(err) + } + if !NeedsProcessRecovery(file) { + t.Fatal("B2 a reused job id was settled by another run's checkpoint") + } +} + +func seedE13ProcessRecovery(t *testing.T, state string) string { + t.Helper() + dir := t.TempDir() + file := filepath.Join(dir, "transcript.jsonl") + if err := os.Mkdir(filepath.Join(dir, "processes"), 0700); err != nil { + t.Fatal(err) + } + files := map[string]string{ + "transcript.jsonl": "", + "processes/head.json": `{"epoch":"e13","high":1}`, + "processes/1.json": `{"ID":1,"Command":"say hi","Kind":"command","State":"running","Started":"2026-10-27T23:58:18Z","Elapsed":379625,"ToolCallID":"call_e13","StartedBy":"chat"}`, + "tasks.json": `{"type":"tasks","version":1,"seq":1,"runs":[{"id":1,"kind":"job","title":"say hi","state":"` + state + `","report":"job 1","job_command":"say hi","job_call_id":"call_e13","startedAt":"2026-10-27T23:58:18Z","endedAt":"2026-10-27T23:58:18.000379625Z"}]}`, + } + for name, raw := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(raw), 0600); err != nil { + t.Fatal(err) + } + } + return file +} diff --git a/internal/session/session.go b/internal/session/session.go index a38775f4e5..74de956b77 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -1482,6 +1482,11 @@ type Config struct { APIKey string BaseURL string Sources modelsource.Set + // PrepareModel resolves a continued chat's unavailable model before any + // model work. A failure leaves it pending for setup and the next turn; + // success applies the replacement through the conversation's model rules + // and records the notice once. Ordinary conversations leave it nil. + PrepareModel func(model string, sources modelsource.Set) (replacement, notice string, err error) // There is no app-attribution field here any more. The three that used to // be forwarded to the provider client — a referer, a title, a category @@ -2711,6 +2716,9 @@ type Agent struct { conversationModels *ConversationModels modelRequestSlot string fallbackNoticeOwed bool + // continuedModelNoticed suppresses the next main-request announcement when + // the receiver already explained this exact change on continuation. + continuedModelNoticed string modelIdentity ModelIdentity automaticModels atomic.Int32 diff --git a/internal/session/task_read.go b/internal/session/task_read.go index e1ab74721f..2d57cc0ecb 100644 --- a/internal/session/task_read.go +++ b/internal/session/task_read.go @@ -61,9 +61,16 @@ func NeedsTaskRecovery(sessionFile string) bool { return false } -// NeedsProcessRecovery finds saved running jobs even when the last presence -// was removed. Paging reads process records alone, never the conversation. +// NeedsProcessRecovery finds unfinished jobs even when the last presence was +// removed. A running record whose run the checkpoint already settled is not +// unfinished: the ending is known locally, so nobody has to open the chat (and +// take its lease) to learn it. This only reads; the record itself is repaired +// by the next recovery that holds the chat (recoverTasks). func NeedsProcessRecovery(sessionFile string) bool { + ended := map[int]JobNotice{} + if document, ok := loadTaskCheckpoint(taskCheckpointPath(sessionFile)); ok { + ended = knownJobEndings(document) + } ask := ProcessPageAsk{Limit: ProcessPageMax} for { page, err := ReadProcessPage(context.Background(), sessionFile, ask) @@ -71,7 +78,10 @@ func NeedsProcessRecovery(sessionFile string) bool { return true } for _, row := range page.Rows { - if row.State == JobRunning { + if row.State != JobRunning { + continue + } + if n, ok := ended[row.ID]; !ok || !sameJobRun(row.JobNotice, n) { return true } } diff --git a/internal/session/task_store.go b/internal/session/task_store.go index d1fc0dc17a..66e867e15f 100644 --- a/internal/session/task_store.go +++ b/internal/session/task_store.go @@ -1903,6 +1903,11 @@ func (a *Agent) recoverTasks() { // second crash between here and the first turn must not hand the same // interrupt to a second recovery. graph.checkpoint() + // A recovered job must settle both durable views. Leaving its process + // file running would make the next background look open this chat again. + if err := reconcileProcessRecords(a.config.SessionFile, graph.document()); err != nil { + log.Printf("session: settling recovered processes: %v", err) + } // AND THE HAND-BACK IS TOLD. Every landing the floor took off the model on the // way in is one ordinary task update, on the lane a surface already folds into // the row it is drawing — the same lane and the same shape the end-of-turn diff --git a/internal/session/truth_seal_test.go b/internal/session/truth_seal_test.go index 282339454a..eeebd78a9e 100644 --- a/internal/session/truth_seal_test.go +++ b/internal/session/truth_seal_test.go @@ -30,7 +30,7 @@ func TestCellLayoutKeepsTruthUnderTheCellDir(t *testing.T) { if err != nil { t.Fatal(err) } - for _, field := range []string{"effort", "approval", "archived", "places", "launchDir"} { + for _, field := range []string{"model", "modelProvider", "modelIdentity", "effort", "approval", "archived", "places", "launchDir"} { if strings.Contains(string(raw), `"`+field+`"`) { t.Errorf("meta.json still holds %s:\n%s", field, raw) } @@ -45,7 +45,7 @@ func TestCellLayoutKeepsTruthUnderTheCellDir(t *testing.T) { // Seal a session, materialize ONLY the sealed tree (workspace plus .cell/) into // a fresh folder, and open it: the dials and cursors are the same. // C27.7 exercises this behaviour without an opt-in switch. -func TestSealedTreeCarriesTheSessionTruth(t *testing.T) { +func TestSealedTreeCarriesTheSessionTruthC1bModelChoice(t *testing.T) { // The suite pins furrow away; this proof uses the explicit test engine. t.Setenv(furrow.BinaryEnvVar, os.Getenv("CODEAF_TEST_FURROW")) bin, err := furrow.ResolveBinary() @@ -63,6 +63,8 @@ func TestSealedTreeCarriesTheSessionTruth(t *testing.T) { if err != nil { t.Fatal(err) } + seed.Model, seed.ModelProvider = "fixture/current", "openrouter" + seed.ModelIdentity = &ModelIdentity{Selected: seed.Model, Models: &ConversationModels{Main: seed.Model, Fallback: "fixture/backup"}} RunLandedTasks(t, dir, "map the parser") // tasks.json holds journal paths the law must see if err := SaveMeta(dir, seed); err != nil { t.Fatal(err) @@ -90,6 +92,9 @@ func TestSealedTreeCarriesTheSessionTruth(t *testing.T) { if err != nil || meta.Effort != "high" || meta.Approval != "allow" || !meta.Archived || len(meta.ArchivedTasks) != 1 { t.Errorf("truth meta = %+v, %v", meta, err) } + if meta.Model != seed.Model || meta.ModelProvider != seed.ModelProvider || meta.ModelIdentity == nil || !reflect.DeepEqual(meta.ModelIdentity.Models, seed.ModelIdentity.Models) { + t.Fatalf("C1b sealed model choice = %+v, identity=%+v", meta, meta.ModelIdentity) + } assertResolvedHere(t, meta, freshDir, there) assertNoAbsolutePaths(t, filepath.Join(freshDir, cellStateDir)) place := Place{Dir: freshDir} diff --git a/internal/syncsetup/confirm_work.go b/internal/syncsetup/confirm_work.go index a760af4ceb..9428b8e8b4 100644 --- a/internal/syncsetup/confirm_work.go +++ b/internal/syncsetup/confirm_work.go @@ -13,23 +13,60 @@ import ( // and decides how to degrade a failed read. True confirms the current fence; // false means either a viewer or a drive that cannot confirm ownership yet. func (d *Drive) ConfirmWork(ctx context.Context) (bool, error) { - if _, viewer := d.Viewer(); viewer || d.batcher == nil { + if d.batcher == nil { return false, nil } id, fence := d.batcher.Lease() + // Preserving orphaned work in a branch does not revive this displaced + // window. A deliberate reopen owns that branch through a new drive. + if id != d.directoryID { + return false, nil + } v, err := d.sync.Dir.Cell(ctx, id) if err != nil { return false, err } + v, err = d.renewIdleLease(ctx, v, fence) + if err != nil { + return false, err + } lease := v.Cell.Lease // A first publish or branch can change the lease during the read. That // answer confirms neither a takeover nor the batcher's new lease. if currentID, currentFence := d.batcher.Lease(); currentID != id || currentFence != fence { return false, nil } + if lease.Expires <= v.Now { + d.available() + // Never resume an older tree over work saved by another computer. + head, err := cellstore.Head(d.cell) + if err != nil || head == nil || head.Turn.ID != v.Cell.Head { + return false, err + } + resumed, err := d.batcher.Reacquire(ctx, v) + if err != nil || !resumed { + return false, err + } + d.mu.Lock() + d.viewer = Viewer{} + d.mu.Unlock() + if err := d.recordSendLease(); err != nil { + d.available() + return false, err + } + if d.opt.OnDriving != nil { + d.opt.OnDriving() + } + return true, nil + } if lease.Device == d.sync.Device.ID() && lease.Fence == fence { + d.resumeViewer() return true, nil } + if lease.Device == d.sync.Device.ID() { + d.available() + return false, nil + } // Naming the holder shares the admission budget. Once the fence is known // lost, a slow name lookup cannot turn the refusal back into admission. @@ -70,3 +107,30 @@ func (d *Drive) ConfirmWork(ctx context.Context) (bool, error) { }() return false, nil } + +func (d *Drive) resumeViewer() { + d.mu.Lock() + if d.stopped { + d.mu.Unlock() + return + } + changed := d.viewer.Line != "" + d.viewer = Viewer{} + d.mu.Unlock() + if changed && d.opt.OnDriving != nil { + d.opt.OnDriving() + } +} + +// available retains the work gate until admission acquires a new fence, without +// claiming that the last holder is still working in a released chat. +func (d *Drive) available() { + viewer := Viewer{Line: chatlist.Available} + d.mu.Lock() + changed := d.viewer != viewer + d.viewer = viewer + d.mu.Unlock() + if changed && d.opt.OnViewer != nil { + d.opt.OnViewer(viewer) + } +} diff --git a/internal/syncsetup/continue_ownership_test.go b/internal/syncsetup/continue_ownership_test.go index 8b8f2f5e57..96e18ddc7e 100644 --- a/internal/syncsetup/continue_ownership_test.go +++ b/internal/syncsetup/continue_ownership_test.go @@ -42,6 +42,9 @@ func TestContinueChecksOwnershipBeforeRetiring(t *testing.T) { view, err := h.a.Dir.Cell(ctx, h.cell.ID) mustNil(t, err) seen := &directory.LeaseExpectation{Device: h.b.Device.ID(), Fence: view.Cell.Lease.Fence - 1} + if own { + seen = &directory.LeaseExpectation{Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence, Head: view.Cell.Head} + } if !own { seen = &directory.LeaseExpectation{Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence} _, err = h.b.Dir.Acquire(ctx, h.cell.ID, directory.AcquireOpts{Force: true, Expect: seen, RequireConditional: true}) diff --git a/internal/syncsetup/copy.go b/internal/syncsetup/copy.go index 3502dbf0ba..deac3acd5d 100644 --- a/internal/syncsetup/copy.go +++ b/internal/syncsetup/copy.go @@ -38,7 +38,7 @@ func (s *Sync) PublishCopy(ctx context.Context, e cellstore.Engine, c cell.Cell, return err } info := s.publishInfo(c, title) - view, err := s.Dir.Create(ctx, c.ID, directory.CellInit{Head: head.Turn.ID, Class: info.Class, Title: info.Title, Size: info.Size, Keys: info.Keys}) + view, err := s.Dir.Create(ctx, c.ID, directory.CellInit{Head: head.Turn.ID, SavedAt: head.Turn.SealedAtMs, Class: info.Class, Title: info.Title, Size: info.Size, Keys: info.Keys}) if errors.Is(err, directory.ErrExists) { return nil } diff --git a/internal/syncsetup/devicerecord_test.go b/internal/syncsetup/devicerecord_test.go index fa808acfc6..3e998602ac 100644 --- a/internal/syncsetup/devicerecord_test.go +++ b/internal/syncsetup/devicerecord_test.go @@ -10,6 +10,31 @@ import ( "github.com/Agent-Field/codeaf/internal/identity" ) +type countedDeviceDirectory struct { + directory.Client + puts int +} + +func (d *countedDeviceDirectory) PutDevice(ctx context.Context, id string, rec directory.Device) error { + d.puts++ + return d.Client.PutDevice(ctx, id, rec) +} + +func TestA1PutOwnDeviceDoesNotRewriteUnchangedRecord(t *testing.T) { + home := t.TempDir() + id, err := identity.Ensure(home) + mustNil(t, err) + dev, err := identity.Device(home) + mustNil(t, err) + dir := &countedDeviceDirectory{Client: directory.NewMemory(time.Now).For(dev.ID())} + for _, name := range []string{"desk", "desk", "renamed", "renamed"} { + mustNil(t, PutOwnDevice(t.Context(), dir, id, dev, name)) + } + if dir.puts != 2 { + t.Fatalf("A1: %d PUTs, want one per real name change", dir.puts) + } +} + // A device's record names the system it runs on, so the device list can say so. func TestDeviceRecordCarriesThePlatform(t *testing.T) { id, err := identity.Mint() diff --git a/internal/syncsetup/devices.go b/internal/syncsetup/devices.go index 33fca595ab..3e6a72dac1 100644 --- a/internal/syncsetup/devices.go +++ b/internal/syncsetup/devices.go @@ -22,11 +22,13 @@ const DeviceRequestWithin = 20 * time.Second // DeviceSnapshot contains only facts already read by this process. A missing // Online entry means unknown, including after a presence socket goes away. type DeviceSnapshot struct { - Devices map[string]directory.Device - Online map[string]bool - Version uint64 - Refused error - mutation uint64 + Devices map[string]directory.Device + Online map[string]bool + Complete bool + Version uint64 + Refused error + mutation uint64 + presenceAt map[string]int64 } // A removal's sequence orders it against reads from every client in this @@ -83,6 +85,7 @@ func (o *DeviceObserver) Snapshot(id identity.Identity, dev identity.Dev) Device } s := o.state s.Devices, s.Online = maps.Clone(s.Devices), maps.Clone(s.Online) + s.presenceAt = maps.Clone(s.presenceAt) return s } @@ -104,9 +107,13 @@ func rememberDevices(home string, binding deviceBinding, edit func(*DeviceSnapsh } next := o.state next.Devices, next.Online = maps.Clone(next.Devices), maps.Clone(next.Online) + next.presenceAt = maps.Clone(next.presenceAt) if next.Devices == nil { next.Devices = map[string]directory.Device{} } + if next.presenceAt == nil { + next.presenceAt = map[string]int64{} + } edit(&next) if reflect.DeepEqual(next, o.state) { continue @@ -123,8 +130,11 @@ func rememberDevices(home string, binding deviceBinding, edit func(*DeviceSnapsh // Embedding the HTTP client preserves its holding and conditional-take doors. type deviceDirectory struct { *directory.HTTP - home string - binding deviceBinding + home string + binding deviceBinding + watchMu sync.Mutex + watchEpoch uint64 + presenceExpiry *time.Timer } func (d *deviceDirectory) List(ctx context.Context) (directory.Listing, error) { @@ -140,6 +150,10 @@ func (d *deviceDirectory) List(ctx context.Context) (directory.Listing, error) { if devices == nil { devices = map[string]directory.Device{} } + for id, rec := range devices { + rec.LastSeen = max(rec.LastSeen, s.Devices[id].LastSeen) + devices[id] = rec + } // Equal or unversioned reads cannot retract a removal. A read // begun before a completed removal cannot replace any facts. if l.Version == 0 || l.Version == s.Version { @@ -154,7 +168,7 @@ func (d *deviceDirectory) List(ctx context.Context) (directory.Listing, error) { } } } - s.Devices, s.Version = devices, max(s.Version, l.Version) + s.Devices, s.Version, s.Complete = devices, max(s.Version, l.Version), true }) } return l, err @@ -193,17 +207,23 @@ func (d *deviceDirectory) Presence(ctx context.Context) (directory.PresenceView, p, err := d.HTTP.Presence(ctx) d.refusal(err) rememberDevices(d.home, d.binding, func(s *DeviceSnapshot) { - s.Online = nil if err != nil { + s.Online = nil return } - s.Online = map[string]bool{} + if s.Online == nil { + s.Online = map[string]bool{} + } for id, seen := range p.Devices { + if p.Now < s.presenceAt[id] { + continue + } + s.presenceAt[id] = p.Now s.Online[id] = seen.Online // An online answer stamps now, not a past sighting. Keeping // that clock out prevents a refresh/event/refresh loop. if rec, ok := s.Devices[id]; ok && !seen.Online { - rec.LastSeen = seen.LastSeen + rec.LastSeen = max(rec.LastSeen, seen.LastSeen) s.Devices[id] = rec } } @@ -237,7 +257,44 @@ func (d *deviceDirectory) Watch(ctx context.Context) (dirwatch.Stream, error) { d.refusal(err) return nil, err } - return &deviceStream{Stream: s, dir: d, presence: true}, nil + d.watchMu.Lock() + reconnected := d.watchEpoch != 0 + d.watchEpoch++ + epoch := d.watchEpoch + if d.presenceExpiry != nil { + d.presenceExpiry.Stop() + d.presenceExpiry = nil + } + d.watchMu.Unlock() + if reconnected { + // Reconnect snapshots announce only online peers. Reconcile once so a + // computer that lapsed while this socket was down does not stay online. + // Finish before the new socket's first ping is due. + read, cancel := context.WithTimeout(ctx, dirwatch.KeepEvery) + _, err := d.Presence(read) + cancel() + if err != nil { + rememberDevices(d.home, d.binding, func(v *DeviceSnapshot) { v.Online = nil }) + } + } + return &deviceStream{Stream: s, dir: d, presence: true, epoch: epoch}, nil +} + +// expirePresence bounds retained online facts while a watch cannot reconnect. +// Losing observation makes presence unknown; only the relay announces offline. +// The caller holds watchMu, also used by the timer and the next connection. +func (d *deviceDirectory) expirePresence(epoch uint64) { + if d.presenceExpiry != nil { + d.presenceExpiry.Stop() + } + d.presenceExpiry = time.AfterFunc(dirwatch.DeadAfter, func() { + d.watchMu.Lock() + defer d.watchMu.Unlock() + if d.watchEpoch == epoch { + rememberDevices(d.home, d.binding, func(v *DeviceSnapshot) { v.Online = nil }) + d.presenceExpiry = nil + } + }) } func (d *deviceDirectory) WatchHolding(ctx context.Context, holds []directory.Hold) (dirwatch.Stream, error) { @@ -253,6 +310,7 @@ type deviceStream struct { dirwatch.Stream dir *deviceDirectory presence bool + epoch uint64 } func (s *deviceStream) Next(ctx context.Context) (dirwatch.Frame, error) { @@ -260,11 +318,22 @@ func (s *deviceStream) Next(ctx context.Context) (dirwatch.Frame, error) { if f.Pong && err == nil { return f, err } + if s.presence { + s.dir.watchMu.Lock() + defer s.dir.watchMu.Unlock() + if s.epoch != s.dir.watchEpoch { + return f, err + } + if err != nil && ctx.Err() == nil { + s.dir.expirePresence(s.epoch) + } + } rememberDevices(s.dir.home, s.dir.binding, func(v *DeviceSnapshot) { switch { case err != nil: - delete(v.Online, s.dir.binding.self) - if s.presence { + // A dropped socket says nothing about another computer's presence. + // A deliberate owner shutdown still retires its local observation. + if s.presence && ctx.Err() != nil { v.Online = nil } if errors.Is(err, dirwatch.ErrRevoked) || errors.Is(err, dirwatch.ErrRotated) { @@ -287,23 +356,26 @@ func (s *deviceStream) Next(ctx context.Context) (dirwatch.Frame, error) { v.Devices[e.Device] = rec delete(v.Online, e.Device) case dirwatch.EventPresence: - if e.Online != nil { + if e.Online != nil && e.At >= v.presenceAt[e.Device] { + v.presenceAt[e.Device] = e.At if v.Online == nil { v.Online = map[string]bool{} } v.Online[e.Device] = *e.Online if !*e.Online && e.At > 0 { - rec.LastSeen = e.At + rec.LastSeen = max(rec.LastSeen, e.At) v.Devices[e.Device] = rec } } } default: v.Version = max(v.Version, f.Version) - if v.Online == nil { - v.Online = map[string]bool{} + if s.presence { + if v.Online == nil { + v.Online = map[string]bool{} + } + v.Online[s.dir.binding.self] = true } - v.Online[s.dir.binding.self] = true } }) return f, err diff --git a/internal/syncsetup/drive.go b/internal/syncsetup/drive.go index 8304328bc4..e2479d7382 100644 --- a/internal/syncsetup/drive.go +++ b/internal/syncsetup/drive.go @@ -233,16 +233,34 @@ func (s *Sync) putDevice(ctx context.Context, name string) error { return PutOwnDevice(ctx, s.Dir, s.Identity, s.Device, name) } +var ownDeviceWrites sync.Map + // PutOwnDevice upserts a device's own directory record at dir: the same sealed // name and capability row every chat start makes ([Sync.putDevice]), exposed so // the doors that finish a pairing can make it too. A device may only ever write // its own record, the name travels sealed under the identity's key, and the // relay never learns it. func PutOwnDevice(ctx context.Context, dir directory.Client, id identity.Identity, dev identity.Dev, name string) error { + // Chat starts, pairing and the app publisher can arrive together. Compare + // and write under one device lock so they cannot publish duplicate rows. + value, _ := ownDeviceWrites.LoadOrStore(dev.ID(), &sync.Mutex{}) + mu := value.(*sync.Mutex) + mu.Lock() + defer mu.Unlock() rec, err := deviceRecord(id, name) if err != nil { return err } + listing, err := dir.List(ctx) + if err != nil { + return err + } + if old, ok := listing.Devices[dev.ID()]; ok { + opened, err := directory.OpenName(directory.MetadataKey(id.CellKey()), old.Name) + if err == nil && opened == name && old.V == rec.V && old.Platform == rec.Platform && old.Caps == rec.Caps && old.AddedBy == rec.AddedBy { + return nil + } + } return dir.PutDevice(ctx, dev.ID(), rec) } @@ -286,7 +304,10 @@ func (s *Sync) driving(ctx context.Context, c cell.Cell) (cellsync.Driving, stri } var localHead string wasFree := view.Cell.Lease.Expires <= view.Now - if wasFree { + // An outage can leave our own saved head ahead of the relay. The + // unchanged, unreleased hold can renew without discarding that work. + resumeOwn := view.Cell.Lease.Device == s.Device.ID() && view.Cell.Lease.Expires != 0 + if wasFree && !resumeOwn { head, err := cellstore.Head(c) if err != nil { return drv, "", err @@ -298,19 +319,27 @@ func (s *Sync) driving(ctx context.Context, c cell.Cell) (cellsync.Driving, stri return drv, nonemptyHolder(view.Cell.Lease.Device), nil } } - v, err := s.Dir.Acquire(ctx, drv.ID(), directory.AcquireOpts{RequireConditional: wasFree, Expect: &directory.LeaseExpectation{ - Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence, - }}) + var v directory.CellView + if resumeOwn { + // Opening an installed copy, including just after a take or a process + // restart, resumes the same hold. Heartbeat checks its fence atomically + // and can renew an expired lease that nobody else acquired. + v, err = s.Dir.Heartbeat(ctx, drv.ID(), directory.Beat{Fence: view.Cell.Lease.Fence, Pending: view.Cell.Lease.Pending}) + } else { + v, err = s.Dir.Acquire(ctx, drv.ID(), directory.AcquireOpts{RequireConditional: wasFree, Expect: &directory.LeaseExpectation{ + Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence, + }}) + } switch { case errors.Is(err, directory.ErrNotFound): return drv, "", nil - case errors.Is(err, directory.ErrLeaseHeld): + case errors.Is(err, directory.ErrLeaseHeld), errors.Is(err, directory.ErrFenceStale): return drv, s.holder(ctx, drv.ID()), nil case err != nil: return drv, "", err } known, err = cellstore.KnowsHead(c, v.Cell.Head) - if err != nil || !known || (wasFree && localHead != v.Cell.Head) { + if err != nil || !known || (wasFree && !resumeOwn && localHead != v.Cell.Head) { _ = s.Dir.Release(ctx, drv.ID(), v.Cell.Lease.Fence) return drv, nonemptyHolder(view.Cell.Lease.Device), err } @@ -371,9 +400,34 @@ func (d *Drive) info() cellsync.PublishInfo { // becomeViewer is the Batcher's word that another device took the chat over: // from now on this window only shows it, and says who has it. func (d *Drive) becomeViewer(s cellsync.Superseded) { + if s.Branch == "" && d.directoryID != "" { + ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) + v, err := d.sync.Dir.Cell(ctx, d.directoryID) + cancel() + if err == nil && v.Cell.Lease.Expires <= v.Now { + d.available() + return + } + if err == nil && s.By == "" { + s.By = v.Cell.Lease.Device + } + } + if s.By != "" && s.By == d.sync.Device.ID() { + d.available() + return + } d.mu.Lock() + previous := d.viewer + if s.By == "" { + s.By = previous.ID + } changed := d.viewer.Line == "" d.viewer = Viewer{ID: s.By, Line: chatlist.Superseded(unknownDevice)} + // Repeated ownership checks must not replace a known holder with the + // temporary unnamed refusal while its display name is read again. + if previous.ID == s.By && previous.Name != "" { + d.viewer = previous + } expected := sendLease{Cell: d.directoryID, Fence: d.recordedFence} d.mu.Unlock() if err := d.sync.forgetSendLease(d.cell.ID, expected); err != nil { @@ -466,16 +520,27 @@ func (d *Drive) RenewalStatus() (time.Time, time.Duration, bool) { // directory that cannot be asked answers true, because a drive side that cannot // tell must not be thrown away for it. func (d *Drive) HoldsLease(ctx context.Context) bool { - if _, viewer := d.Viewer(); viewer || d.batcher == nil { + if line, viewer := d.Viewer(); viewer && line != chatlist.Available || d.batcher == nil { return false } v, err := d.sync.Dir.Cell(ctx, d.directoryID) + expired := err == nil && v.Cell.Lease.Expires <= v.Now + if err == nil { + v, err = d.renewIdleLease(ctx, v, d.batcher.Fence()) + } if err != nil { + if errors.Is(err, directory.ErrFenceStale) || errors.Is(err, cellstore.ErrReadOnly) || expired && !cellsync.Unreachable(err) { + return false + } return true } lease := v.Cell.Lease // The first publish may establish the fence while the read is in flight. - return lease.Expires > v.Now && lease.Device == d.sync.Device.ID() && lease.Fence == d.batcher.Fence() + held := lease.Expires > v.Now && lease.Device == d.sync.Device.ID() && lease.Fence == d.batcher.Fence() + if held { + d.resumeViewer() + } + return held } // CheckLease closes the stale-window gap before a tool runs, even when no @@ -484,16 +549,60 @@ func (d *Drive) CheckLease(ctx context.Context) error { if err := cellstore.Captured(ctx, d.cell); err != nil { return err } - if _, viewer := d.Viewer(); viewer || d.batcher == nil { + if d.batcher == nil { return nil } + id, fence := d.batcher.Lease() v, err := d.sync.Dir.Cell(ctx, d.directoryID) + if currentID, currentFence := d.batcher.Lease(); currentID != id || currentFence != fence { + return nil + } + if err == nil { + v, err = d.renewIdleLease(ctx, v, fence) + } + if err == nil && v.Cell.Lease.Expires <= v.Now { + d.available() + return nil + } if err == nil && (v.Cell.Lease.Expires <= v.Now || v.Cell.Lease.Device != d.sync.Device.ID() || v.Cell.Lease.Fence != d.batcher.Fence()) { d.becomeViewer(cellsync.Superseded{By: v.Cell.Lease.Device}) + } else if err == nil { + d.resumeViewer() } return nil } +// renewIdleLease covers the gap between a watch reconnect and its ordinary +// heartbeat. Expiry alone never retires an unchanged owner that can still +// renew; an explicit release or a different fence still needs continuation. +func (d *Drive) renewIdleLease(ctx context.Context, v directory.CellView, fence uint64) (directory.CellView, error) { + lease := v.Cell.Lease + if lease.Expires != 0 && lease.Expires <= v.Now && lease.Device == d.sync.Device.ID() && lease.Fence == fence { + // Close stops writers under the same lock before its release. A late + // ownership read must never resurrect that released fence. + d.sealMu.Lock() + d.mu.Lock() + stopped := d.stopped + d.mu.Unlock() + if stopped { + d.sealMu.Unlock() + return v, cellstore.ErrReadOnly + } + renewed, err := d.sync.Dir.Heartbeat(ctx, d.directoryID, directory.Beat{Fence: fence, Pending: d.batcher.Pending()}) + d.sealMu.Unlock() + if errors.Is(err, directory.ErrFenceStale) { + d.becomeViewer(cellsync.Superseded{}) + } else if err != nil && !cellsync.Unreachable(err) { + d.available() + } + if err != nil { + return v, err + } + return renewed, err + } + return v, nil +} + func nonemptyHolder(id string) string { if id == "" { return unknownDevice diff --git a/internal/syncsetup/drive_refusal_safety_test.go b/internal/syncsetup/drive_refusal_safety_test.go index ab18352543..bd825ce251 100644 --- a/internal/syncsetup/drive_refusal_safety_test.go +++ b/internal/syncsetup/drive_refusal_safety_test.go @@ -9,7 +9,7 @@ import ( ) func TestAStaleDriveRefusalImmediatelyMakesTheWindowAViewer(t *testing.T) { - d := &Drive{sync: &Sync{}} + d := &Drive{sync: &Sync{Home: t.TempDir()}} d.refusal(directory.ErrFenceStale) if d.Gate() == nil { t.Fatal("a refused drive still allows new work") @@ -32,7 +32,7 @@ func TestTheSendProofNamesTheDrivesActualFence(t *testing.T) { } } -func TestAnExpiredHoldCannotAuthorizeTools(t *testing.T) { +func TestFenceF1ReleasedHoldCannotAuthorizeTools(t *testing.T) { r := newDriveRig(t) sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) mustNil(t, err) @@ -42,9 +42,10 @@ func TestAnExpiredHoldCannotAuthorizeTools(t *testing.T) { mustNil(t, err) defer drive.Close(t.Context()) r.clock.Advance(directory.LeaseTTL + time.Second) + mustNil(t, r.a.Dir.Release(t.Context(), r.cell.ID, drive.batcher.Fence())) mustNil(t, drive.CheckLease(t.Context())) if drive.Gate() == nil { - t.Fatal("expired ownership still authorizes tools") + t.Fatal("released ownership still authorizes tools") } } diff --git a/internal/syncsetup/drive_watch.go b/internal/syncsetup/drive_watch.go index 235706e573..bdc37c07de 100644 --- a/internal/syncsetup/drive_watch.go +++ b/internal/syncsetup/drive_watch.go @@ -19,7 +19,7 @@ func (d *Drive) followOwnership(ctx context.Context) { _ = d.CheckLease(read) cancel() } - if _, viewer := d.Viewer(); viewer || state.Refused != nil { + if state.Refused != nil { return } select { diff --git a/internal/syncsetup/fence_stability_test.go b/internal/syncsetup/fence_stability_test.go new file mode 100644 index 0000000000..e9d7e6639e --- /dev/null +++ b/internal/syncsetup/fence_stability_test.go @@ -0,0 +1,221 @@ +package syncsetup + +import ( + "bufio" + "context" + "net" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/directory" +) + +func TestFenceF1StartupRenewsExistingHold(t *testing.T) { + r := newDriveRig(t) + sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + created, err := r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, sealed.Turn.ID)) + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer drive.Close(t.Context()) + after, err := r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if after.Cell.Lease.Fence != created.Cell.Lease.Fence { + t.Fatalf("F1 opening own live hold changed fence: %d -> %d", created.Cell.Lease.Fence, after.Cell.Lease.Fence) + } +} + +func TestFenceF1IdleExpiredHoldRenewsWithoutAcquiring(t *testing.T) { + r := newDriveRig(t) + sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + created, err := r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, sealed.Turn.ID)) + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer drive.Close(t.Context()) + r.clock.Advance(directory.LeaseTTL + time.Second) + mustNil(t, drive.CheckLease(t.Context())) + if drive.Gate() != nil || !drive.HoldsLease(t.Context()) { + t.Fatal("F1 idle owner was retired instead of renewing its unchanged fence") + } + view, err := r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if view.Cell.Lease.Fence != created.Cell.Lease.Fence || view.Cell.Lease.Expires <= view.Now { + t.Fatalf("F1 idle renewal changed fence or stayed expired: %+v", view) + } +} + +// Restart retains a live lease; a clean quit intentionally releases it for +// immediate continuation elsewhere and therefore needs a new lease on reopen. +func TestFenceF1IdleWatchReconnectAndProcessResume(t *testing.T) { + connections := make(chan net.Conn, 16) + var acquires, releases atomic.Int32 + r := newDriveRig(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if strings.HasSuffix(req.URL.Path, "/acquire") { + acquires.Add(1) + } + if strings.HasSuffix(req.URL.Path, "/release") { + releases.Add(1) + } + if req.URL.Path == "/v1/dir/watch" && len(req.URL.Query()["hold"]) == 0 { + next.ServeHTTP(fenceReconnectWriter{w, connections}, req) + return + } + next.ServeHTTP(w, req) + }) + }) + sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + created, err := r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, sealed.Turn.ID)) + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer drive.Close(t.Context()) + var conn net.Conn + select { + case conn = <-connections: + case <-time.After(10 * time.Second): + t.Fatal("F1 initial watch missing") + } + mustNil(t, conn.Close()) + select { + case <-connections: + case <-time.After(10 * time.Second): + t.Fatal("F1 watch did not reconnect") + } + mustNil(t, drive.CheckLease(t.Context())) + view, err := r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if view.Cell.Lease.Fence != created.Cell.Lease.Fence || acquires.Load() != 0 || releases.Load() != 0 { + t.Fatalf("F1 reconnect changed hold: %+v; acquires=%d releases=%d", view.Cell.Lease, acquires.Load(), releases.Load()) + } + // End the old process's loops without its clean-quit release. + drive.cancel() + <-drive.done + resumed, err := openSync(t, r.homeA).Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer resumed.Close(t.Context()) + view, err = r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if view.Cell.Lease.Fence != created.Cell.Lease.Fence || acquires.Load() != 0 || releases.Load() != 0 { + t.Fatalf("F1 idle restart changed hold: %+v; acquires=%d releases=%d", view.Cell.Lease, acquires.Load(), releases.Load()) + } +} + +type fenceReconnectWriter struct { + http.ResponseWriter + connections chan net.Conn +} + +func (w fenceReconnectWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + c, rw, err := w.ResponseWriter.(http.Hijacker).Hijack() + if err == nil { + w.connections <- c + } + return c, rw, err +} + +func TestRelayOutageO4FailedIdleRenewalKeepsWorkLocal(t *testing.T) { + r := newDriveRig(t) + wire := &fenceRenewalControl{Client: r.a.Dir} + r.a.Dir = wire + sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + _, err = r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, sealed.Turn.ID)) + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer drive.Close(t.Context()) + r.clock.Advance(directory.LeaseTTL + time.Second) + wire.refuse.Store(true) + mustNil(t, drive.CheckLease(t.Context())) + if drive.Gate() != nil || !drive.HoldsLease(t.Context()) { + t.Fatal("O4 unreachable renewal retired an unchanged own hold") + } + wire.refuse.Store(false) + if !drive.HoldsLease(t.Context()) || drive.Gate() != nil { + t.Fatal("F1 recovered idle owner could not renew its original fence") + } +} + +func TestFenceF1LateIdleReadCannotResurrectClosedHold(t *testing.T) { + r := newDriveRig(t) + wire := &fenceRenewalControl{Client: r.a.Dir, read: make(chan struct{}), resume: make(chan struct{})} + defer func() { + select { + case <-wire.resume: + default: + close(wire.resume) + } + }() + r.a.Dir = wire + sealed, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + _, err = r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, sealed.Turn.ID)) + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{}) + mustNil(t, err) + defer drive.Close(t.Context()) + r.clock.Advance(directory.LeaseTTL + time.Second) + wire.delay.Store(true) + done := make(chan error, 1) + go func() { done <- drive.CheckLease(t.Context()) }() + select { + case <-wire.read: + case <-time.After(10 * time.Second): + t.Fatal("F1 ownership read missing") + } + mustNil(t, drive.Close(t.Context())) + beats := wire.beats.Load() + close(wire.resume) + select { + case err := <-done: + mustNil(t, err) + case <-time.After(10 * time.Second): + t.Fatal("F1 late ownership read stuck") + } + view, err := r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if view.Cell.Lease.Expires != 0 || wire.beats.Load() != beats { + t.Fatalf("F1 late read renewed a closed hold: %+v", view.Cell.Lease) + } +} + +type fenceRenewalControl struct { + directory.Client + refuse, delay atomic.Bool + beats atomic.Int32 + read, resume chan struct{} +} + +func (c *fenceRenewalControl) Cell(ctx context.Context, id string) (directory.CellView, error) { + v, err := c.Client.Cell(ctx, id) + if c.delay.CompareAndSwap(true, false) { + close(c.read) + select { + case <-c.resume: + case <-ctx.Done(): + return v, ctx.Err() + } + } + return v, err +} +func (c *fenceRenewalControl) Heartbeat(ctx context.Context, id string, b directory.Beat) (directory.CellView, error) { + c.beats.Add(1) + if c.refuse.Load() { + return directory.CellView{}, directory.ErrUnreachable + } + return c.Client.Heartbeat(ctx, id, b) +} diff --git a/internal/syncsetup/member_client_test.go b/internal/syncsetup/member_client_test.go index 214a81db85..08a000b26b 100644 --- a/internal/syncsetup/member_client_test.go +++ b/internal/syncsetup/member_client_test.go @@ -438,6 +438,10 @@ func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { a := r.home(nil) id := a.Identity b := r.home(&id) + // M1 requires usable provider credentials before a model preference can arrive. + for _, s := range []*Sync{a, b} { + mustNil(t, config.WriteAPIKey(s.Home, fakeKeyOne)) + } readyMember(t, a) readyMember(t, b) mustNil(t, a.SyncState(t.Context())) diff --git a/internal/syncsetup/outage_test.go b/internal/syncsetup/outage_test.go new file mode 100644 index 0000000000..79864a845f --- /dev/null +++ b/internal/syncsetup/outage_test.go @@ -0,0 +1,37 @@ +package syncsetup + +import ( + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/directory" +) + +func TestRelayOutageO1O2OwnExpiredLeaseWithUnpublishedHead(t *testing.T) { + r := newDriveRig(t) + r.spokenFixture() + first, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + created, err := r.a.Dir.Create(t.Context(), r.cell.ID, directory.CellInit{Head: first.Turn.ID}) + mustNil(t, err) + mustNil(t, cellstore.NotePublished(r.cell, first.Turn.ID)) + // The relay missed a locally saved turn and all renewal traffic while asleep. + appendTo(t, transcriptOf(r.cell), "{\"type\":\"message\",\"role\":\"user\",\"content\":\"during outage\"}\n") + latest, err := r.engine.Seal(t.Context(), r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + r.clock.Advance(directory.LeaseTTL + time.Second) + var notices noticeLog + drive, err := r.a.Drive(t.Context(), r.engine, r.cell, DriveOptions{DeviceName: "this computer", OnNotice: notices.add}) + mustNil(t, err) + defer drive.Close(t.Context()) + if err := drive.Gate(); err != nil { + t.Fatalf("O1/O2 own expired lease refused local work: %v; notices %v", err, notices.all()) + } + waitFor(t, "O2 unpublished descendant to reach the relay", func() bool { return r.directoryHead(r.cell.ID) == latest.Turn.ID }) + v, err := r.a.Dir.Cell(t.Context(), r.cell.ID) + mustNil(t, err) + if v.Cell.Lease.Device != r.a.Device.ID() || v.Cell.Lease.Fence != created.Cell.Lease.Fence || v.Cell.Lease.Expires <= v.Now { + t.Fatalf("O2 recovery did not renew the unchanged own hold: %+v", v) + } +} diff --git a/internal/syncsetup/preferences.go b/internal/syncsetup/preferences.go index 924a9b3731..ec6d4b7eba 100644 --- a/internal/syncsetup/preferences.go +++ b/internal/syncsetup/preferences.go @@ -1,10 +1,15 @@ package syncsetup import ( - "github.com/Agent-Field/codeaf/internal/config" - "github.com/Agent-Field/codeaf/internal/vaultsync" + "encoding/json" "path/filepath" + "strings" "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/roles" + "github.com/Agent-Field/codeaf/internal/vaultsync" ) type preferenceKeys struct{ dir string } @@ -26,3 +31,45 @@ func preferences(dir, home string) vaultsync.Keyed { func (p preferenceKeys) ApplyIf(changes, expected map[string]string) error { return config.ApplyPreferencesIf(p.dir, changes, expected) } + +// Accept uses the same resolved connections and readiness rule as model calls. +// Local and sign-in provider prefixes cannot fall through to the default +// router just because this computer has never configured that provider. +func (p preferenceKeys) Accept(key, value, providerID string) bool { + if !strings.HasPrefix(key, "model.") || value == "" { + return true + } + var model string + if json.Unmarshal([]byte(value), &model) != nil { + return false + } + if model == "" { + return true + } + sources := config.ResolveSources(p.dir, config.APIKeyAt(p.dir), config.DefaultBaseURL) + return config.ModelUsable(sources, model, providerID) +} + +// Metadata records the provider actually selected on the sending computer. +// Custom aliases otherwise look like ordinary OpenRouter author prefixes on a +// receiver that has no such connection. No endpoint or credential travels here. +func (p preferenceKeys) Metadata(key, value string) string { + if !strings.HasPrefix(key, "model.") { + return "" + } + var model string + if json.Unmarshal([]byte(value), &model) != nil || model == "" { + return "" + } + model, _ = roles.SplitEffort(model) + source, _ := config.ResolveSources(p.dir, config.APIKeyAt(p.dir), config.DefaultBaseURL).For(model) + return source.Source.ID +} + +// MayAnnotate migrates only an explicitly configured provider at the original +// edit clock. A receiver's restore gets a different local clock; in particular, +// falling through to OpenRouter is never evidence about an old custom alias. +func (p preferenceKeys) MayAnnotate(key, value string, stamp int64) bool { + providerID := p.Metadata(key, value) + return providerID != "" && providerID != modelsource.DefaultID && p.KeyTime(key).UnixMilli() == stamp +} diff --git a/internal/syncsetup/preferences_model_test.go b/internal/syncsetup/preferences_model_test.go new file mode 100644 index 0000000000..fad5c45c5e --- /dev/null +++ b/internal/syncsetup/preferences_model_test.go @@ -0,0 +1,237 @@ +package syncsetup + +import ( + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/keys" + "github.com/Agent-Field/codeaf/internal/vaultsync" +) + +func TestModelSyncM1UnavailableProvider(t *testing.T) { + t.Setenv(config.APIKeyEnv, "") + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, filepath.Dir(config.BudgetConfigPath(m.profile)))} + } + key := config.ModelSettingKey("talk") + at := time.Now().Add(-time.Hour) + b.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, key: "deepseek/deepseek-chat"}, at) + a.saveConfig(t, map[string]any{key: "ollama/llama3.2:latest"}, at.Add(time.Minute)) + before := config.PreferenceTime(b.profile, key) + a.push(t) + b.pull(t) + if got := b.config(t)[key]; got != "deepseek/deepseek-chat" { + t.Fatalf("M1 unavailable provider replaced local model: %v", got) + } + if got := config.PreferenceTime(b.profile, key); !got.Equal(before) { + t.Fatalf("M2 skip advanced local clock: %v -> %v", before, got) + } +} + +func TestModelSyncM1MissingCustomProvider(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + m.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.KeyChatModel: "openai/gpt-5"}, time.Now().Add(-time.Hour)) + } + mustNil(t, config.WriteSources(a.profile, []config.PersistedSource{{ID: "custom-local", Written: "local-models", Address: "http://127.0.0.1:11434/v1", KeyOptional: true}})) + mustNil(t, config.WriteChatModel(a.profile, "local-models/qwen3")) + a.push(t) + b.pull(t) + if got := config.ChatModelAt(b.profile); got != "openai/gpt-5" { + t.Fatalf("M1 missing custom provider fell through to OpenRouter: %q", got) + } +} + +func TestModelSyncM1EverySlotUnavailable(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + av, bv := map[string]any{}, map[string]any{config.KeyAPIKey: fakeKeyOne} + for _, slot := range config.ModelSlots() { + key := config.ModelSettingKey(slot.Slot) + av[key], bv[key] = "ollama/qwen3", "openai/gpt-5" + } + a.saveConfig(t, av, time.Now().Add(-time.Minute)) + b.saveConfig(t, bv, time.Now().Add(-time.Hour)) + mustNil(t, config.WriteSources(a.profile, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: "http://127.0.0.1:11434/v1"}})) + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + } + a.push(t) + b.pull(t) + for _, slot := range config.ModelSlots() { + key := config.ModelSettingKey(slot.Slot) + if b.config(t)[key] != "openai/gpt-5" { + t.Fatalf("M1 unavailable model applied to %s", key) + } + } +} + +func TestModelSyncM2DifferentProvidersNeverFlap(t *testing.T) { + for _, name := range config.ProviderKeyEnvNamesAt(t.TempDir()) { + t.Setenv(name, "") + } + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + } + key := config.ModelSettingKey("talk") + a.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, key: "deepseek/deepseek-chat"}, time.Now().Add(-time.Hour)) + b.saveConfig(t, map[string]any{key: "ollama/llama3.2:latest"}, time.Now().Add(-time.Minute)) + mustNil(t, config.WriteSources(b.profile, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: "http://127.0.0.1:11434/v1"}})) + mustNil(t, config.WriteDefaultEffort(a.profile, "high")) + mustNil(t, config.WriteDailyBudgetUSD(b.profile, 7)) + for range 5 { + a.push(t) + b.push(t) + a.pull(t) + b.pull(t) + if a.config(t)[key] != "deepseek/deepseek-chat" || b.config(t)[key] != "ollama/llama3.2:latest" { + t.Fatalf("M2 model flapped: A=%v B=%v", a.config(t)[key], b.config(t)[key]) + } + } + budget, err := config.DailyBudgetUSDAt(a.profile) + mustNil(t, err) + if config.DefaultEffortAt(b.profile) != "high" || budget != 7 { + t.Fatal("M2 non-model preferences stopped travelling") + } +} + +func TestModelSyncM3UsableProviderTravels(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + m.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne}, time.Now()) + } + mustNil(t, config.WriteChatModel(a.profile, "deepseek/deepseek-chat")) + a.push(t) + b.pull(t) + if config.ChatModelAt(b.profile) != "deepseek/deepseek-chat" { + t.Fatal("M3 usable model did not arrive") + } +} + +func TestModelSyncM3CustomAliasWithDifferentLocalIDs(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for i, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + id := []string{"custom", "custom-lab"}[i] + mustNil(t, config.WriteSources(m.profile, []config.PersistedSource{{ID: id, Written: "lab", Address: "http://127.0.0.1:11434/v1", KeyOptional: true}})) + } + mustNil(t, config.WriteChatModel(a.profile, "lab/qwen3")) + a.push(t) + b.pull(t) + if config.ChatModelAt(b.profile) != "lab/qwen3" { + t.Fatal("M3 usable custom alias was refused because of a machine-local ID") + } +} + +func TestModelSyncM1LegacyChoiceMetadataKeepsItsClock(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + mustNil(t, config.WriteSources(a.profile, []config.PersistedSource{{ID: "custom", Written: "lab", Address: "http://127.0.0.1:11434/v1", KeyOptional: true}})) + mustNil(t, config.WriteChatModel(a.profile, "lab/qwen3")) + b.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.KeyChatModel: "openai/gpt-5"}, time.Now().Add(-time.Hour)) + ledger := filepath.Join(a.profile, "preferences-sync.ledger") + a.Carry = []vaultsync.Carrier{vaultsync.Keys("preferences", legacyModelPreferences{a.profile}, ledger)} + a.push(t) + before, err := a.Vault.(*keys.Vault).Revisions("home:keys:preferences") + mustNil(t, err) + a.Carry = []vaultsync.Carrier{preferences(a.profile, a.profile)} + b.Carry = []vaultsync.Carrier{preferences(b.profile, b.profile)} + a.push(t) + b.pull(t) + after, err := a.Vault.(*keys.Vault).Revisions("home:keys:preferences") + mustNil(t, err) + if after[config.KeyChatModel].Stamp != before[config.KeyChatModel].Stamp { + t.Fatal("M1 metadata upgrade advanced the model choice clock") + } + if config.ChatModelAt(b.profile) != "openai/gpt-5" { + t.Fatal("M1 a legacy custom selection reached a receiver without its provider") + } +} + +func TestModelSyncM1LegacyReceiverCannotDescribeSendersCustomProvider(t *testing.T) { + r := newKeyRig(t) + a, b, c := r.machine(), r.machine(), r.machine() + b.saveConfig(t, map[string]any{}, time.Now().Add(-time.Hour)) + mustNil(t, config.WriteSources(a.profile, []config.PersistedSource{{ID: "custom", Written: "lab", Address: "http://127.0.0.1:11434/v1", KeyOptional: true}})) + mustNil(t, config.WriteChatModel(a.profile, "lab/qwen3")) + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{vaultsync.Keys("preferences", legacyModelPreferences{m.profile}, filepath.Join(m.profile, "preferences-sync.ledger"))} + } + a.push(t) + b.pull(t) + b.Carry = []vaultsync.Carrier{preferences(b.profile, b.profile)} + b.push(t) + revisions, err := b.Vault.(*keys.Vault).Revisions("home:keys:preferences") + mustNil(t, err) + if revisions[config.KeyChatModel].Metadata != "" { + t.Fatal("M1 receiver claimed provider identity for an old received model") + } + a.Carry = []vaultsync.Carrier{preferences(a.profile, a.profile)} + a.push(t) + c.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.KeyChatModel: "openai/gpt-5"}, time.Now().Add(-time.Hour)) + c.Carry = []vaultsync.Carrier{preferences(c.profile, c.profile)} + c.pull(t) + if config.ChatModelAt(c.profile) != "openai/gpt-5" { + t.Fatal("M1 receiver-first upgrade poisoned the provider metadata") + } +} + +type legacyModelPreferences struct{ dir string } + +func (p legacyModelPreferences) Values() (map[string]string, error) { + return config.ReadPreferences(p.dir) +} +func (p legacyModelPreferences) Holds(key string) bool { return config.PreferenceKey(key) } +func (p legacyModelPreferences) Apply(changes map[string]string) error { + return config.ApplyPreferences(p.dir, changes) +} +func (p legacyModelPreferences) KeyTime(key string) time.Time { + return config.PreferenceTime(p.dir, key) +} + +func TestModelSyncM4ProviderSetupDoesNotReplaySkippedChoice(t *testing.T) { + r := newKeyRig(t) + a, b := r.machine(), r.machine() + for _, m := range []*keyMachine{a, b} { + m.Carry = []vaultsync.Carrier{preferences(m.profile, m.profile)} + m.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.ModelSettingKey("talk"): "deepseek/deepseek-chat"}, time.Now().Add(-time.Hour)) + } + mustNil(t, config.WriteSources(a.profile, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: "http://127.0.0.1:11434/v1"}})) + mustNil(t, config.WriteChatModel(a.profile, "ollama/llama3.2:latest")) + a.push(t) + b.pull(t) + mustNil(t, config.WriteSources(b.profile, []config.PersistedSource{{ID: "ollama", Written: "ollama", Address: "http://127.0.0.1:11434/v1"}})) + // Recreate the carrier to prove the rejection survives a process restart. + b.Carry = []vaultsync.Carrier{preferences(b.profile, b.profile)} + for range 3 { + b.push(t) + a.push(t) + b.pull(t) + } + if config.ChatModelAt(b.profile) != "deepseek/deepseek-chat" { + t.Fatal("M4 provider setup replayed an old choice") + } + mustNil(t, config.WriteChatModel(a.profile, "ollama/qwen3:latest")) + a.push(t) + b.pull(t) + if config.ChatModelAt(b.profile) != "ollama/qwen3:latest" { + t.Fatalf("M4 a newer usable choice did not arrive: A=%q B=%q", config.ChatModelAt(a.profile), config.ChatModelAt(b.profile)) + } + mustNil(t, config.WriteChatModel(a.profile, "ollama/llama3.2:latest")) + a.push(t) + b.pull(t) + if config.ChatModelAt(b.profile) != "ollama/llama3.2:latest" { + t.Fatal("M4 a new revision of the originally skipped value did not arrive") + } +} diff --git a/internal/syncsetup/receiver_workspace_test.go b/internal/syncsetup/receiver_workspace_test.go new file mode 100644 index 0000000000..1ac6e35521 --- /dev/null +++ b/internal/syncsetup/receiver_workspace_test.go @@ -0,0 +1,41 @@ +package syncsetup + +import ( + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/session" +) + +// W5 keeps unavailable sender paths out of the receiver's engine target. +func TestReceiverWorkspacePathsW5(t *testing.T) { + for _, recorded := range []string{ + filepath.Join(t.TempDir(), "missing-project"), + `C:\Users\sender\missing-codeaf-e6b-project`, + "/Users/sender/missing-codeaf-e6b-project", + } { + t.Run(recorded, func(t *testing.T) { + at, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if err != nil { + t.Fatal(err) + } + if err := session.SaveMeta(at.Root, session.Meta{ID: at.ID, Workspace: recorded}); err != nil { + t.Fatal(err) + } + c := (&Sync{}).Continuer(cellstore.Engine{DataRoot: t.TempDir()}, TakeOptions{ + RootFor: func(string) string { return at.Root }, + }) + target := c.eng.Sync(cellstore.SyncKeys{}, "").Target(at) + root, err := filepath.EvalSymlinks(at.Root) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(root, "work") + if target.Tree != want { + t.Fatalf("W5 receiver tree %q, want %q for sender path %q", target.Tree, want, recorded) + } + }) + } +} diff --git a/internal/syncsetup/sendlease.go b/internal/syncsetup/sendlease.go index ab418af0d0..60f7f67f74 100644 --- a/internal/syncsetup/sendlease.go +++ b/internal/syncsetup/sendlease.go @@ -7,6 +7,7 @@ import ( "errors" "os" "path/filepath" + "strings" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/filelock" @@ -18,15 +19,22 @@ type sendLease struct { Fence uint64 `json:"fence"` } -func (s *Sync) sendLeasePath(id string) string { +func (s *Sync) sendLeasePath(id string) (string, error) { + if strings.TrimSpace(s.Home) == "" { + return "", errors.New("sync: a home directory is required for chat holds") + } key := sha256.Sum256([]byte(s.Ledger + "/" + s.Device.ID() + "/" + id)) - return filepath.Join(s.Home, "v3", "sync", "holds", hex.EncodeToString(key[:])) + return filepath.Join(s.Home, "v3", "sync", "holds", hex.EncodeToString(key[:])), nil } // SendLease names the hold the local session actually publishes under. It // stays outside the saved chat so restoring another computer's copy cannot // grant this window permission to send. func (s *Sync) SendLease(id string) (string, uint64, error) { + path, err := s.sendLeasePath(id) + if err != nil { + return "", 0, err + } current, err := s.currentDisplaySync() if err != nil { return "", 0, err @@ -34,7 +42,7 @@ func (s *Sync) SendLease(id string) (string, uint64, error) { if current != s { return current.SendLease(id) } - raw, err := os.ReadFile(s.sendLeasePath(id)) + raw, err := os.ReadFile(path) if err != nil { return "", 0, err } @@ -84,7 +92,10 @@ func (s *Sync) recordSendLease(id string, held sendLease) error { if err != nil { return err } - path := s.sendLeasePath(id) + path, err := s.sendLeasePath(id) + if err != nil { + return err + } if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return err } @@ -114,7 +125,10 @@ func (s *Sync) forgetSendLease(id string, expected sendLease) error { return err } defer lock.Close() - path := s.sendLeasePath(id) + path, err := s.sendLeasePath(id) + if err != nil { + return err + } raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return nil @@ -137,7 +151,10 @@ func (s *Sync) forgetSendLease(id string, expected sendLease) error { } func (s *Sync) lockSendLease(id string) (*os.File, error) { - path := s.sendLeasePath(id) + path, err := s.sendLeasePath(id) + if err != nil { + return nil, err + } if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return nil, err } diff --git a/internal/syncsetup/sendlease_home_test.go b/internal/syncsetup/sendlease_home_test.go new file mode 100644 index 0000000000..5cfe58203c --- /dev/null +++ b/internal/syncsetup/sendlease_home_test.go @@ -0,0 +1,26 @@ +package syncsetup + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSendLeaseI1EmptyHomeCannotWriteRelativeHold(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + s := &Sync{} + for name, action := range map[string]func() error{ + "read": func() error { _, _, err := s.SendLease("chat"); return err }, + "record": func() error { return s.recordSendLease("chat", sendLease{Cell: "chat", Fence: 1}) }, + "forget": func() error { return s.forgetSendLease("chat", sendLease{Cell: "chat", Fence: 1}) }, + } { + if err := action(); err == nil || !strings.Contains(err.Error(), "home") { + t.Errorf("%s with empty home = %v, want a home error", name, err) + } + } + if _, err := os.Stat(filepath.Join(dir, "v3")); !os.IsNotExist(err) { + t.Fatalf("empty home created a relative sync directory: %v", err) + } +} diff --git a/internal/syncsetup/state_test.go b/internal/syncsetup/state_test.go index 5b91f6ce20..9e3364b4f2 100644 --- a/internal/syncsetup/state_test.go +++ b/internal/syncsetup/state_test.go @@ -21,12 +21,12 @@ func TestC30PreferencesMergePerSetting(t *testing.T) { a.Carry = []vaultsync.Carrier{preferences(a.profile, filepath.Dir(config.BudgetConfigPath(a.profile)))} b.Carry = []vaultsync.Carrier{preferences(b.profile, filepath.Dir(config.BudgetConfigPath(b.profile)))} at := time.Unix(1700000000, 0) - a.saveConfig(t, map[string]any{config.ModelSettingKey("talk"): "stub/one", config.KeyIcons: "plain", "machine.path": "/a"}, at) - b.saveConfig(t, map[string]any{"machine.path": "/b"}, at) + a.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.ModelSettingKey("talk"): "stub/one", config.KeyIcons: "plain", "machine.path": "/a"}, at) + b.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, "machine.path": "/b"}, at) a.push(t) b.pull(t) - a.saveConfig(t, map[string]any{config.ModelSettingKey("talk"): "stub/older", config.KeyIcons: "plain", config.KeyDailyBudget: 7.0, "machine.path": "/a"}, at.Add(time.Second)) - b.saveConfig(t, map[string]any{config.ModelSettingKey("talk"): "stub/later", config.KeyIcons: "ascii", "machine.path": "/b"}, at.Add(2*time.Second)) + a.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.ModelSettingKey("talk"): "stub/older", config.KeyIcons: "plain", config.KeyDailyBudget: 7.0, "machine.path": "/a"}, at.Add(time.Second)) + b.saveConfig(t, map[string]any{config.KeyAPIKey: fakeKeyOne, config.ModelSettingKey("talk"): "stub/later", config.KeyIcons: "ascii", "machine.path": "/b"}, at.Add(2*time.Second)) b.push(t) a.push(t) b.pull(t) diff --git a/internal/syncsetup/take.go b/internal/syncsetup/take.go index 88706bb97f..fb0578048e 100644 --- a/internal/syncsetup/take.go +++ b/internal/syncsetup/take.go @@ -167,6 +167,9 @@ func (c *Continuer) take(ctx context.Context, id string, seen *directory.LeaseEx return Continued{}, errors.New("these turns are no longer available to open separately") } lease := view.Cell.Lease + if seen != nil && !seen.Matches(view.Cell) { + return Continued{}, directory.ErrLeaseHeld + } // Kept turns may name this device before their separate copy exists here. known, err := cellstore.KnowsHead(cell.Cell{ID: id, Root: root}, view.Cell.Head) if err != nil { @@ -187,9 +190,6 @@ func (c *Continuer) take(ctx context.Context, id string, seen *directory.LeaseEx Cell: opened, Driving: cellsync.Driving{Cell: opened, Fence: lease.Fence, Head: view.Cell.Head, Frames: view.Cell.Frames}, }}, nil } - if seen != nil && (lease.Device != seen.Device || lease.Fence != seen.Fence) { - return Continued{}, directory.ErrLeaseHeld - } if seen == nil { seen = &directory.LeaseExpectation{Device: lease.Device, Fence: lease.Fence} } diff --git a/internal/vaultsync/keyed.go b/internal/vaultsync/keyed.go index d21cbba53f..e518ba15e9 100644 --- a/internal/vaultsync/keyed.go +++ b/internal/vaultsync/keyed.go @@ -66,9 +66,12 @@ func (k Keyed) id(name string) string { return k.scope + "/" + name } // ledger as it is being updated. type keyedRun struct { Keyed - v VaultFile - held map[string]string - digests map[string]string + v VaultFile + held map[string]string + digests map[string]string + skipped map[string]int64 + metadata map[string]string + revisions map[string]keys.Revision } func (k Keyed) begin(v VaultFile) (*keyedRun, error) { @@ -76,8 +79,8 @@ func (k Keyed) begin(v VaultFile) (*keyedRun, error) { if err != nil { return nil, err } - digests, err := readLedger(k.ledger) - return &keyedRun{Keyed: k, v: v, held: held, digests: digests}, err + ledger, err := readLedger(k.ledger) + return &keyedRun{Keyed: k, v: v, held: held, digests: ledger.Digests, skipped: ledger.Skipped}, err } func (k Keyed) capture(v VaultFile) error { @@ -112,12 +115,31 @@ func (r *keyedRun) finish(err error) error { if err != nil { return err } - return writeLedger(r.ledger, r.digests) + return writeLedger(r.ledger, keyedLedger{Digests: r.digests, Skipped: r.skipped}) } // captureAll stores each key whose value changed since it was last seen, and // tombstones each key the ledger knew that is no longer held. func (r *keyedRun) captureAll() error { + if described, ok := r.set.(interface{ Metadata(string, string) string }); ok { + r.metadata = map[string]string{} + for name, value := range r.held { + if metadata := described.Metadata(name, value); metadata != "" { + r.metadata[name] = metadata + } + } + if len(r.metadata) > 0 { + if revised, ok := r.v.(interface { + Revisions(string) (map[string]keys.Revision, error) + }); ok { + var err error + r.revisions, err = revised.Revisions(r.scope) + if err != nil { + return err + } + } + } + } for name, value := range r.held { if err := r.captureKey(name, value); err != nil { return err @@ -142,16 +164,28 @@ func (r *keyedRun) captureAll() error { func (r *keyedRun) captureKey(name, value string) error { if r.digests[name] == digest(value) { - return nil + return r.annotateKey(name, value) } - if e, err := r.v.Get(r.id(name)); err != nil || e.Value != value { + if e, err := r.v.Get(r.id(name)); err == nil && e.Value == value { + r.digests[name] = digest(value) + return r.annotateKey(name, value) + } else { at := time.Now() if timed, ok := r.set.(interface{ KeyTime(string) time.Time }); ok { if known := timed.KeyTime(name); !known.IsZero() { at = known } } - if err := r.v.PutAt(r.id(name), keys.Entry{Name: name, Value: value, Scope: r.scope}, at); err != nil { + entry := keys.Entry{Name: name, Value: value, Scope: r.scope} + put := func() error { return r.v.PutAt(r.id(name), entry, at) } + if metadata := r.metadata[name]; metadata != "" { + if annotated, ok := r.v.(interface { + PutMetadataAt(string, keys.Entry, time.Time, string) error + }); ok { + put = func() error { return annotated.PutMetadataAt(r.id(name), entry, at, metadata) } + } + } + if err := put(); err != nil { return err } } @@ -159,6 +193,25 @@ func (r *keyedRun) captureKey(name, value string) error { return nil } +func (r *keyedRun) annotateKey(name, value string) error { + metadata := r.metadata[name] + if metadata == "" || r.revisions[name].Metadata != "" { + return nil + } + owner, ok := r.set.(interface { + MayAnnotate(string, string, int64) bool + }) + if !ok || !owner.MayAnnotate(name, value, r.revisions[name].Stamp) { + return nil + } + if annotated, ok := r.v.(interface { + Annotate(string, keys.Entry, string) error + }); ok { + return annotated.Annotate(r.id(name), keys.Entry{Name: name, Value: value, Scope: r.scope}, metadata) + } + return nil +} + // restoreAll writes each vault key this machine holds differently, removes each // key the vault deleted that this machine had received or sent, and records what // it now shares with the vault. @@ -171,6 +224,35 @@ func (r *keyedRun) restoreAll() error { if err != nil { return err } + if accept, ok := r.set.(interface { + Accept(string, string, string) bool + }); ok && len(changes) > 0 { + stamped, ok := r.v.(interface { + Revisions(string) (map[string]keys.Revision, error) + }) + if !ok { + return errors.New("vaultsync: selective restore needs slot revisions") + } + revisions, err := stamped.Revisions(r.scope) + if err != nil { + return err + } + for name, value := range changes { + if value == "" { + continue + } + revision := revisions[name] + if revision.Stamp <= r.skipped[name] || !accept.Accept(name, value, revision.Metadata) { + r.skipped[name] = max(r.skipped[name], revision.Stamp) + // Keep the local digest, so rejecting a remote value never + // masquerades as a new edit on the next capture. + if held, ok := r.held[name]; ok { + r.digests[name] = digest(held) + } + delete(changes, name) + } + } + } if len(changes) > 0 { apply := func() error { return r.set.Apply(changes) } if guarded, ok := r.set.(interface { @@ -238,26 +320,41 @@ func digest(value string) string { return hex.EncodeToString(sum[:]) } -func readLedger(path string) (map[string]string, error) { +type keyedLedger struct { + Digests map[string]string `json:"digests"` + Skipped map[string]int64 `json:"skipped,omitempty"` +} + +func readLedger(path string) (keyedLedger, error) { raw, err := os.ReadFile(path) - digests := map[string]string{} + ledger := keyedLedger{Digests: map[string]string{}, Skipped: map[string]int64{}} if errors.Is(err, os.ErrNotExist) { - return digests, nil + return ledger, nil } if err != nil { - return nil, err + return ledger, err } - if json.Unmarshal(raw, &digests) != nil { + var old map[string]string + if json.Unmarshal(raw, &old) == nil { + if old != nil { + ledger.Digests = old + } + return ledger, nil + } + if json.Unmarshal(raw, &ledger) != nil || ledger.Digests == nil { // A damaged ledger only forgets which keys were shared, so a removal is // not passed on; it never loses a key. - return map[string]string{}, nil + return keyedLedger{Digests: map[string]string{}, Skipped: map[string]int64{}}, nil + } + if ledger.Skipped == nil { + ledger.Skipped = map[string]int64{} } - return digests, nil + return ledger, nil } // writeLedger replaces the ledger file, at mode 0600 like the vault it sits by. -func writeLedger(path string, digests map[string]string) error { - raw, err := json.Marshal(digests) +func writeLedger(path string, ledger keyedLedger) error { + raw, err := json.Marshal(ledger) if err != nil { return err } diff --git a/internal/vaultsync/keyed_reads_test.go b/internal/vaultsync/keyed_reads_test.go index fd877ef456..bf2df22d54 100644 --- a/internal/vaultsync/keyed_reads_test.go +++ b/internal/vaultsync/keyed_reads_test.go @@ -43,6 +43,11 @@ func (c *countedVault) Removed(prefix string) ([]string, error) { return c.VaultFile.Removed(prefix) } +func (c *countedVault) Revisions(scope string) (map[string]keys.Revision, error) { + c.calls++ + return c.VaultFile.(*keys.Vault).Revisions(scope) +} + // heldKeys is a Keyset that holds its keys in memory. type heldKeys map[string]string @@ -59,6 +64,42 @@ func (h heldKeys) Apply(changes map[string]string) error { return nil } +type rejectedKeys struct{ heldKeys } + +func (rejectedKeys) Accept(string, string, string) bool { return false } + +func TestSelectiveRestoreM1BoundedVaultReads(t *testing.T) { + for _, n := range []int{16, 256} { + t.Run(fmt.Sprint(n), func(t *testing.T) { + home := t.TempDir() + _, err := identity.Ensure(home) + must(t, err) + v, err := keys.Open(home) + must(t, err) + set := rejectedKeys{heldKeys{}} + for i := range n { + set.heldKeys[fmt.Sprint(i)] = "before" + } + carrier := Keys("selective", set, filepath.Join(home, "selective.ledger")) + must(t, carrier.capture(v)) + for name := range set.heldKeys { + must(t, v.PutAt(carrier.id(name), keys.Entry{Name: name, Value: "incoming", Scope: carrier.scope}, time.Now().Add(time.Minute))) + } + counted := &countedVault{VaultFile: v} + must(t, carrier.restore(counted)) + if counted.calls > 3 { + t.Fatalf("M1 %d rejected keys opened the vault %d times; cap 3", n, counted.calls) + } + for name, value := range set.heldKeys { + if value != "before" { + t.Fatalf("M1 rejected value applied to %s", name) + } + } + t.Logf("%d rejected keys: %d vault reads", n, counted.calls) + }) + } +} + // TestARestoreOpensTheVaultAFixedNumberOfTimes holds the idle cost of a state // round flat in the number of keys a machine shares. A restore runs after every // pull, which an idle terminal makes every few seconds; a profile with a diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 50e02ca336..7498d3f9bb 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -52,7 +52,10 @@ export class IdentityDO extends DurableObject { } fetch(request) { - return respond(() => this.#serve(request)); + return respond(() => this.#serve(request)).then(reply => { + if (new URL(request.url).pathname.startsWith('/v1/dir/')) reply.headers.set('Codeaf-Saved-Turn-Times', '1'); + return reply; + }); } /** Peer frames drive replication; the platform answers watch keepalives without waking this object. */ diff --git a/relay/hosted/src/liveness.js b/relay/hosted/src/liveness.js index 84ff056ac5..8d769916e1 100644 --- a/relay/hosted/src/liveness.js +++ b/relay/hosted/src/liveness.js @@ -1,8 +1,11 @@ +import { PRESENCE, windowMs } from './limits.js'; +import { presenceOf } from './presence.js'; + // Who is online, and the event frames that say so (docs/ux-pairing-contract.md, section 5; contract 21.12). A device // is online iff it holds at least one live watch socket: one that has not been flagged lapsed and whose sign of life -// is inside its window. The only thing kept besides the platform's own socket list is the set of devices whose last -// socket closed less than OFFLINE_DEBOUNCE_MS ago: they were told to be online and have not yet been told otherwise. -// That set is in storage, because the alarm that ends it wakes a fresh object. +// is inside its window. Devices whose last socket closed less than OFFLINE_DEBOUNCE_MS ago are kept in storage, +// because the alarm that ends their debounce wakes a fresh object. Local observation evidence distinguishes a real +// lapse from a computer this new presence memory has not heard from yet. /** How long a device may be without a socket before its peers are told it is offline (contract 5). */ export const OFFLINE_DEBOUNCE_MS = 15_000; @@ -37,6 +40,8 @@ export class Liveness { this.arm = arm; this.clock = clock; this.seen = seen; + this.started = clock(); + this.observed = new Set(); } #livePeers() { @@ -52,6 +57,7 @@ export class Liveness { } #announce(device, online) { + this.observed.add(device); this.tell({ t: 'presence', device, online }, device); } @@ -143,9 +149,24 @@ export class Liveness { return new Set(this.#livePeers().map((p) => p.device)); } + /** present includes computers inside the reconnect debounce, matching watch events. */ + present() { + return new Set([...this.online(), ...this.pending.keys()]); + } + + /** A fresh relay cannot call an unobserved computer offline until a presence window passes. */ + presence(devices, now) { + const online = this.present(), unknown = new Set(); + if (this.clock() - this.started < windowMs(PRESENCE.defaultBeat)) { + const observed = new Set([...this.observed, ...this.peers().map((p) => p.device)]); + for (const id of Object.keys(devices)) if (!online.has(id) && !observed.has(id)) unknown.add(id); + } + return presenceOf(devices, online, now, unknown); + } + /** snapshot is the presence frames a new event socket of `device` hears on connect: the others that peers believe online. */ snapshot(device) { - const believed = new Set([...this.online(), ...this.pending.keys()]); + const believed = this.present(); believed.delete(device); const at = this.clock(); return [...believed].map((d) => eventFrame('presence', { device: d, online: true, at })); diff --git a/relay/hosted/src/presence.js b/relay/hosted/src/presence.js index c83cdf76c1..8564064098 100644 --- a/relay/hosted/src/presence.js +++ b/relay/hosted/src/presence.js @@ -18,10 +18,10 @@ export function parseBeat(values) { return beat; } -/** presenceOf answers {now, devices} for every device of the directory that is not revoked. */ -export function presenceOf(devices, online, now) { +/** presenceOf omits revoked computers and computers whose presence is not yet known. */ +export function presenceOf(devices, online, now, unknown = new Set()) { const entries = Object.entries(devices) - .filter(([, d]) => !d.revoked) + .filter(([id, d]) => !d.revoked && !unknown.has(id)) .map(([id, d]) => [id, online.has(id) ? { online: true, last_seen: now } : { online: false, last_seen: d.last_seen ?? 0 }]); return { now, devices: Object.fromEntries(entries) }; } diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index 6f8bf28542..eafa2c07fe 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -13,7 +13,7 @@ import { RaftEffectsFacade } from './effects.js'; import { Wire, empty, json, unwrap } from '../wire.js'; import { prefixOf, signedUri } from '../base.js'; import { fromBase64Url } from '../codec.js'; -import { presenceOf, parseBeat } from '../presence.js'; +import { parseBeat } from '../presence.js'; import { parseHolds } from '../vouch.js'; import { rotationView } from '../rotation.js'; import { CommittedWatchers, CLOSE_REVOKED, CLOSE_ROTATED } from '../watch.js'; @@ -667,7 +667,7 @@ export class HostedIdentity { if(request.method==='GET'){ if(path==='/v1/identity/rotation')return json(rotationView(identity.rotation,Number(snapshot.app.now),this.tenant.limits)); if(path==='/v1/dir/list'||path==='/v1/dir/presence'){ - const raw=await this.listingJSON(b,snapshot.app),reply=path.endsWith('/presence')?json(presenceOf(JSON.parse(raw).devices,this.watchers.online(),Number(snapshot.app.now))):this.jsonBytes(raw);reply.headers.set('Codeaf-Dir-Version',version);return reply; + const raw=await this.listingJSON(b,snapshot.app),reply=path.endsWith('/presence')?json(this.watchers.presence(JSON.parse(raw).devices,Number(snapshot.app.now))):this.jsonBytes(raw);reply.headers.set('Codeaf-Dir-Version',version);return reply; } const row=await this.members.get(b.node).hostView(b.index,'state','cell/'+segment(decodeURIComponent(route.args[0])));if(!row.raw)throw new Wire('not_found',404); const reply=this.jsonBytes('{"now":'+row.app.now+',"cell":'+this.projectJSON('cell',row.raw)+'}');reply.headers.set('codeaf-conditional-takes','1');return reply; diff --git a/relay/hosted/src/routes.js b/relay/hosted/src/routes.js index d79cdbbd0e..e50297385f 100644 --- a/relay/hosted/src/routes.js +++ b/relay/hosted/src/routes.js @@ -4,7 +4,7 @@ // and answer a Response or throw a refusal; the Durable Object does everything around them. import { MAX_FRAME } from './frame.js'; import { approve, deny } from './link/decide.js'; -import { parseBeat, presenceOf } from './presence.js'; +import { parseBeat } from './presence.js'; import { parseHolds } from './vouch.js'; import { Wire, badRequest, json, empty, notFound } from './wire.js'; @@ -52,7 +52,7 @@ function watching(c) { // The directory's version rides on a presence answer as on a list. const presence = (c) => { const { tenant } = c; - const answer = presenceOf(tenant.dir.list().devices, tenant.watchers.online(), tenant.clock()); + const answer = tenant.watchers.presence(tenant.dir.list().devices, tenant.clock()); return withVersion(json(answer), tenant.dir.version); }; @@ -103,12 +103,17 @@ const ROUTES = [ ['POST', /^\/v1\/store\/locate$/, STORE, async (c) => json({ at: await c.tenant.locate(asRids(object(c.body))) })], ['POST', /^\/v1\/store\/has$/, STORE, async (c) => json({ have: await c.tenant.has(asRids(object(c.body))) })], ['GET', /^\/v1\/store\/stats$/, STORE, (c) => json(c.tenant.stats.snapshot())], - ['GET', /^\/v1\/dir\/list$/, DIR, (c) => withVersion(json(c.tenant.dir.list()), c.tenant.dir.version)], + ['GET', /^\/v1\/dir\/list$/, DIR, (c) => { + const response = withVersion(json(c.tenant.dir.list()), c.tenant.dir.version); + response.headers.set('Codeaf-Head-Takes', '1'); + return response; + }], ['GET', /^\/v1\/dir\/watch$/, DIR_WRITE, (c) => watching(c)], // C7: the client reads this promise before it can displace a holder. ['GET', /^\/v1\/dir\/cells\/([^/]+)$/, DIR, (c, [id]) => { const response = json(c.tenant.dir.cell(id)); response.headers.set('codeaf-conditional-takes', '1'); + response.headers.set('Codeaf-Head-Takes', '1'); return response; }], ['PUT', /^\/v1\/dir\/devices\/([^/]+)$/, DIR_WRITE, (c, [id]) => { diff --git a/relay/hosted/src/rules.js b/relay/hosted/src/rules.js index 692379ae22..14972bbe26 100644 --- a/relay/hosted/src/rules.js +++ b/relay/hosted/src/rules.js @@ -29,7 +29,9 @@ export function makeRules({ ttlMs, renewOnPublish }) { // force is the amendment: a person who chose "continue here" takes a live lease at once. // The fence still goes up, so the old holder's next write is refused as before. const acquire = (c, device, now, force = false, expect = undefined) => { - if (expect != null && (c.lease.device !== expect.device || c.lease.fence !== expect.fence)) throw refuse('lease_held'); + if (expect != null && (c.lease.device !== expect.device || + (expect.head && c.head !== expect.head) || + (c.lease.fence !== expect.fence && !(expect.head && c.lease.fence > expect.fence)))) throw refuse('lease_held'); if (!force && c.lease.expires > now && c.lease.device !== device) throw refuse('lease_held'); return { ...c, lease: { device, fence: c.lease.fence + 1, expires: now + ttlMs, pending: 0 } }; }; @@ -46,9 +48,9 @@ export function makeRules({ ttlMs, renewOnPublish }) { const expires = renewOnPublish ? now + ttlMs : c.lease.expires; // The frame plan describes the head it was set with, so every publish replaces it // wholesale: a publish that says nothing is a true statement, a stale one is not kept. - const { frames, frame_hints, ...kept } = c; + const { frames, frame_hints, saved_at, ...kept } = c; const next = { ...kept, head: p.head, size: p.size, class: p.class, durable_at: now, lease: { ...c.lease, expires, pending: p.pending } }; - return omitEmpty(next, { title, frames: p.frames, frame_hints: boundedFrameHints(p.frame_hints) }); + return omitEmpty(next, { title, frames: p.frames, frame_hints: boundedFrameHints(p.frame_hints), saved_at: p.saved_at > 0 ? p.saved_at : (p.head === p.old_head ? saved_at : undefined) }); }; const releaseOf = (c, device, fence, now) => { @@ -59,7 +61,7 @@ export function makeRules({ ttlMs, renewOnPublish }) { const created = (init, device, now) => { const c = { V: 1, head: init.head, durable_at: now, class: init.class, size: init.size, keys: init.keys, lease: { device, fence: 1, expires: now + ttlMs, pending: 0 } }; - return omitEmpty(c, { parent_cell: init.parent_cell, title: init.title, orphan_turns: init.orphan_turns, frames: init.frames, frame_hints: boundedFrameHints(init.frame_hints) }); + return omitEmpty(c, { parent_cell: init.parent_cell, title: init.title, orphan_turns: init.orphan_turns, frames: init.frames, frame_hints: boundedFrameHints(init.frame_hints), saved_at: init.saved_at }); }; return { acquire, heartbeat, publishTo, releaseOf, created }; diff --git a/relay/hosted/src/watch.js b/relay/hosted/src/watch.js index 03d75bdca2..af53e078ab 100644 --- a/relay/hosted/src/watch.js +++ b/relay/hosted/src/watch.js @@ -176,6 +176,15 @@ export class Watchers { return this.liveness.online(); } + /** present agrees with presence events while a computer reconnects. */ + present() { + return this.liveness.present(); + } + + presence(devices, now) { + return this.liveness.presence(devices, now); + } + /** announce sends an event frame to every event socket except the sockets of `except`. */ announce(frame, except) { this.liveness.tell(frame, except); diff --git a/relay/hosted/test/conditional-take.test.js b/relay/hosted/test/conditional-take.test.js index 36e6382d70..6503c369d5 100644 --- a/relay/hosted/test/conditional-take.test.js +++ b/relay/hosted/test/conditional-take.test.js @@ -17,3 +17,19 @@ test('C7 the Worker states conditional takes on the cell read', async () => { assert.throws(() => dir.acquire('c7', 'dev_b', true, { device: 'someone_else', fence: cell.lease.fence }), (e) => e.code === 'lease_held'); assert.equal(dir.cell('c7').cell.lease.device, 'dev_a'); }); + +test('F2 a same-holder fence bump is harmless only with the same saved head', () => { + for (const change of ['fence', 'head', 'holder']) { + const dir = new Directory(memorySql(), 'id_f2', () => 1_000_000, AMENDED); + dir.create('f2', init(), 'dev_a'); + const original = dir.cell('f2').cell; + const seen = { device: original.lease.device, fence: original.lease.fence, head: original.head }; + dir.acquire('f2', change === 'holder' ? 'dev_c' : 'dev_a', true); + if (change === 'head') { + const current = dir.cell('f2').cell; + dir.publish('f2', 'dev_a', { fence: current.lease.fence, old_head: current.head, head: 'new-head' }); + } + if (change === 'fence') assert.equal(dir.acquire('f2', 'dev_b', true, seen).cell.lease.device, 'dev_b'); + else assert.throws(() => dir.acquire('f2', 'dev_b', true, seen), (e) => e.code === 'lease_held'); + } +}); diff --git a/relay/hosted/test/directory.test.js b/relay/hosted/test/directory.test.js index 625015e71b..8b8c1958d5 100644 --- a/relay/hosted/test/directory.test.js +++ b/relay/hosted/test/directory.test.js @@ -25,6 +25,23 @@ test('create makes a held cell, and a second create is refused', () => { assert.equal(code(() => dir.create('c1', init(), 'dev_b')), 'exists'); }); +test('R2 saved_at follows the sealed head independently of publication and release', () => { + const { dir, clock } = open(); + dir.create('c1', { ...init(), saved_at: 700_000 }, 'dev_a'); + assert.equal(dir.cell('c1').cell.saved_at, 700_000); + clock.now += 10_000; + const p = { fence: 1, old_head: 'h0', head: 'h1', size: 1, class: 'work', pending: 0, saved_at: 800_000 }; + dir.publish('c1', 'dev_a', p); + assert.equal(dir.cell('c1').cell.saved_at, 800_000); + assert.equal(dir.cell('c1').cell.durable_at, 1_010_000); + dir.release('c1', 'dev_a', 1); + assert.equal(dir.cell('c1').cell.saved_at, 800_000); + dir.publish('c1', 'dev_a', { ...p, old_head: 'h1', saved_at: undefined }); + assert.equal(dir.cell('c1').cell.saved_at, 800_000); + dir.publish('c1', 'dev_a', { ...p, old_head: 'h1', head: 'h2', saved_at: undefined }); + assert.equal(dir.cell('c1').cell.saved_at, undefined); +}); + test('a live lease refuses another device, an expired one does not, force takes a live one', () => { const { dir, clock } = open(); dir.create('c1', init(), 'dev_a'); diff --git a/relay/hosted/test/liveness.test.js b/relay/hosted/test/liveness.test.js index 5988e65e62..6a8d1240a7 100644 --- a/relay/hosted/test/liveness.test.js +++ b/relay/hosted/test/liveness.test.js @@ -1,7 +1,8 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { OFFLINE_DEBOUNCE_MS, Liveness } from '../src/liveness.js'; -import { windowMs } from '../src/limits.js'; +import { PRESENCE, windowMs } from '../src/limits.js'; +import { matchRoute } from '../src/routes.js'; /** A room is a set of fake sockets and the clock; each socket keeps the frames it was sent. */ function room() { @@ -44,6 +45,31 @@ function room() { const flat = (s) => s.got.map((f) => `${f.device}:${f.online}`); +test('P2 restarted presence memory omits unseen computers, observes returns, and reports real lapses', async () => { + const r = room(), devices = { dev_a: { last_seen: 1 }, dev_b: { last_seen: 2 } }; + const tenant = { + dir: { list: () => ({ devices }), version: 7 }, clock: () => r.now.t, + watchers: { present: () => r.liveness.present(), presence: (rows, now) => r.liveness.presence(rows, now) }, + }; + const read = async () => { + const route = matchRoute('GET', '/v1/dir/presence'); + const reply = route.handler({ tenant }, route.args); + assert.equal(reply.headers.get('Codeaf-Dir-Version'), '7'); + return (await reply.json()).devices; + }; + assert.deepEqual(await read(), {}, 'P2 a restarted relay has not seen either stored computer'); + const b = r.join('dev_b', true, 10); + assert.equal((await read()).dev_b.online, true); + r.leave(b); + assert.equal((await read()).dev_b.online, true, 'P2 reconnect debounce still applies'); + r.now.t += OFFLINE_DEBOUNCE_MS; + r.liveness.expire(); + assert.equal((await read()).dev_b.online, false, 'P2 an observed lapse is evidence even during startup'); + assert.equal((await read()).dev_a, undefined); + r.now.t += windowMs(PRESENCE.defaultBeat); + assert.equal((await read()).dev_a.online, false, 'P2 no return for a full window is offline'); +}); + test('the first socket of a device is news to the other event sockets, a second socket is not', () => { const r = room(); const a = r.join('dev_a'); @@ -282,3 +308,13 @@ test('a device that comes back after being told offline is told online at once', r.join('dev_b', false, 10); assert.deepEqual(flat(viewer), ['dev_b:true', 'dev_b:false', 'dev_b:true']); }); + +test('P1 presence queries retain a computer until its reconnect debounce lapses', () => { + const r = room(); + const a = r.join('dev_a'); + r.leave(a); + assert.deepEqual([...r.liveness.present()], ['dev_a']); + r.now.t += OFFLINE_DEBOUNCE_MS; + r.liveness.expire(); + assert.deepEqual([...r.liveness.present()], []); +});