From 2c7f108510163b0eda351f642313731d2bcabf0a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 2 Nov 2026 19:34:02 -0500 Subject: [PATCH 01/28] processgroup: macOS reads process groups, so left-running services and the job bound work there (#2039) * processgroup: read macOS process groups and job subtrees without cgo Members lists a group through kern.proc.pgrp with a bounded retry, and Group.Usage walks the recorded leader's descendants through the kernel's proc_info interface, converting task time with hw.tbfrequency. The new darwinproc package owns the raw trap, the SDK layouts and the argv and listening-socket readers the executor uses next. Co-Authored-By: Claude Opus 5.5 * executor: name and port left-running services on macOS cmdline reads kern.procargs2 and the ports come from each member's TCP listeners. describe reads each member's sockets once for both the naming member and the port set. The two service tests run on Linux and macOS, and the listening-service test polls the record until the backgrounded shell has exec'd its server. Co-Authored-By: Claude Opus 5.5 * docs: the job subtree bound, and that macOS now applies it Co-Authored-By: Claude Opus 5.5 * processgroup, darwinproc: bound the macOS walk and keep strangers out Review fixes: - A queued parent is re-read after its children are listed and its start time compared, so a pid handed to another process cannot bring that process's children into the job's reading. - A child listing that fills the largest buffer, and a walk that reaches 1024 processes, count as lower bounds; any unexpected listing failure makes the reading unknown instead of partial. - kern.procargs2 is parsed at the kernel's eight-byte boundary after the executable path, so an empty argv[0] can never shift an environment string into argv (checked for path lengths 1-40 on arm64 and under Rosetta). - The listening-service test's shell now waits for the server's port before it exits, so the record the call returns is asserted directly; the membersOf helpers it no longer needs are gone. Co-Authored-By: Claude Opus 5.5 * processgroup, session: a partial CPU sum yields no rate Review fix: a reading that left part of the tree out (the walk limit, a truncated child listing, a process that could not be read) still counts processes as a lower bound, but its CPU sum is marked partial. The job bound takes no rate from it and does not use it as the next baseline, so a process missing from one reading and present in the next cannot read as a burst of its whole lifetime's CPU and invent a strike. Linux readings are always complete, so nothing changes there. Co-Authored-By: Claude Opus 5.5 * processgroup: a parent that cannot be re-read makes the sum partial Review fix: when a queued parent read once but its re-read after the child listing fails for a reason other than having exited, its children are left out of this reading only, so the CPU sum is marked partial and no rate is taken from it. Co-Authored-By: Claude Opus 5.5 * docs: number the change entry for #2039 Co-Authored-By: Claude Opus 5.5 * processgroup: a child counts only while its parent is still the same Review fix: the macOS walk now reads a parent's listed children first and re-reads the parent after them. A pid is never handed on while its process lives, so a parent with the same start time after its children were read was their parent while they were read; a parent and child whose pids both changed hands just after the parent's own re-read can no longer bring a stranger's busy child into the job's reading. The partial-sum regression now starts from one complete, quiet reading, so it also fails if a partial reading ever keeps that earlier baseline. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../2039-darwin-process-inspection.md | 19 ++ internal/darwinproc/doc.go | 3 + internal/darwinproc/procinfo_darwin.go | 267 ++++++++++++++++++ internal/darwinproc/procinfo_darwin_test.go | 257 +++++++++++++++++ internal/executor/executor.go | 6 +- internal/executor/service.go | 48 ++-- internal/executor/service_darwin.go | 22 ++ internal/executor/service_describe_test.go | 64 +++++ internal/executor/service_linux.go | 18 +- internal/executor/service_linux_test.go | 4 - internal/executor/service_other.go | 6 +- internal/executor/service_other_test.go | 5 - internal/executor/service_unix_test.go | 19 +- internal/executor/watched_unix_test.go | 4 +- internal/manual/chat/what-i-can-do.md | 19 ++ internal/processgroup/members_darwin.go | 69 +++++ internal/processgroup/members_darwin_test.go | 71 +++++ internal/processgroup/members_other.go | 12 +- internal/processgroup/usage.go | 15 +- internal/processgroup/usage_darwin.go | 125 ++++++++ internal/processgroup/usage_darwin_test.go | 80 ++++++ .../processgroup/usage_darwin_walk_test.go | 230 +++++++++++++++ internal/processgroup/usage_other.go | 10 +- internal/session/jobbound.go | 18 +- internal/session/jobbound_test.go | 36 +++ 25 files changed, 1356 insertions(+), 71 deletions(-) create mode 100644 docs/changes/unreleased/2039-darwin-process-inspection.md create mode 100644 internal/darwinproc/doc.go create mode 100644 internal/darwinproc/procinfo_darwin.go create mode 100644 internal/darwinproc/procinfo_darwin_test.go create mode 100644 internal/executor/service_darwin.go create mode 100644 internal/executor/service_describe_test.go delete mode 100644 internal/executor/service_other_test.go create mode 100644 internal/processgroup/members_darwin.go create mode 100644 internal/processgroup/members_darwin_test.go create mode 100644 internal/processgroup/usage_darwin.go create mode 100644 internal/processgroup/usage_darwin_test.go create mode 100644 internal/processgroup/usage_darwin_walk_test.go diff --git a/docs/changes/unreleased/2039-darwin-process-inspection.md b/docs/changes/unreleased/2039-darwin-process-inspection.md new file mode 100644 index 0000000000..5a91eb265e --- /dev/null +++ b/docs/changes/unreleased/2039-darwin-process-inspection.md @@ -0,0 +1,19 @@ +--- +kind: fixed +title: macOS reads process groups for left-running services and the job subtree bound +pr: 2039 +surface: [engine, docs] +invalidates: + - "On macOS, a server left running by a shell command was never reported, and service records named the shell without its server's argv or listening ports. These records now inspect the surviving processes." + - "The job subtree CPU and process bound never cut on macOS because subtree usage could not be read. macOS now measures the whole tree and applies the existing bound." +--- + +On macOS, a registered bash job that sustains more than three fifths of the +machine's logical cores for thirty seconds, or grows past 128 processes for +that long, is now ended like on Linux. The CPU bound applies when no CPU quota +or affinity mask holds codeaf below the machine's cores. Both limits require +15 consecutive readings two seconds apart, and the run is told why it ended. + +Process inspection works without cgo, including release and sidecar builds. +Service descriptions read each surviving member's sockets once, select the +first listening member's argv and collect the sorted, unique listening ports. diff --git a/internal/darwinproc/doc.go b/internal/darwinproc/doc.go new file mode 100644 index 0000000000..88c2428641 --- /dev/null +++ b/internal/darwinproc/doc.go @@ -0,0 +1,3 @@ +// Package darwinproc reads macOS process identity, descendants, CPU usage, +// arguments and listening sockets without cgo. +package darwinproc diff --git a/internal/darwinproc/procinfo_darwin.go b/internal/darwinproc/procinfo_darwin.go new file mode 100644 index 0000000000..b190e614d3 --- /dev/null +++ b/internal/darwinproc/procinfo_darwin.go @@ -0,0 +1,267 @@ +//go:build darwin + +package darwinproc + +import ( + "bytes" + "encoding/binary" + "slices" + "sync" + "unsafe" + + "golang.org/x/sys/unix" +) + +// The raw proc_info trap uses Go's existing per-architecture syscall stubs, +// avoiding assembly, dynamic imports and linkname dependencies that garbling +// could break. Unlike libproc's wrappers, the trap preserves errno directly. +// Apple promises stability for neither the trap numbers nor libproc, but both +// have remained unchanged since macOS 10.5. The private XNU call selectors and +// SDK layouts below are identical on arm64 and amd64; both are little-endian. +const ( + callListPIDs = 1 + callPIDInfo = 2 + callPIDFDInfo = 3 + ppidOnly = 6 + pidListFDs = 1 + pidTaskAllInfo = 2 + pidFDSocketInfo = 3 + fdTypeSocket = 2 + socketInfoTCP = 2 + tcpStateListen = 1 + familyIPv4 = 2 + familyIPv6 = 30 + + // StatusZombie marks exited processes, which cannot contribute live work. + StatusZombie = 5 + + pidSize = 4 + taskAllInfoSize = 232 + bsdInfoOffset = 0 + bsdInfoSize = 136 + bsdStatusOffset = bsdInfoOffset + 4 + bsdPIDOffset = bsdInfoOffset + 12 + bsdPPIDOffset = bsdInfoOffset + 16 + bsdPGIDOffset = bsdInfoOffset + 100 + bsdStartSecondsOffset = bsdInfoOffset + 120 + bsdStartMicrosOffset = bsdInfoOffset + 128 + taskInfoOffset = bsdInfoSize + taskInfoSize = 96 + taskResidentOffset = taskInfoOffset + 8 + taskUserOffset = taskInfoOffset + 16 + taskSystemOffset = taskInfoOffset + 24 + fdInfoSize = 8 + fdNumberOffset = 0 + fdTypeOffset = 4 + socketFDInfoSize = 792 + socketInfoOffset = 24 + socketInfoSize = 768 + socketFamilyOffset = socketInfoOffset + 160 + socketKindOffset = socketInfoOffset + 232 + socketProtocolOffset = socketInfoOffset + 240 + socketLocalPortOffset = socketProtocolOffset + 4 + socketTCPStateOffset = socketProtocolOffset + 80 + + childrenInitialPIDs = 64 + childrenGrowth = 4 + childrenAttempts = 4 + + // procArgsAlign is the boundary the kernel pads the executable path to + // before the arguments begin: the pointer size of every supported Mac. + procArgsAlign = 8 +) + +// procInfo keeps pointer conversion in the syscall expression so the buffer +// remains alive and cannot move while the kernel writes it. A nil buffer is +// used only by interfaces that return a sizing estimate. +func procInfo(call, a1, a2, a3 int, buf []byte) (int, error) { + if len(buf) > 1<<31-1 { + return 0, unix.EINVAL + } + var n uintptr + var errno unix.Errno + if len(buf) == 0 { + n, _, errno = unix.Syscall6(unix.SYS_PROC_INFO, uintptr(call), uintptr(a1), uintptr(a2), uintptr(a3), 0, 0) + } else { + n, _, errno = unix.Syscall6(unix.SYS_PROC_INFO, uintptr(call), uintptr(a1), uintptr(a2), uintptr(a3), uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) + } + if errno != 0 { + return 0, errno + } + if n > 1<<31-1 || (len(buf) > 0 && n > uintptr(len(buf))) { + return 0, unix.EIO + } + return int(n), nil +} + +// Children lists immediate children, bounding retries when the tree grows +// between reads. When even the last buffer fills, the children it holds are +// returned with ENOMEM: they are a lower bound on the children, never the +// whole list, and a caller counting them must say so. +func Children(ppid int) ([]int, error) { + if ppid <= 0 { + return nil, unix.EINVAL + } + var initial [childrenInitialPIDs * pidSize]byte + buf := initial[:] + for attempt := 0; attempt < childrenAttempts; attempt++ { + n, err := procInfo(callListPIDs, ppidOnly, ppid, 0, buf) + if err != nil { + return nil, err + } + if n%pidSize != 0 { + return nil, unix.EIO + } + if n < len(buf) { + return childPIDs(buf[:n]), nil + } + if attempt+1 < childrenAttempts { + buf = make([]byte, len(buf)*childrenGrowth) + } + } + return childPIDs(buf), unix.ENOMEM +} + +func childPIDs(buf []byte) []int { + pids := make([]int, 0, len(buf)/pidSize) + for offset := 0; offset+pidSize <= len(buf); offset += pidSize { + if pid := int(int32(binary.LittleEndian.Uint32(buf[offset:]))); pid > 0 { + pids = append(pids, pid) + } + } + return pids +} + +// TaskInfo combines identity and CPU from one kernel snapshot so callers can +// validate ancestry before attributing a process's accumulated CPU time. +type TaskInfo struct { + PID, PPID, PGID, Status int + // Start is the process's start time in microseconds. A pid handed out + // again names a process with another start, so a reader that must not + // confuse the two compares it. + Start uint64 + CPUSeconds float64 + Resident uint64 +} + +var ( + frequencyOnce sync.Once + tickFrequency uint64 +) + +// Task reads one process. Kernel task times are absolute ticks even under +// Rosetta; hw.tbfrequency converts them correctly on both supported CPUs. +func Task(pid int) (TaskInfo, error) { + if pid <= 0 { + return TaskInfo{}, unix.EINVAL + } + frequencyOnce.Do(func() { tickFrequency, _ = unix.SysctlUint64("hw.tbfrequency") }) + if tickFrequency == 0 { + return TaskInfo{}, unix.EIO + } + var buf [taskAllInfoSize]byte + n, err := procInfo(callPIDInfo, pid, pidTaskAllInfo, 0, buf[:]) + if err != nil { + return TaskInfo{}, err + } + if n != taskAllInfoSize { + return TaskInfo{}, unix.EIO + } + return TaskInfo{ + PID: int(int32(binary.LittleEndian.Uint32(buf[bsdPIDOffset:]))), + PPID: int(int32(binary.LittleEndian.Uint32(buf[bsdPPIDOffset:]))), + PGID: int(int32(binary.LittleEndian.Uint32(buf[bsdPGIDOffset:]))), + Status: int(binary.LittleEndian.Uint32(buf[bsdStatusOffset:])), + Start: binary.LittleEndian.Uint64(buf[bsdStartSecondsOffset:])*1_000_000 + binary.LittleEndian.Uint64(buf[bsdStartMicrosOffset:]), + CPUSeconds: float64(binary.LittleEndian.Uint64(buf[taskUserOffset:]))/float64(tickFrequency) + float64(binary.LittleEndian.Uint64(buf[taskSystemOffset:]))/float64(tickFrequency), + Resident: binary.LittleEndian.Uint64(buf[taskResidentOffset:]), + }, nil +} + +// ListenPorts inspects only this process's socket descriptors. Descriptor races +// are skipped individually, and a shared socket buffer avoids one allocation +// per descriptor. +func ListenPorts(pid int) []int { + if pid <= 0 { + return nil + } + size, err := procInfo(callPIDInfo, pid, pidListFDs, 0, nil) + if err != nil || size == 0 || size%fdInfoSize != 0 { + return nil + } + fds := make([]byte, size) + n, err := procInfo(callPIDInfo, pid, pidListFDs, 0, fds) + if err != nil || n%fdInfoSize != 0 { + return nil + } + var socket [socketFDInfoSize]byte + var ports []int + for offset := 0; offset < n; offset += fdInfoSize { + fd := fds[offset : offset+fdInfoSize] + if binary.LittleEndian.Uint32(fd[fdTypeOffset:]) != fdTypeSocket { + continue + } + number := int(int32(binary.LittleEndian.Uint32(fd[fdNumberOffset:]))) + count, err := procInfo(callPIDFDInfo, pid, pidFDSocketInfo, number, socket[:]) + if err != nil || count != socketFDInfoSize { + continue + } + family := binary.LittleEndian.Uint32(socket[socketFamilyOffset:]) + if (family != familyIPv4 && family != familyIPv6) || binary.LittleEndian.Uint32(socket[socketKindOffset:]) != socketInfoTCP || binary.LittleEndian.Uint32(socket[socketTCPStateOffset:]) != tcpStateListen { + continue + } + // The low sixteen bits contain network-order bytes on both macOS CPUs. + port := int(binary.BigEndian.Uint16(socket[socketLocalPortOffset:])) + ports = append(ports, port) + } + slices.Sort(ports) + return slices.Compact(ports) +} + +// Argv discards the raw sysctl response after parsing because it also contains +// environment data that must never enter a service record or diagnostic. +func Argv(pid int) []string { + if pid <= 0 { + return nil + } + raw, err := unix.SysctlRaw("kern.procargs2", pid) + if err != nil { + return nil + } + return parseProcArgs(raw) +} + +func parseProcArgs(raw []byte) []string { + if len(raw) < pidSize { + return nil + } + argc := int(int32(binary.LittleEndian.Uint32(raw))) + if argc <= 0 || argc > len(raw)-pidSize { + return nil + } + rest := raw[pidSize:] + end := bytes.IndexByte(rest, 0) + if end < 0 { + return nil + } + // The kernel pads the executable path with NULs so the arguments start on + // a pointer-size boundary of the string area. Skipping every NUL instead + // would swallow an empty argv[0] and shift the first environment string, + // which can hold a secret, into the arguments. Padding that is not all NUL + // is a layout this parser does not know, and it reads as nothing. + start := (end + 1 + procArgsAlign - 1) &^ (procArgsAlign - 1) + if start > len(rest) || len(bytes.Trim(rest[end+1:start], "\x00")) != 0 { + return nil + } + rest = rest[start:] + argv := make([]string, 0, argc) + for range argc { + end = bytes.IndexByte(rest, 0) + if end < 0 { + return nil + } + argv = append(argv, string(rest[:end])) + rest = rest[end+1:] + } + return argv +} diff --git a/internal/darwinproc/procinfo_darwin_test.go b/internal/darwinproc/procinfo_darwin_test.go new file mode 100644 index 0000000000..f4ce0b1c5c --- /dev/null +++ b/internal/darwinproc/procinfo_darwin_test.go @@ -0,0 +1,257 @@ +//go:build darwin + +package darwinproc + +import ( + "encoding/binary" + "math" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "syscall" + "testing" + "time" +) + +// procArgsFixture lays out a kern.procargs2 answer the way the kernel does: +// argc, the executable path padded with NULs to the next eight-byte boundary, +// then the arguments and the environment. +func procArgsFixture(argc int32, exec string, strings ...string) []byte { + raw := make([]byte, 4) + binary.LittleEndian.PutUint32(raw, uint32(argc)) + raw = append(raw, exec...) + raw = append(raw, 0) + for (len(raw)-4)%procArgsAlign != 0 { + raw = append(raw, 0) + } + for _, s := range strings { + raw = append(raw, s...) + raw = append(raw, 0) + } + return raw +} + +func TestDarwinProcArgsStopsBeforeEnvironment(t *testing.T) { + header := func(argc int32, body string) []byte { + raw := make([]byte, 4, 4+len(body)) + binary.LittleEndian.PutUint32(raw, uint32(argc)) + return append(raw, body...) + } + for _, tc := range []struct { + name string + raw []byte + want []string + }{ + {"padding and environment", procArgsFixture(4, "/bin/tool", "tool", "", "two words", "last", "PRIVATE=value"), []string{"tool", "", "two words", "last"}}, + {"empty argv[0] keeps the environment out", procArgsFixture(2, "/bin/tool", "", "argument", "PRIVATE=value"), []string{"", "argument"}}, + {"path that fills its boundary", procArgsFixture(1, "/bin/to", "tool", "PRIVATE=value"), []string{"tool"}}, + {"padding that is not NUL", header(1, "/bin/tool\x00\x00tool\x00PRIVATE=value\x00"), nil}, + {"zero", procArgsFixture(0, "/bin/tool", "tool"), nil}, + {"negative", procArgsFixture(-1, "/bin/tool", "tool"), nil}, + {"absurd", procArgsFixture(1<<30, "/bin/tool", "tool"), nil}, + {"missing argument", procArgsFixture(2, "/bin/tool", "tool"), nil}, + {"truncated argument", header(1, "/bin/tool\x00\x00\x00\x00\x00\x00\x00tool"), nil}, + {"truncated padding", header(1, "/bin/tool\x00\x00"), nil}, + {"truncated executable", header(1, "/bin/tool"), nil}, + {"short header", []byte{1, 0, 0}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := parseProcArgs(tc.raw); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("argv = %q, want %q", got, tc.want) + } + }) + } +} + +// An empty argv[0] is read in place on a live process, with the environment +// left out, whatever the length of the executable path. +func TestDarwinArgvKeepsAnEmptyFirstArgument(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"s", "sl", "sleeper", "sleeperx"} { + link := filepath.Join(dir, name) + if err := os.Symlink("/bin/sleep", link); err != nil { + t.Fatal(err) + } + cmd := exec.Command(link, "30") + cmd.Args[0] = "" + cmd.Env = []string{"PRIVATE=value"} + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + var got []string + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if got = Argv(cmd.Process.Pid); reflect.DeepEqual(got, []string{"", "30"}) { + break + } + } + if !reflect.DeepEqual(got, []string{"", "30"}) { + t.Fatalf("%s: argv = %q, want [\"\" \"30\"]", name, got) + } + } +} + +func sleepChild(t testing.TB) *exec.Cmd { + t.Helper() + cmd := exec.Command("sleep", "30") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + return cmd +} + +func TestDarwinArgvReadsALiveChild(t *testing.T) { + cmd := sleepChild(t) + if got := Argv(cmd.Process.Pid); !reflect.DeepEqual(got, []string{"sleep", "30"}) { + t.Fatalf("argv = %q", got) + } +} + +func TestDarwinTaskMatchesSelf(t *testing.T) { + deadline := time.Now().Add(150 * time.Millisecond) + var work uint64 + for time.Now().Before(deadline) { + for range 10000 { + work = work*1664525 + 1013904223 + } + } + if work == 0 { + t.Log("bounded loop finished at zero") + } + task, err := Task(os.Getpid()) + if err != nil { + t.Fatal(err) + } + if task.PID != os.Getpid() || task.PPID != os.Getppid() { + t.Fatalf("identity = %+v", task) + } + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + t.Fatal(err) + } + seconds := float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6 + if math.Abs(task.CPUSeconds-seconds) > math.Max(seconds*.2, .03) { + t.Fatalf("CPU seconds = %f, getrusage = %f", task.CPUSeconds, seconds) + } + gone := exec.Command("true") + gone.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := gone.Run(); err != nil { + t.Fatal(err) + } + if _, err := Task(gone.Process.Pid); err == nil { + t.Fatal("gone process returned a task") + } +} + +func TestDarwinChildrenListsASpawnedChild(t *testing.T) { + cmd := sleepChild(t) + pids, err := Children(os.Getpid()) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(pids, cmd.Process.Pid) { + t.Fatalf("children %v omit %d", pids, cmd.Process.Pid) + } +} + +func TestDarwinListeningPorts(t *testing.T) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + want := []int{listener.Addr().(*net.TCPAddr).Port} + listener6, err := net.Listen("tcp6", "[::1]:0") + if err == nil { + t.Cleanup(func() { _ = listener6.Close() }) + want = append(want, listener6.Addr().(*net.TCPAddr).Port) + } else { + t.Logf("IPv6 listener unavailable: %v", err) + } + conn, err := net.DialTimeout("tcp4", listener.Addr().String(), time.Second) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + _ = listener.(*net.TCPListener).SetDeadline(time.Now().Add(time.Second)) + accepted, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = accepted.Close() }) + udp, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = udp.Close() }) + // A short directory keeps the socket path within macOS's sockaddr_un limit. + socketDir, err := os.MkdirTemp("", "dp-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(socketDir) }) + local, err := net.Listen("unix", filepath.Join(socketDir, "s")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = local.Close() }) + got := ListenPorts(os.Getpid()) + for _, p := range want { + if !slices.Contains(got, p) { + t.Fatalf("ports %v omit listener %d", got, p) + } + } + for _, p := range []int{conn.LocalAddr().(*net.TCPAddr).Port, udp.LocalAddr().(*net.UDPAddr).Port} { + // A UDP or ephemeral TCP port can coincide with an independently allocated listener. + if !slices.Contains(want, p) && slices.Contains(got, p) { + t.Fatalf("ports %v include non-listener %d", got, p) + } + } + if !slices.IsSorted(got) || len(slices.Compact(slices.Clone(got))) != len(got) { + t.Fatalf("ports not sorted and unique: %v", got) + } +} + +func BenchmarkTask(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := Task(os.Getpid()); err != nil { + b.Fatal(err) + } + } +} +func BenchmarkChildren(b *testing.B) { + _ = sleepChild(b) + b.ReportAllocs() + for b.Loop() { + if _, err := Children(os.Getpid()); err != nil { + b.Fatal(err) + } + } +} +func BenchmarkListenPorts(b *testing.B) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = listener.Close() }) + b.ReportAllocs() + for b.Loop() { + _ = ListenPorts(os.Getpid()) + } +} +func BenchmarkArgv(b *testing.B) { + cmd := sleepChild(b) + b.ReportAllocs() + for b.Loop() { + if len(Argv(cmd.Process.Pid)) == 0 { + b.Fatal("argv unavailable") + } + } +} diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 8fd16ec67c..34dfb48de2 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -115,10 +115,10 @@ type Chunk struct { // Service is a process group still alive after its call returned. type Service struct { - Name string // basename of the first member's argv[0] - Argv []string // the first member's command line (the request's off Linux) + Name string // Basename of the naming member's argv[0]. + Argv []string // The naming member's command line on Linux and macOS; otherwise the request's. PGID int - Ports []int // listening TCP ports, when known (Linux only) + Ports []int // Listening TCP ports, when known on Linux and macOS. DataDir string // cell-relative data directory, when the service keeps one } diff --git a/internal/executor/service.go b/internal/executor/service.go index 42ac58bf38..d0e13c9871 100644 --- a/internal/executor/service.go +++ b/internal/executor/service.go @@ -4,6 +4,7 @@ import ( "context" "os/exec" "path/filepath" + "slices" "strings" "github.com/Agent-Field/codeaf/internal/processgroup" @@ -37,16 +38,42 @@ func Left(call Call) []Service { // listens names it, because it is the server; with none listening the first // member does. The ports of all members are its ports. func describe(req ExecRequest, root string, pgid int, members []int) Service { + service := describeWith(req, root, pgid, members, pidPorts, cmdline) + if len(service.Ports) == 0 { + // An empty query preserves each platform's unknown-versus-empty result + // without inspecting any member's sockets again. + service.Ports = listenPorts(nil) + } + return service +} + +// describeWith uses one socket snapshot for naming and aggregation so a member +// is not inspected twice while a service is recorded. +func describeWith(req ExecRequest, root string, pgid int, members []int, ports func(int) []int, cmdline func(int) []string) Service { + allPorts := make([]int, 0) + naming := 0 + for _, member := range members { + listening := ports(member) + if naming == 0 && len(listening) > 0 { + naming = member + } + allPorts = append(allPorts, listening...) + } + if naming == 0 && len(members) > 0 { + naming = members[0] + } + slices.Sort(allPorts) + allPorts = slices.Compact(allPorts) argv := req.Argv - if name := namingMember(members); name > 0 { - if a := cmdline(name); len(a) > 0 { + if naming > 0 { + if a := cmdline(naming); len(a) > 0 { argv = a } } name := filepath.Base(argv[0]) return Service{ Name: name, Argv: argv, PGID: pgid, - Ports: listenPorts(members), + Ports: allPorts, DataDir: dataDir(name, argv, root, req.Dir), } } @@ -100,18 +127,3 @@ func inside(root, dir, path string) string { } return filepath.ToSlash(rel) } - -// namingMember is the member that names a service: the first that listens on a -// port, else the first member, else 0. A shell that started a server lingers -// beside it, and the shell's command line says nothing of what is being served. -func namingMember(members []int) int { - for _, m := range members { - if len(listenPorts([]int{m})) > 0 { - return m - } - } - if len(members) > 0 { - return members[0] - } - return 0 -} diff --git a/internal/executor/service_darwin.go b/internal/executor/service_darwin.go new file mode 100644 index 0000000000..aa748b30b0 --- /dev/null +++ b/internal/executor/service_darwin.go @@ -0,0 +1,22 @@ +//go:build darwin + +package executor + +import ( + "slices" + + "github.com/Agent-Field/codeaf/internal/darwinproc" +) + +func cmdline(pid int) []string { return darwinproc.Argv(pid) } + +func pidPorts(pid int) []int { return darwinproc.ListenPorts(pid) } + +func listenPorts(pids []int) []int { + ports := make([]int, 0) + for _, pid := range pids { + ports = append(ports, pidPorts(pid)...) + } + slices.Sort(ports) + return slices.Compact(ports) +} diff --git a/internal/executor/service_describe_test.go b/internal/executor/service_describe_test.go new file mode 100644 index 0000000000..4ac262a0cc --- /dev/null +++ b/internal/executor/service_describe_test.go @@ -0,0 +1,64 @@ +package executor + +import ( + "reflect" + "testing" +) + +func TestDescribeInspectsEachMemberOnce(t *testing.T) { + members := []int{10, 11, 12} + wantArgv := []string{"postgres", "-D", "data"} + reads := make(map[int]int) + ports := func(pid int) []int { + reads[pid]++ + return map[int][]int{11: {8080, 5432}, 12: {8080}}[pid] + } + argv := func(pid int) []string { + if pid != 11 { + t.Fatalf("argv read for member %d, want first listener 11", pid) + } + return wantArgv + } + got := describeWith(ExecRequest{Argv: []string{"sh"}}, "/w", 10, members, ports, argv) + for _, pid := range members { + if reads[pid] != 1 { + t.Errorf("member %d inspected %d times, want once", pid, reads[pid]) + } + } + if got.Name != "postgres" || !reflect.DeepEqual(got.Argv, wantArgv) || + !reflect.DeepEqual(got.Ports, []int{5432, 8080}) || got.DataDir != "data" || got.PGID != 10 { + t.Fatalf("service = %+v", got) + } + + t.Run("no listeners", func(t *testing.T) { + for _, fallback := range []bool{false, true} { + reads := make(map[int]int) + req := ExecRequest{Argv: []string{"/bin/request", "argument"}} + wantArgv := []string{"/bin/sleep", "30"} + wantName := "sleep" + if fallback { + wantArgv, wantName = req.Argv, "request" + } + got := describeWith(req, "/w", 10, members, func(pid int) []int { + reads[pid]++ + return nil + }, func(pid int) []string { + if pid != members[0] { + t.Fatalf("argv read for member %d, want first member %d", pid, members[0]) + } + if fallback { + return nil + } + return wantArgv + }) + for _, pid := range members { + if reads[pid] != 1 { + t.Errorf("member %d inspected %d times, want once", pid, reads[pid]) + } + } + if got.Name != wantName || !reflect.DeepEqual(got.Argv, wantArgv) || len(got.Ports) != 0 { + t.Fatalf("fallback=%t: service = %+v", fallback, got) + } + } + }) +} diff --git a/internal/executor/service_linux.go b/internal/executor/service_linux.go index 559a307076..f51c85e82f 100644 --- a/internal/executor/service_linux.go +++ b/internal/executor/service_linux.go @@ -26,11 +26,8 @@ func cmdline(pid int) []string { func listenPorts(pids []int) []int { seen := map[int]bool{} for _, pid := range pids { - listening := listeningInodes(pid) - for inode := range socketInodes(pid) { - if port, ok := listening[inode]; ok { - seen[port] = true - } + for _, port := range pidPorts(pid) { + seen[port] = true } } ports := make([]int, 0, len(seen)) @@ -41,6 +38,17 @@ func listenPorts(pids []int) []int { return ports } +func pidPorts(pid int) []int { + listening := listeningInodes(pid) + ports := make([]int, 0) + for inode := range socketInodes(pid) { + if port, ok := listening[inode]; ok { + ports = append(ports, port) + } + } + return ports +} + func listeningInodes(pid int) map[uint64]int { out := map[uint64]int{} for _, table := range []string{"tcp", "tcp6"} { diff --git a/internal/executor/service_linux_test.go b/internal/executor/service_linux_test.go index 0eaf13ae71..b913825cb6 100644 --- a/internal/executor/service_linux_test.go +++ b/internal/executor/service_linux_test.go @@ -5,12 +5,8 @@ package executor import ( "reflect" "testing" - - "github.com/Agent-Field/codeaf/internal/processgroup" ) -func membersOf(pgid int) []int { return processgroup.Members(pgid) } - const tcpFixture = ` sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode 0: 0100007F:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 12345 1 0000 100 0 0 10 0 1: 0100007F:0050 0100007F:9C40 01 00000000:00000000 00:00000000 00000000 1000 0 22222 1 0000 100 0 0 10 0 diff --git a/internal/executor/service_other.go b/internal/executor/service_other.go index 0a4ffe3f19..f16013f462 100644 --- a/internal/executor/service_other.go +++ b/internal/executor/service_other.go @@ -1,8 +1,10 @@ -//go:build !linux +//go:build !linux && !darwin package executor -// Off Linux the record keeps the request's argv and no ports. +// Where process inspection is unavailable, the record keeps the request's argv and no ports. func cmdline(int) []string { return nil } +func pidPorts(int) []int { return nil } + func listenPorts([]int) []int { return nil } diff --git a/internal/executor/service_other_test.go b/internal/executor/service_other_test.go deleted file mode 100644 index 328bd36598..0000000000 --- a/internal/executor/service_other_test.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build !linux - -package executor - -func membersOf(int) []int { return nil } diff --git a/internal/executor/service_unix_test.go b/internal/executor/service_unix_test.go index 80e1a8d473..679d03064a 100644 --- a/internal/executor/service_unix_test.go +++ b/internal/executor/service_unix_test.go @@ -1,17 +1,22 @@ -//go:build !windows +//go:build linux || darwin -// K6 keeps Unix process-group service inspection on Unix. +// K6 keeps process-group service inspection on Linux and macOS. package executor import ( "context" "syscall" "testing" - "time" ) func TestLocalRecordsAListeningServiceFully(t *testing.T) { - script := "exec python3 -c 'import socket,time;s=socket.socket();s.bind((\"127.0.0.1\",0));s.listen();print(s.getsockname()[1],flush=True);time.sleep(60)' >port.txt 2>&1 port.txt 2>&1 0 && int(record.Eproc.Pgid) == pgid && record.Proc.P_stat != darwinproc.StatusZombie { + pids = append(pids, int(record.Proc.P_pid)) + } + } + sort.Ints(pids) + return pids +} + +// The shell is reaped before a sweep on macOS (internal/exec/shellwait_other.go), +// so the existing signal probe has no zombie leader to mistake for live work. +func groupHasLiveMember(pgid int) bool { + return Alive(pgid) +} diff --git a/internal/processgroup/members_darwin_test.go b/internal/processgroup/members_darwin_test.go new file mode 100644 index 0000000000..f9b1c5b1e0 --- /dev/null +++ b/internal/processgroup/members_darwin_test.go @@ -0,0 +1,71 @@ +//go:build darwin + +package processgroup + +import ( + "os/exec" + "reflect" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/darwinproc" + "golang.org/x/sys/unix" +) + +func TestDarwinMembersExcludeZombies(t *testing.T) { + records := make([]unix.KinfoProc, 6) + for i, pid := range []int32{30, 10, 20, 40, 0, -1} { + records[i].Proc.P_pid = pid + records[i].Proc.P_stat = 3 + records[i].Eproc.Pgid = 100 + } + records[0].Proc.P_stat = 4 + records[2].Proc.P_stat = darwinproc.StatusZombie + records[3].Eproc.Pgid = 101 + if got := darwinMemberPIDs(100, records); !reflect.DeepEqual(got, []int{10, 30}) { + t.Fatalf("Members = %v, want sleeping and stopped members [10 30]", got) + } +} + +func TestDarwinMembersReadsALiveGroup(t *testing.T) { + cmd := startDarwinGroup(t, "sleep", "30") + pid := cmd.Process.Pid + if got := Members(pid); !reflect.DeepEqual(got, []int{pid}) { + t.Fatalf("Members(%d) = %v, want [%d]", pid, got, pid) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + _ = cmd.Wait() + if got := Members(pid); len(got) != 0 { + t.Fatalf("Members(%d) after wait = %v, want empty", pid, got) + } +} + +func startDarwinGroup(tb testing.TB, name string, args ...string) *exec.Cmd { + tb.Helper() + cmd := exec.Command(name, args...) + Configure(cmd) + if err := cmd.Start(); err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { + if cmd.ProcessState == nil { + _ = Kill(cmd.Process.Pid) + _ = cmd.Wait() + } + }) + return cmd +} + +func BenchmarkMembers(b *testing.B) { + cmd := startDarwinGroup(b, "sh", "-c", strings.Repeat("sleep 30 & ", 9)+"wait") + waitForDarwinUsage(b, CaptureGroup(cmd.Process.Pid), 10) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if got := Members(cmd.Process.Pid); len(got) != 10 || got[0] != cmd.Process.Pid { + b.Fatalf("Members = %v, want the leader and nine sleeping children", got) + } + } +} diff --git a/internal/processgroup/members_other.go b/internal/processgroup/members_other.go index a0777f0b37..137975cd77 100644 --- a/internal/processgroup/members_other.go +++ b/internal/processgroup/members_other.go @@ -1,14 +1,14 @@ -//go:build !windows && !linux +//go:build !windows && !linux && !darwin package processgroup -// groupHasLiveMember is the signal probe off Linux: there the shell is reaped -// before any sweep looks at its group (internal/exec, shellwait_other.go), so -// no zombie leader is ever in the group when this is asked, and kill(-pgid, 0) -// answers correctly. +// groupHasLiveMember retains the signal probe on other Unix platforms: the +// shell is reaped before any sweep looks at its group (internal/exec, +// shellwait_other.go), so no zombie leader is in the group when this is asked, +// and kill(-pgid, 0) answers correctly. func groupHasLiveMember(pgid int) bool { return Alive(pgid) } -// Members is unknown off Linux; callers fall back to what they started. +// Members is unknown off Linux and macOS; callers fall back to what they started. func Members(pgid int) []int { return nil } diff --git a/internal/processgroup/usage.go b/internal/processgroup/usage.go index b3d418024b..7eb8c794f0 100644 --- a/internal/processgroup/usage.go +++ b/internal/processgroup/usage.go @@ -16,11 +16,9 @@ package processgroup // cores the subtree held, which is the reading this product bounds. One // reading on its own is not a rate and must not be read as one. // -// THE READING IS /proc, WHICH IS LINUX'S. A platform without it answers "cannot -// say" rather than a guess, and a bound that cannot say never cuts — the same -// silence rule the admission governor keeps (session's task_pressure.go): a -// person on a machine this cannot measure gets exactly the scheduler they had -// before the bound existed. +// Linux reads /proc and macOS reads the kernel's process-info interface. Other +// platforms answer "cannot say" rather than guess, and a bound that cannot say +// never cuts, preserving the scheduler the person had before the bound existed. // Usage is what one job subtree is using at one instant. type Usage struct { @@ -30,4 +28,11 @@ type Usage struct { // every process in it. Subtract two readings and divide by the wall between // them to get the cores the subtree held. CPUSeconds float64 + // CPUPartial says CPUSeconds left out part of the tree: the walk stopped at + // its limit, a listing was truncated, or a process could not be read. Such + // a sum is not comparable with another reading, because a process missing + // from one and present in the next would read as a burst of its whole + // lifetime's CPU, so no rate may be taken from it. Processes is still a + // lower bound. + CPUPartial bool } diff --git a/internal/processgroup/usage_darwin.go b/internal/processgroup/usage_darwin.go new file mode 100644 index 0000000000..cd87c8d9df --- /dev/null +++ b/internal/processgroup/usage_darwin.go @@ -0,0 +1,125 @@ +//go:build darwin + +package processgroup + +import ( + "errors" + + "github.com/Agent-Field/codeaf/internal/darwinproc" + "golang.org/x/sys/unix" +) + +// usageWalkLimit is how many processes one reading looks at most. Past it the +// process count is a lower bound and the CPU sum is marked partial, so a tree +// that keeps growing cannot make every two-second sample a longer walk. A storm +// that size normally still counts far over the job bound's 128-process ceiling +// and is cut on the count; a lower count can only hold a cut back. +const usageWalkLimit = 1024 + +// usageReader is the kernel's answer for one process and for its children. It +// is a variable so the walk's limits and refusals can be proven on a scripted +// tree instead of real load. +var usageReader = struct { + task func(pid int) (darwinproc.TaskInfo, error) + children func(ppid int) ([]int, error) +}{darwinproc.Task, darwinproc.Children} + +// Usage reports the recorded leader's whole subtree, or says it cannot say. +// Identity is checked around the walk because a reading of a reused pid could +// cause the bound to cut somebody else's work. +func (g Group) Usage() (Usage, bool) { + if ok, _ := g.owned(); !ok { + return Usage{}, false + } + usage, ok := subtreeUsage(g.pid) + if !ok { + return Usage{}, false + } + if ok, _ := g.owned(); !ok { + return Usage{}, false + } + return usage, true +} + +// subtreeUsage walks the tree below root through each process's children, so +// a child that changed process groups is still the job's. +// +// A PARENT IS RE-READ AFTER ITS CHILDREN ARE READ. A queued process can exit +// and its pid be handed to another process while its children are listed or +// read; the children of that stranger would pass the parent-pid check. A pid is +// never handed on while its process lives, so a parent that reads with the same +// start time after its children were read was their parent while they were +// read, and only then are they admitted. +// +// A listing that filled even the largest buffer, a walk that reached +// [usageWalkLimit], and a process that could not be read for any reason other +// than having exited are counted as far as they got, with the CPU sum marked +// partial ([Usage.CPUPartial]): the count is a lower bound and no rate is taken +// from the sum. A process that exited between two reads is skipped, as the +// Linux reading skips it. A listing that fails outright makes the whole +// reading unknown. +func subtreeUsage(root int) (Usage, bool) { + top, err := usageReader.task(root) + if err != nil || top.PID != root || top.Status == darwinproc.StatusZombie { + return Usage{}, false + } + usage := Usage{} + seen := map[int]bool{root: true} + queue := []darwinproc.TaskInfo{top} + for next := 0; next < len(queue); next++ { + parent := queue[next] + usage.Processes++ + usage.CPUSeconds += parent.CPUSeconds + if len(seen) >= usageWalkLimit { + usage.CPUPartial = true + continue + } + children, err := usageReader.children(parent.PID) + switch { + case err == nil: + case errors.Is(err, unix.ENOMEM): + usage.CPUPartial = true + case errors.Is(err, unix.ESRCH): + continue + default: + return Usage{}, false + } + var read []darwinproc.TaskInfo + for _, pid := range children { + if len(seen) >= usageWalkLimit { + usage.CPUPartial = true + break + } + if pid <= 0 || seen[pid] { + continue + } + seen[pid] = true + child, err := usageReader.task(pid) + if err != nil && !errors.Is(err, unix.ESRCH) { + usage.CPUPartial = true + continue + } + // A child can exit, be reparented or have its pid reused between reads. + // Skip that snapshot rather than charging an unrelated process to us. + if err != nil || child.PID != pid || child.PPID != parent.PID || child.Status == darwinproc.StatusZombie { + continue + } + read = append(read, child) + } + again, err := usageReader.task(parent.PID) + if err != nil || again.Start != parent.Start || again.PID != parent.PID { + if next == 0 { + return Usage{}, false + } + // A parent that exited or whose pid changed hands left the tree with + // its children; one that merely could not be read leaves them out of + // this sum only, so the sum is partial. + if err != nil && !errors.Is(err, unix.ESRCH) { + usage.CPUPartial = true + } + continue + } + queue = append(queue, read...) + } + return usage, true +} diff --git a/internal/processgroup/usage_darwin_test.go b/internal/processgroup/usage_darwin_test.go new file mode 100644 index 0000000000..ec769fff9d --- /dev/null +++ b/internal/processgroup/usage_darwin_test.go @@ -0,0 +1,80 @@ +//go:build darwin + +package processgroup + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestDarwinUsageReadsDescendants(t *testing.T) { + cmd := startDarwinGroup(t, "sh", "-c", `sleep 30 & sleep 30 & sh -c 'sleep 30 & wait' & wait`) + group := CaptureGroup(cmd.Process.Pid) + usage := waitForDarwinUsage(t, group, 5) + if usage.CPUSeconds < 0 { + t.Fatalf("Usage CPUSeconds = %v, want nonnegative", usage.CPUSeconds) + } +} + +func TestDarwinUsageRefusesUnownedGroup(t *testing.T) { + if usage, ok := (Group{}).Usage(); ok || usage != (Usage{}) { + t.Fatalf("unowned Usage = %+v, %v, want zero, false", usage, ok) + } + cmd := startDarwinGroup(t, "sleep", "30") + id, ok := CaptureGroup(cmd.Process.Pid).Identity() + if !ok { + t.Fatal("could not capture sleeping leader identity") + } + id.Start++ + if usage, ok := Restore(id).Usage(); ok || usage != (Usage{}) { + t.Fatalf("wrong identity Usage = %+v, %v, want zero, false", usage, ok) + } + + t.Run("identity changes during sample", func(t *testing.T) { + group := CaptureGroup(os.Getpid()) + saved := processStart + t.Cleanup(func() { processStart = saved }) + reads := 0 + processStart = func(pid int) (uint64, bool) { + reads++ + return group.start, reads == 1 + } + if usage, ok := group.Usage(); ok || usage != (Usage{}) { + t.Fatalf("changed identity Usage = %+v, %v, want zero, false", usage, ok) + } + if reads != 2 { + t.Fatalf("identity read %d times, want before and after the sample", reads) + } + }) +} + +func waitForDarwinUsage(tb testing.TB, group Group, processes int) Usage { + tb.Helper() + deadline := time.Now().Add(5 * time.Second) + var usage Usage + var ok bool + for time.Now().Before(deadline) { + usage, ok = group.Usage() + if ok && usage.Processes >= processes { + return usage + } + time.Sleep(10 * time.Millisecond) + } + tb.Fatalf("Usage = %+v, %v, want at least %d processes", usage, ok, processes) + return Usage{} +} + +func BenchmarkUsage(b *testing.B) { + cmd := startDarwinGroup(b, "sh", "-c", strings.Repeat("sleep 30 & ", 9)+"wait") + group := CaptureGroup(cmd.Process.Pid) + waitForDarwinUsage(b, group, 10) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if usage, ok := group.Usage(); !ok || usage.Processes < 10 || usage.CPUSeconds < 0 { + b.Fatalf("Usage = %+v, %v, want the leader and sleeping children", usage, ok) + } + } +} diff --git a/internal/processgroup/usage_darwin_walk_test.go b/internal/processgroup/usage_darwin_walk_test.go new file mode 100644 index 0000000000..cf521e767a --- /dev/null +++ b/internal/processgroup/usage_darwin_walk_test.go @@ -0,0 +1,230 @@ +//go:build darwin + +package processgroup + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/darwinproc" + "golang.org/x/sys/unix" +) + +// scriptedTree answers the walk from a fixed table, so its limits and refusals +// are proven on a tree that exists only as numbers and never as real load. +type scriptedTree struct { + tasks map[int]darwinproc.TaskInfo + children map[int][]int + listErr map[int]error + // restart gives a pid a new start time after its children are listed: the + // pid was handed to another process in between. + restart map[int]bool + listed map[int]bool + taskRead int +} + +func (s *scriptedTree) use(t *testing.T) { + t.Helper() + saved := usageReader + t.Cleanup(func() { usageReader = saved }) + s.listed = map[int]bool{} + usageReader.task = func(pid int) (darwinproc.TaskInfo, error) { + s.taskRead++ + task, ok := s.tasks[pid] + if !ok { + return darwinproc.TaskInfo{}, unix.ESRCH + } + if s.restart[pid] && s.listed[pid] { + task.Start++ + } + return task, nil + } + usageReader.children = func(ppid int) ([]int, error) { + s.listed[ppid] = true + return s.children[ppid], s.listErr[ppid] + } +} + +func task(pid, ppid, pgid int, cpu float64) darwinproc.TaskInfo { + return darwinproc.TaskInfo{PID: pid, PPID: ppid, PGID: pgid, Start: uint64(pid) * 1000, CPUSeconds: cpu, Status: 2} +} + +// The job's tree is its children's children too, including one that moved to +// a process group of its own; each live process is counted once. +func TestDarwinUsageCountsTheWholeTreeOnce(t *testing.T) { + tree := &scriptedTree{ + tasks: map[int]darwinproc.TaskInfo{ + 10: task(10, 1, 10, 1), 11: task(11, 10, 10, 2), 12: task(12, 10, 99, 3), 13: task(13, 11, 10, 4), + }, + children: map[int][]int{10: {11, 12, 11}, 11: {13}, 12: {13}}, + } + tree.use(t) + usage, ok := subtreeUsage(10) + if !ok || usage.Processes != 4 || usage.CPUSeconds != 10 || usage.CPUPartial { + t.Fatalf("usage = %+v, %v, want 4 processes and a complete 10 CPU seconds", usage, ok) + } +} + +// A queued process whose pid went to another process before its children were +// listed contributes itself but none of the stranger's children. +func TestDarwinUsageKeepsAReusedParentsChildrenOut(t *testing.T) { + tree := &scriptedTree{ + tasks: map[int]darwinproc.TaskInfo{ + 10: task(10, 1, 10, 1), 11: task(11, 10, 10, 1), 30: task(30, 11, 30, 50), + }, + children: map[int][]int{10: {11}, 11: {30}}, + restart: map[int]bool{11: true}, + } + tree.use(t) + usage, ok := subtreeUsage(10) + if !ok || usage.Processes != 2 || usage.CPUSeconds != 2 { + t.Fatalf("usage = %+v, %v, want the leader and its own child only", usage, ok) + } + tree.restart, tree.listed = map[int]bool{10: true}, map[int]bool{} + if usage, ok := subtreeUsage(10); ok { + t.Fatalf("a root whose pid changed hands mid-walk was read: %+v", usage) + } +} + +// A listing that filled the largest buffer is a lower bound and still counts; +// an unexpected failure makes the reading unknown rather than partial. +func TestDarwinUsageCountsATruncatedListingAndRefusesAFailedOne(t *testing.T) { + tasks := map[int]darwinproc.TaskInfo{10: task(10, 1, 10, 0)} + var kids []int + for pid := 100; pid < 400; pid++ { + tasks[pid] = task(pid, 10, 10, 0) + kids = append(kids, pid) + } + tree := &scriptedTree{tasks: tasks, children: map[int][]int{10: kids}, listErr: map[int]error{10: unix.ENOMEM}} + tree.use(t) + if usage, ok := subtreeUsage(10); !ok || usage.Processes != 301 || !usage.CPUPartial { + t.Fatalf("truncated listing: usage = %+v, %v, want the 301 it named and a partial CPU sum", usage, ok) + } + tree.listErr = map[int]error{10: unix.EPERM} + if usage, ok := subtreeUsage(10); ok { + t.Fatalf("a listing that failed was read as a partial tree: %+v", usage) + } +} + +// A tree past the walk limit is read as far as the limit, with a bounded +// number of kernel reads, and is still far over the job bound's ceiling. +func TestDarwinUsageStopsAtTheWalkLimit(t *testing.T) { + tasks := map[int]darwinproc.TaskInfo{10: task(10, 1, 10, 0)} + children := map[int][]int{} + parent, next := 10, 11 + for i := 0; i < 5000; i++ { + tasks[next] = task(next, parent, 10, 0) + children[parent] = append(children[parent], next) + if i%3 == 2 { + parent = next + } + next++ + } + tree := &scriptedTree{tasks: tasks, children: children} + tree.use(t) + usage, ok := subtreeUsage(10) + if !ok || usage.Processes != usageWalkLimit || !usage.CPUPartial { + t.Fatalf("usage = %+v, %v, want exactly the walk limit %d and a partial CPU sum", usage, ok, usageWalkLimit) + } + if tree.taskRead > 2*usageWalkLimit+1 { + t.Fatalf("%d task reads for a walk limited to %d processes", tree.taskRead, usageWalkLimit) + } + if len(tree.listed) > usageWalkLimit { + t.Fatalf("%d child listings for a walk limited to %d processes", len(tree.listed), usageWalkLimit) + } +} + +// A child that exited between two reads leaves the tree, as on Linux; one that +// could not be read for another reason makes the CPU sum partial. +func TestDarwinUsageMarksAnUnreadableChildPartial(t *testing.T) { + tree := &scriptedTree{ + tasks: map[int]darwinproc.TaskInfo{10: task(10, 1, 10, 1)}, + children: map[int][]int{10: {11}}, + } + tree.use(t) + if usage, ok := subtreeUsage(10); !ok || usage.Processes != 1 || usage.CPUPartial { + t.Fatalf("an exited child: usage = %+v, %v, want the leader alone and a complete sum", usage, ok) + } + saved := usageReader.task + usageReader.task = func(pid int) (darwinproc.TaskInfo, error) { + if pid == 11 { + return darwinproc.TaskInfo{}, unix.EPERM + } + return saved(pid) + } + if usage, ok := subtreeUsage(10); !ok || usage.Processes != 1 || !usage.CPUPartial { + t.Fatalf("an unreadable child: usage = %+v, %v, want the leader and a partial sum", usage, ok) + } +} + +// A parent that read once and then could not be re-read leaves its children out +// of this reading only, so the sum is partial; one that exited does not. +func TestDarwinUsageMarksAnUnreadableParentPartial(t *testing.T) { + tree := &scriptedTree{ + tasks: map[int]darwinproc.TaskInfo{10: task(10, 1, 10, 1), 11: task(11, 10, 10, 1), 12: task(12, 11, 10, 9)}, + children: map[int][]int{10: {11}, 11: {12}}, + } + tree.use(t) + saved := usageReader.task + reads := map[int]int{} + failWith := unix.EPERM + usageReader.task = func(pid int) (darwinproc.TaskInfo, error) { + reads[pid]++ + if pid == 11 && reads[pid] == 2 { + return darwinproc.TaskInfo{}, failWith + } + return saved(pid) + } + if usage, ok := subtreeUsage(10); !ok || usage.Processes != 2 || !usage.CPUPartial { + t.Fatalf("a parent that could not be re-read: usage = %+v, %v, want 2 processes and a partial sum", usage, ok) + } + reads, failWith = map[int]int{}, unix.ESRCH + if usage, ok := subtreeUsage(10); !ok || usage.Processes != 2 || usage.CPUPartial { + t.Fatalf("a parent that exited: usage = %+v, %v, want 2 processes and a complete sum", usage, ok) + } +} + +// A parent's pid and its listed child's pid can both change hands right after +// the parent read the same again. The child read after that swap is a +// stranger's busy child and must not be charged to the job: a child is +// admitted only when its parent still reads as the same process after it. +func TestDarwinUsageAdmitsAChildOnlyWhileItsParentIsTheSame(t *testing.T) { + saved := usageReader + t.Cleanup(func() { usageReader = saved }) + leader, own := task(10, 1, 10, 1), task(11, 10, 10, 1) + strangerLeader, strangerChild := task(10, 1, 50, 0), task(11, 10, 50, 500) + strangerLeader.Start, strangerChild.Start = 90000, 91000 + strangers := map[int]darwinproc.TaskInfo{12: task(12, 11, 50, 300), 13: task(13, 11, 50, 300)} + leaderReads, swapped := 0, false + usageReader.task = func(pid int) (darwinproc.TaskInfo, error) { + switch { + case pid == 10 && swapped: + return strangerLeader, nil + case pid == 10: + // The swap lands just after the leader's second read. + if leaderReads++; leaderReads == 2 { + swapped = true + } + return leader, nil + case pid == 11 && swapped: + return strangerChild, nil + case pid == 11: + return own, nil + case swapped && strangers[pid].PID == pid: + return strangers[pid], nil + } + return darwinproc.TaskInfo{}, unix.ESRCH + } + usageReader.children = func(ppid int) ([]int, error) { + switch { + case ppid == 10: + return []int{11}, nil + case ppid == 11 && swapped: + return []int{12, 13}, nil + } + return nil, nil + } + usage, ok := subtreeUsage(10) + if !ok || usage.Processes != 2 || usage.CPUSeconds != 2 { + t.Fatalf("usage = %+v, %v, want the leader and its own child only (2 processes, 2 CPU seconds)", usage, ok) + } +} diff --git a/internal/processgroup/usage_other.go b/internal/processgroup/usage_other.go index 4ecf7f23a7..84077706f8 100644 --- a/internal/processgroup/usage_other.go +++ b/internal/processgroup/usage_other.go @@ -1,10 +1,8 @@ -//go:build !linux +//go:build !linux && !darwin package processgroup -// Usage has no answer where there is no /proc to read: the subtree's CPU time -// and process count are Linux's to report here. A platform that cannot say -// answers so, and a bound that cannot say never cuts — it gets the scheduler it -// had before the bound existed rather than a guess at somebody else's process -// group. +// Usage has no answer where the subtree's CPU time and process count cannot be +// read. A bound that cannot say never cuts, preserving the existing scheduler +// rather than guessing at somebody else's process group. func (g Group) Usage() (Usage, bool) { return Usage{}, false } diff --git a/internal/session/jobbound.go b/internal/session/jobbound.go index 8248be1fc7..d0ef847d13 100644 --- a/internal/session/jobbound.go +++ b/internal/session/jobbound.go @@ -58,8 +58,8 @@ package session // // The watch is this program's own and runs everywhere. It reads the subtree // through the process-group handle #1155 gave, which works wherever a process -// group is a process group; the CPU-and-process reading itself is /proc, so off -// Linux it says it cannot say and the bound never cuts ([processgroup.Usage]). +// group is a process group. Linux and macOS read the subtree's CPU and process +// count; other platforms cannot say and the bound never cuts ([processgroup.Usage]). // The subtree's USAGE needs no cgroup — a job is a process group and not a // cgroup. Whether a quota BINDS is a different question — how many cores the // subtree could ever reach — answered from the process's own cgroup CPU quota and @@ -179,10 +179,18 @@ func (b *subtreeBound) strike(now time.Time, usage processgroup.Usage, ok bool) return false } cores := 0.0 - if b.have && now.After(b.last) { - cores = (usage.CPUSeconds - b.lastCPU) / now.Sub(b.last).Seconds() + if usage.CPUPartial { + // A sum that left part of the tree out yields no rate, and it is no + // baseline for the next reading either: only the process count is + // judged, and the next complete reading arms the CPU half again. + b.have = false + } else { + if b.have && now.After(b.last) { + cores = (usage.CPUSeconds - b.lastCPU) / now.Sub(b.last).Seconds() + } + b.last, b.lastCPU, b.have = now, usage.CPUSeconds, true } - b.last, b.lastCPU, b.have, b.lastCores = now, usage.CPUSeconds, true, cores + b.lastCores = cores if usage.Processes > jobBoundProcesses || cores > b.ceilingCores() { b.strikes++ } else { diff --git a/internal/session/jobbound_test.go b/internal/session/jobbound_test.go index cfa0898075..2228ce7108 100644 --- a/internal/session/jobbound_test.go +++ b/internal/session/jobbound_test.go @@ -230,3 +230,39 @@ func TestASubtreeThatCannotBeReadIsNeverCut(t *testing.T) { t.Fatal("a subtree whose usage could not be read was cut: silence is not pressure") } } + +// TestAPartialCPUSumNeverMakesARate: a reading that left part of the tree out +// (macOS's walk limit, a truncated listing) still counts processes, but its CPU +// sum is not comparable with the next one. A storm of sleepers read partially +// for fourteen readings that then shrinks to a leader and one old child — whose +// lifetime CPU the partial sums never held — must not be cut on that jump. The +// storm starts after one complete, quiet reading, so the partial readings must +// also drop that earlier baseline rather than leave it waiting for the jump. +func TestAPartialCPUSumNeverMakesARate(t *testing.T) { + registry := &jobRegistry{} + one := &job{id: 1, kind: jobKindBash, state: jobRunning, done: make(chan struct{})} + readings := []processgroup.Usage{{Processes: 2, CPUSeconds: 1}} + for i := 0; i < jobBoundStrikes-1; i++ { + readings = append(readings, processgroup.Usage{Processes: 1100, CPUSeconds: 1, CPUPartial: true}) + } + readings = append(readings, processgroup.Usage{Processes: 2, CPUSeconds: 1001}, processgroup.Usage{Processes: 2, CPUSeconds: 1001}) + subtreeReadings(t, readings, 2*time.Second) + if driveBound(registry, one, newTestBound(), len(readings)) { + t.Fatal("a jump between a partial and a complete CPU sum was read as a storm and cut") + } +} + +// TestAPartialReadingStillCountsTowardsTheProcessBackstop: the count from a +// partial reading is a lower bound, and a fork storm read that way is cut. +func TestAPartialReadingStillCountsTowardsTheProcessBackstop(t *testing.T) { + registry := &jobRegistry{} + one := &job{id: 1, kind: jobKindBash, state: jobRunning, done: make(chan struct{})} + readings := storm(0.5, 1024, long, time.Second) + for i := range readings { + readings[i].CPUPartial = true + } + subtreeReadings(t, readings, time.Second) + if !driveBound(registry, one, newTestBound(), long) { + t.Fatal("a fork storm read partially was never cut by the process backstop") + } +} From d19f0ff866188fe6167d36f817c67f7325febeec Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:50:33 -0500 Subject: [PATCH 02/28] factory: mark a draft pull request ready from the app (#2238) * test: specify marking a job pull request ready Co-Authored-By: Claude Opus 5.5 * feat: mark job pull requests ready through the app Reuse the GitHub ready mutation with the caller credential and persist its confirmed draft state before returning. Co-Authored-By: Claude Opus 5.5 * test: cover GitHub ready conflicts and permission refusals Co-Authored-By: Claude Opus 5.5 * fix: preserve definite ready-for-review refusals Keep automatic publication idempotent while explicit ready requests conflict on an already ready PR and preserve GitHub permission refusals. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: pr-ready is a server-owned verb in the client contract Its own serve tests cover it against the real store. Co-Authored-By: Claude Opus 5.5 * test: a draft's merge block now names what readiness waits for Co-Authored-By: Claude Opus 5.5 * Take the gated copy of the union merged docs --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_actions.go | 5 + cmd/codeaf/appfactory_job_pr_ready_test.go | 170 ++++++++++++++++++ .../unreleased/2238-factory-job-pr-ready.md | 13 ++ docs/design/app-v3/PROTOCOL.md | 17 ++ internal/appwire/factory_job_pr_ready.go | 52 ++++++ internal/appwire/server.go | 1 + internal/factory/api/codec.go | 1 + internal/factory/api/verbs.go | 1 + internal/factory/api/wire.go | 2 + internal/factory/client/contract_test.go | 2 +- internal/factory/github/readiness.go | 29 +++ internal/factory/record/activity.go | 2 + internal/factory/record/record.go | 6 +- internal/factory/serve/pr_ready.go | 95 ++++++++++ internal/factory/serve/repo_off.go | 4 +- internal/factory/serve/verbs.go | 3 +- .../chat/factory-stage-conversations.md | 23 +++ internal/manual/chat/factory.md | 18 ++ .../manual/chat/running-from-the-terminal.md | 11 +- internal/manual/chat/settings.md | 6 +- internal/praf/github/forge.go | 6 + internal/praf/github/readiness.go | 32 +++- 22 files changed, 487 insertions(+), 12 deletions(-) create mode 100644 cmd/codeaf/appfactory_job_pr_ready_test.go create mode 100644 docs/changes/unreleased/2238-factory-job-pr-ready.md create mode 100644 internal/appwire/factory_job_pr_ready.go create mode 100644 internal/factory/serve/pr_ready.go diff --git a/cmd/codeaf/appfactory_actions.go b/cmd/codeaf/appfactory_actions.go index 1edf0a50ea..92fb8da1d4 100644 --- a/cmd/codeaf/appfactory_actions.go +++ b/cmd/codeaf/appfactory_actions.go @@ -469,6 +469,11 @@ func (b *appBackend) FactoryJobCreate(ctx context.Context, p appwire.FactoryJobC return h.acted(ctx, p.Factory, appwire.JobActCreate, a.ID, a), nil } +// FactoryJobPRReady marks the job's draft ready with the caller's credential. +func (b *appBackend) FactoryJobPRReady(ctx context.Context, p appwire.FactoryJobActionParams) (appwire.FactoryJobActed, error) { + return b.jobWrite(ctx, p.Factory, appwire.JobActPRReady, p.Job, "pr-ready", guardBody{p.Expect}) +} + // FactoryJobRunAsNew starts a separate job and answers its new number. func (b *appBackend) FactoryJobRunAsNew(ctx context.Context, p appwire.FactoryJobRunAsNewParams) (appwire.FactoryJobActed, error) { return b.jobWrite(ctx, p.Factory, appwire.JobActRunAsNew, p.Job, "run-as-new", struct { diff --git a/cmd/codeaf/appfactory_job_pr_ready_test.go b/cmd/codeaf/appfactory_job_pr_ready_test.go new file mode 100644 index 0000000000..cbd362341e --- /dev/null +++ b/cmd/codeaf/appfactory_job_pr_ready_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "testing" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" +) + +func TestAppFactoryJobPRReady(t *testing.T) { + f := startActionFactory(t) + alice := signedIn(t, f, 1, "alice") + bob := signedIn(t, f, 2, "bob") + var mu sync.Mutex + calls, status := 0, http.StatusOK + remoteReady, graphRefusal := false, false + f.github.mu.Lock() + f.github.rest = func(r *http.Request) (int, string, bool) { + if r.URL.Path != "/repos/acme/api/pulls/42" && r.URL.Path != "/graphql" { + return 0, "", false + } + if r.Header.Get("Authorization") != "Bearer github-token-secret-1" { + t.Error("ready did not use the caller's credential") + } + if r.Method == http.MethodGet { + mu.Lock() + draft := !remoteReady + mu.Unlock() + return 200, fmt.Sprintf(`{"node_id":"PR_42","state":"open","draft":%t,"head":{"sha":%q}}`, draft, actionHead), true + } + var body struct { + Query string `json:"query"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if !strings.Contains(body.Query, "markPullRequestReadyForReview") { + t.Error("unexpected GitHub mutation") + } + mu.Lock() + defer mu.Unlock() + calls++ + if graphRefusal { + return 200, `{"errors":[{"type":"FORBIDDEN","message":"not permitted"}]}`, true + } + return status, fmt.Sprintf(`{"data":{"markPullRequestReadyForReview":{"pullRequest":{"isDraft":false,"headRefOid":%q}}}}`, actionHead), true + } + f.github.mu.Unlock() + sent := func() int { mu.Lock(); defer mu.Unlock(); return calls } + draft := func() factory.Item { + return factory.Item{Repo: "acme/api", State: factory.StateLanded, Assignee: "alice", Publish: "pr", HeadCommit: actionHead, + PullRequest: &factory.PullRequest{Number: 42, URL: "https://github.com/acme/api/pull/42", State: factory.PROpen, Draft: true, HeadSHA: actionHead, Mergeable: factory.MergeableYes, ChecksRead: true, ReviewsRead: true}} + } + canReady := func(j api.Job) bool { + raw, _ := json.Marshal(j.Can) + var can map[string]bool + _ = json.Unmarshal(raw, &can) + return can["pr_ready"] + } + t.Run("hello", func(t *testing.T) { + if !strings.Contains(alice.w.hello, `"factory.job_pr_ready"`) { + t.Fatal("hello missing factory.job_pr_ready") + } + }) + t.Run("ready immediately removes the draft merge block", func(t *testing.T) { + job := f.create(t, draft()) + before := alice.job(t, job.ID) + if !canReady(before) { + t.Error("draft PR does not offer can.pr_ready") + } + if before.Merge == nil || !strings.HasPrefix(before.Merge.Blocked, "the pull request is a draft") { + t.Fatalf("draft merge gate: %+v", before.Merge) + } + n := sent() + got := alice.act(t, "factory.job_pr_ready", job.ID, nil) + if got.Action != "pr_ready" || got.Current == nil || got.Current.PR == nil || got.Current.PR.Draft || canReady(*got.Current) { + t.Fatalf("ready receipt: %+v", got) + } + after := alice.job(t, job.ID) + if after.PR.Draft || canReady(after) || !after.Can.Merge { + t.Fatalf("ready read: %+v", after) + } + if sent() != n+1 || f.routePOSTs("/pr-ready") != 1 { + t.Fatal("ready must send one typed POST and one GitHub mutation") + } + if f.count("pr_ready") != 1 { + t.Fatal("missing pr_ready record event") + } + alice.refused(t, "factory.job_pr_ready", alice.params(t, job.ID, nil), appwire.CodeConflict) + }) + t.Run("another person is forbidden", func(t *testing.T) { + job := f.create(t, draft()) + n := sent() + if canReady(bob.job(t, job.ID)) { + t.Error("another person is offered ready") + } + bob.refused(t, "factory.job_pr_ready", bob.params(t, job.ID, nil), appwire.CodeForbidden) + if sent() != n { + t.Fatal("forbidden reached GitHub") + } + }) + t.Run("no PR and already ready conflict", func(t *testing.T) { + for _, hasPR := range []bool{false, true} { + it := draft() + if hasPR { + it.PullRequest.Draft = false + } else { + it.PullRequest = nil + } + job := f.create(t, it) + n := sent() + alice.refused(t, "factory.job_pr_ready", alice.params(t, job.ID, nil), appwire.CodeConflict) + if sent() != n { + t.Fatal("conflict reached GitHub") + } + } + }) + t.Run("stale page never reaches GitHub", func(t *testing.T) { + job := f.create(t, draft()) + params := alice.params(t, job.ID, nil) + n := sent() + alice.act(t, "factory.job_set_priority", job.ID, map[string]any{"priority": "urgent"}) + alice.refused(t, "factory.job_pr_ready", params, appwire.CodeStale) + if sent() != n { + t.Fatal("stale reached GitHub") + } + }) + t.Run("already ready on GitHub conflicts without a mutation", func(t *testing.T) { + job := f.create(t, draft()) + n := sent() + mu.Lock() + remoteReady = true + mu.Unlock() + defer func() { mu.Lock(); remoteReady = false; mu.Unlock() }() + alice.refused(t, "factory.job_pr_ready", alice.params(t, job.ID, nil), appwire.CodeConflict) + if sent() != n { + t.Fatal("already ready reached GitHub mutation") + } + }) + t.Run("GraphQL permission refusal is forbidden and never retried", func(t *testing.T) { + job := f.create(t, draft()) + n := sent() + mu.Lock() + graphRefusal = true + mu.Unlock() + defer func() { mu.Lock(); graphRefusal = false; mu.Unlock() }() + alice.refused(t, "factory.job_pr_ready", alice.params(t, job.ID, nil), appwire.CodeForbidden) + if sent() != n+1 { + t.Fatal("GraphQL refusal was retried") + } + }) + t.Run("uncertain is never retried", func(t *testing.T) { + job := f.create(t, draft()) + n := sent() + mu.Lock() + status = http.StatusBadGateway + mu.Unlock() + alice.refused(t, "factory.job_pr_ready", alice.params(t, job.ID, nil), appwire.CodeUncertain) + if sent() != n+1 { + t.Fatal("uncertain ready was retried") + } + if !alice.job(t, job.ID).PR.Draft { + t.Fatal("uncertain claimed success") + } + }) +} diff --git a/docs/changes/unreleased/2238-factory-job-pr-ready.md b/docs/changes/unreleased/2238-factory-job-pr-ready.md new file mode 100644 index 0000000000..e93238a9d1 --- /dev/null +++ b/docs/changes/unreleased/2238-factory-job-pr-ready.md @@ -0,0 +1,13 @@ +--- +kind: added +title: Mark a job pull request ready for review +pr: 2238 +surface: [engine] +invalidates: + - "Was true: a draft PR blocked Merge and required GitHub or the manager to mark it ready. Is true now: the job offers a direct ready-for-review action to the person who may merge it." +--- + +Mark an open draft ready with your own GitHub sign-in. The job immediately +shows the confirmed ready state, without waiting for a poll. Merge keeps its +other checks. Changed jobs and unavailable actions are refused; an uncertain +outcome is never retried automatically. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index cfa498a0e9..6c08f36c44 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -247,6 +247,7 @@ decides which combinations are accepted. | `factory.job_drop` | `in.FactoryJobDropParams` | `FactoryJobActed` | | `factory.job_follow` | `in.FactoryJobFollowParams` | `FactoryJobActed` | | `factory.job_hold` | `in.FactoryJobActionParams` | `FactoryJobActed` | +| `factory.job_pr_ready` | `in.FactoryJobActionParams` | `FactoryJobActed` | | `factory.job_merge` | `in.FactoryJobMergeParams` | `FactoryJobActed` | | `factory.job_reject_plan` | `in.FactoryJobRejectPlanParams` | `FactoryJobActed` | | `factory.job_reopen` | `in.FactoryJobActionParams` | `FactoryJobActed` | @@ -1511,6 +1512,22 @@ when no catalog knows. Under one-model mode `one_model` names the only model. Remote owners answer errands over their own connection when they advertise it; older ones answer `unavailable`. +### Mark a job pull request ready + +Hello feature `factory.job_pr_ready` advertises the optional method of the same +name. Params are `{factory, job, expect}`; `expect.version` is required as for +Merge. It sends one `POST /v1/jobs/{id}/pr-ready` with `{expect}` and returns +`FactoryJobActed` with `action: "pr_ready"`, the record event and the current job. +Job reads offer `can.pr_ready` to a signed-in person allowed to merge this job +while its published PR is open and draft. The job's own GitHub repository and +the caller's credential are used, with no model call or automatic retry. + +A confirmed mutation stores `pr.draft: false` immediately; no follow read is +needed. Merge still applies every other gate. Another person is `forbidden`, +a changed expected version is `stale`, and no PR or an already ready PR is +`conflict`. An unconfirmed outcome is `uncertain`: read the job before deciding +whether to try again. Ready has no Undo window and never merges the PR. + ### Job note receipts and pull request comments A manager's `factory_item` note remains saved on the job after all steps finish. diff --git a/internal/appwire/factory_job_pr_ready.go b/internal/appwire/factory_job_pr_ready.go new file mode 100644 index 0000000000..27f8c398f1 --- /dev/null +++ b/internal/appwire/factory_job_pr_ready.go @@ -0,0 +1,52 @@ +package appwire + +import ( + "context" + "encoding/json" +) + +// FeatureFactoryJobPRReady announces the person's ready-for-review verb. +const FeatureFactoryJobPRReady = "factory.job_pr_ready" + +// JobActPRReady names a successful ready-for-review action. +const JobActPRReady = "pr_ready" + +// FactoryPRReadier marks a job's draft pull request ready once, as the caller. +type FactoryPRReadier interface { + FactoryJobPRReady(context.Context, FactoryJobActionParams) (FactoryJobActed, error) +} + +func withFactoryJobPRReadyFeature(hello Hello, e Engine) Hello { + if _, ok := e.(Factory); !ok { + return hello + } + if _, ok := e.(FactoryPRReadier); ok { + hello = withFeature(hello, FeatureFactoryJobPRReady) + } + return hello +} + +func init() { + methods[FeatureFactoryJobPRReady] = shaped[FactoryJobActionParams, FactoryJobActed](prReady) +} + +func prReady(e Engine, raw json.RawMessage) (*FactoryJobActed, error) { + r, ok := e.(FactoryPRReadier) + if _, isFactory := e.(Factory); !ok || !isFactory { + return nil, Errorf(CodeUnknownMethod, "this engine does not serve %s", FeatureFactoryJobPRReady) + } + var p FactoryJobActionParams + if err := decodeStrict(FeatureFactoryJobPRReady, raw, &p); err != nil { + return nil, err + } + if err := checkJob(&p.Factory, p.Job, p.Expect); err != nil { + return nil, err + } + ctx, cancel := factoryCall() + defer cancel() + out, err := r.FactoryJobPRReady(ctx, p) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/internal/appwire/server.go b/internal/appwire/server.go index 9be52b9c41..88059931f9 100644 --- a/internal/appwire/server.go +++ b/internal/appwire/server.go @@ -230,6 +230,7 @@ func (s *Server) Serve(ctx context.Context, in io.Reader, hello Hello, engine En hello = withFactoryJobStepEditModelFeature(hello, engine) hello = withFactoryIssueCommentFeature(hello, engine) hello = withFactoryJobRejectPlanFeature(hello, engine) + hello = withFactoryJobPRReadyFeature(hello, engine) hello = withFactoryJobBriefSeenFeature(hello, engine) hello = withFactoryJobChatsFeature(hello, engine) hello = withFactoryUsageFeature(hello, engine) diff --git a/internal/factory/api/codec.go b/internal/factory/api/codec.go index 3f4a2048a0..e4fb441bc7 100644 --- a/internal/factory/api/codec.go +++ b/internal/factory/api/codec.go @@ -339,6 +339,7 @@ func CanFor(it factory.Item, caller string, clocks ...time.Time) Can { RetryPublish: a.RetryPublish, Answer: answer, AssignToMe: a.AssignToMe, Reply: a.Reply, SpendAnswer: owner && it.State == factory.StateNeedsYou && it.QKind == factory.QKindSpendStopped, RejectPlan: answer && planSendsBack(it), + PRReady: owner && factory.NormalizeHandle(caller) != "" && it.Kind != factory.KindPR && it.Publish == "pr" && it.DoneHow != "dropped" && it.PullRequest != nil && it.PullRequest.Number > 0 && it.PullRequest.State == factory.PROpen && it.PullRequest.Draft, Merge: owner && factory.NormalizeHandle(caller) != "" && factory.MergeGate(it, factory.MergeAsk{By: caller}).OK, Restore: owner && factory.JobDoneHow(it) == "dropped", } diff --git a/internal/factory/api/verbs.go b/internal/factory/api/verbs.go index a0f8e40043..35df9eb10f 100644 --- a/internal/factory/api/verbs.go +++ b/internal/factory/api/verbs.go @@ -30,6 +30,7 @@ var Verbs = []Verb{ {"retry_publish", nil}, // Merge is the factory's own: its gate and the lease's merge signal. {"merge", nil}, + {"pr-ready", nil}, // A job's model is the factory's own record and the lease's model signal. {"model", nil}, {"request_changes", []string{"SendBack"}}, diff --git a/internal/factory/api/wire.go b/internal/factory/api/wire.go index 195d70f367..38b4f0cf4e 100644 --- a/internal/factory/api/wire.go +++ b/internal/factory/api/wire.go @@ -683,6 +683,8 @@ type Can struct { RejectPlan bool `json:"reject_plan"` // Merge is the merge gate passing for the caller now: Merge would go. Merge bool `json:"merge"` + // PRReady lets the job's person mark its open draft ready for review. + PRReady bool `json:"pr_ready"` // Restore is POST /v1/jobs/{id}/restore (Bring back) for this caller now: // the job was dropped and the caller may act on it. A factory from before // the route never sends it, which reads as false. diff --git a/internal/factory/client/contract_test.go b/internal/factory/client/contract_test.go index 5f88a885cf..36d84a2d95 100644 --- a/internal/factory/client/contract_test.go +++ b/internal/factory/client/contract_test.go @@ -236,7 +236,7 @@ func TestContractCoversEverySeamVerb(t *testing.T) { // same way (internal/factory/serve/merge_test.go). Close and reopen // keep a job's work locally and are the floor's own // (internal/factory/serve/close_test.go). - if !strings.Contains(" take release retry_publish merge model start_now run_again run-as-new stop-after-step spend_answer restore dismiss assign_to_me change_assignee unassign nudge priority raise_budget undo close reopen ", " "+verb.Name+" ") { + if !strings.Contains(" take release retry_publish merge model start_now run_again run-as-new pr-ready stop-after-step spend_answer restore dismiss assign_to_me change_assignee unassign nudge priority raise_budget undo close reopen ", " "+verb.Name+" ") { t.Errorf("new server-owned verb %q needs a contract", verb.Name) } continue diff --git a/internal/factory/github/readiness.go b/internal/factory/github/readiness.go index 6ca0d85782..9ce9191adc 100644 --- a/internal/factory/github/readiness.go +++ b/internal/factory/github/readiness.go @@ -157,3 +157,32 @@ func (s *Source) readyOnce(ctx context.Context, it factory.Item) (factory.PullRe } return after, nil } + +// MarkReady is a person's explicit decision, using the job's own GitHub road. +// Unlike automatic readiness it needs no model review and never retries or +// waits for a follow read: the mutation itself confirms the head and draft. +func (s *Source) MarkReady(ctx context.Context, it factory.Item) (factory.PullRequest, error) { + a, ok := s.api.(interface { + MarkDraftPullReady(context.Context, string, string, int, string) error + }) + if !ok { + return factory.PullRequest{}, factory.ErrReadOnly + } + owner, name, ok := pullRepo(it) + if !ok { + return factory.PullRequest{}, factory.ErrNotOnSource + } + if !s.watching(owner + "/" + name) { + return factory.PullRequest{}, factory.ErrReadOnly + } + pr := it.PullRequest + if pr == nil || pr.Number <= 0 || pr.State != factory.PROpen || !pr.Draft { + return factory.PullRequest{}, fmt.Errorf("this job has no open draft pull request") + } + if err := a.MarkDraftPullReady(ctx, owner, name, pr.Number, pr.HeadSHA); err != nil { + return factory.PullRequest{}, err + } + ready := *pr + ready.Draft = false + return ready, nil +} diff --git a/internal/factory/record/activity.go b/internal/factory/record/activity.go index 8d10994df4..1f63737b38 100644 --- a/internal/factory/record/activity.go +++ b/internal/factory/record/activity.go @@ -41,6 +41,8 @@ func enrich(e *Event) { e.Verb = "published" case RequestChanges: e.Verb = "requested changes" + case PRReady: + e.Verb = "marked ready for review" case PRMerged: e.Verb = "merged" case PRClosed, SourceClosed: diff --git a/internal/factory/record/record.go b/internal/factory/record/record.go index 8fe3d3d112..31c75d4130 100644 --- a/internal/factory/record/record.go +++ b/internal/factory/record/record.go @@ -58,6 +58,8 @@ const ( // PRMerged and PRClosed are the poll noticing a pull request this factory // opened. The handle, when GitHub named one, is who merged it. PRMerged = "pr_merged" + // PRReady records a person marking the job's draft ready for review. + PRReady = "pr_ready" PRClosed = "pr_closed" // Restored is a person bringing a dropped job back to Backlog. Restored = "restored" @@ -200,7 +202,7 @@ func (r *Record) Append(ctx context.Context, event Event) (Event, error) { func validate(e Event) error { switch e.Kind { - case RunAsNew, StopAfterStepChanged, StepEdited, StepStopped, StepRerun, HandOffNotNow, Dismissed, Assigned, Unassigned, Nudged, PriorityChanged, BudgetRaised, BudgetChanged, Undone, Started, Held, Resumed, Stopped, AutoStarted, AskedFirst, Approved, Answered, Posted, Took, Dropped, ClosedLocally, Restored, Released, RequestChanges, PRMerged, PRClosed, SourceClosed, MergeAsked, MergeQueued, MergeRefused, StepFailed, RunnerAdded, RunnerChanged, ReposChanged, LinkCreated, LinkRevoked, SessionsRevoked, OperatorClaimed, GitHubGranted: + case RunAsNew, StopAfterStepChanged, StepEdited, StepStopped, StepRerun, HandOffNotNow, Dismissed, Assigned, Unassigned, Nudged, PriorityChanged, BudgetRaised, BudgetChanged, Undone, Started, Held, Resumed, Stopped, AutoStarted, AskedFirst, Approved, Answered, Posted, Took, Dropped, ClosedLocally, Restored, Released, RequestChanges, PRMerged, PRClosed, SourceClosed, MergeAsked, MergeQueued, MergeRefused, StepFailed, RunnerAdded, RunnerChanged, ReposChanged, LinkCreated, LinkRevoked, SessionsRevoked, OperatorClaimed, GitHubGranted, PRReady: default: return fmt.Errorf("factory record: unknown event kind %q", e.Kind) } @@ -210,7 +212,7 @@ func validate(e Event) error { return errors.New("factory record: actor must be person, agent or factory") } switch e.Verb { - case "ran as new", "edited steps", "reran step", "put away hand-off", "started", "held", "resumed", "stopped", "asked", "approved", "answered", "published", "assigned", "dropped", "brought back", "unassigned", "requested changes", "merged", "closed", "closed locally", "pressed merge", "sent merge", "refused merge", "failed", "added runner", "changed runner", "connected repo", "disconnected repo", "created access link", "revoked access link", "revoked sessions", "dismissed", "nudged", "changed priority", "raised budget", "changed budget", "changed spending stop", "undone", "claimed factory", "granted GitHub read access": + case "ran as new", "edited steps", "reran step", "put away hand-off", "started", "held", "resumed", "stopped", "asked", "approved", "answered", "published", "assigned", "dropped", "brought back", "unassigned", "requested changes", "merged", "closed", "closed locally", "pressed merge", "sent merge", "refused merge", "failed", "added runner", "changed runner", "connected repo", "disconnected repo", "created access link", "revoked access link", "revoked sessions", "dismissed", "nudged", "changed priority", "raised budget", "changed budget", "changed spending stop", "undone", "claimed factory", "granted GitHub read access", "marked ready for review": default: return errors.New("factory record: unknown Activity verb") } diff --git a/internal/factory/serve/pr_ready.go b/internal/factory/serve/pr_ready.go new file mode 100644 index 0000000000..ddf8b88b1a --- /dev/null +++ b/internal/factory/serve/pr_ready.go @@ -0,0 +1,95 @@ +package serve + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + factorygithub "github.com/Agent-Field/codeaf/internal/factory/github" + "github.com/Agent-Field/codeaf/internal/factory/record" + forge "github.com/Agent-Field/codeaf/internal/praf/github" +) + +// prReady is the person's explicit ready-for-review decision. postJob holds +// the consent lock from the guarded read through the confirmed local update. +func (h *Handler) prReady(w http.ResponseWriter, r *http.Request, it factory.Item) { + var body struct { + Expect *api.Expect `json:"expect,omitempty"` + } + if !h.decode(w, r, &body, true) || !h.fresh(w, r, it, "pr-ready", body.Expect) { + return + } + who, err := h.callerHandle(r) + if h.failed(w, err) { + return + } + if !api.CanFor(it, who, h.clock()).PRReady { + h.refuse(w, http.StatusConflict, "this job has no open draft pull request to mark ready", "conflict", "", nil, factory.JobState(it)) + return + } + token, _ := h.personGitHubToken(r) + if token == "" { + h.fail(w, http.StatusForbidden, "Sign in to GitHub before marking this job ready") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + httpClient := h.githubClient() + client := forge.NewClient(token).WithHTTPClient(&httpClient) + source := factorygithub.New(client, []string{factory.RepoOf(it)}, h.clock) + ready, err := source.MarkReady(ctx, it) + if err != nil { + h.prReadyFailed(w, err) + return + } + if h.failed(w, h.config.Store.Update(it.ID, func(cur *factory.Item) error { + // A worker may have refreshed other PR facts while GitHub was answering. + // Only the same pull request and head receive this confirmation. + if cur.PullRequest == nil || cur.PullRequest.Number != ready.Number || cur.PullRequest.HeadSHA != ready.HeadSHA { + return errors.New("the job changed while marking ready; read the job before trying again") + } + pr := *cur.PullRequest + pr.Draft = false + cur.PullRequest = &pr + return nil + })) { + return + } + event := "" + if h.config.Record != nil { + receipt, err := h.config.Record.Append(r.Context(), record.Event{Kind: record.PRReady, Job: it.ID, Handle: who}) + if h.failed(w, err) { + return + } + event = receipt.ID + } + h.write(w, http.StatusOK, struct { + Factory api.Factory `json:"factory"` + Event string `json:"event,omitempty"` + }{h.factory, event}) +} + +func (h *Handler) prReadyFailed(w http.ResponseWriter, err error) { + var refusal *forge.ReadyRefusal + if errors.As(err, &refusal) { + status := http.StatusConflict + if refusal.Permission { + status = http.StatusForbidden + } + h.fail(w, status, refusal.Error()) + return + } + var apiErr *forge.APIError + if errors.As(err, &apiErr) && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 { + if apiErr.StatusCode == http.StatusUnauthorized || apiErr.StatusCode == http.StatusForbidden || apiErr.StatusCode == http.StatusNotFound { + h.fail(w, http.StatusForbidden, "Your GitHub sign-in cannot mark this pull request ready") + return + } + h.fail(w, http.StatusConflict, "GitHub refused to mark this pull request ready; read the job before trying again") + return + } + h.fail(w, http.StatusBadGateway, "GitHub did not confirm ready for review; read the job before trying again") +} diff --git a/internal/factory/serve/repo_off.go b/internal/factory/serve/repo_off.go index 63b728f7c5..c3ffaf3362 100644 --- a/internal/factory/serve/repo_off.go +++ b/internal/factory/serve/repo_off.go @@ -26,7 +26,7 @@ import ( var offVerbs = map[string]bool{ "run": true, "start_now": true, "run_again": true, "run-as-new": true, "steps/{sid}/rerun": true, "resume": true, "approve": true, "retry_publish": true, "merge": true, "request_changes": true, "recheck": true, - "sync": true, "issue_comment": true, + "sync": true, "issue_comment": true, "pr-ready": true, } // repoSwitch is the switch the store keeps. A factory with no store of its @@ -82,6 +82,6 @@ func offJob(j *api.Job) { j.RepoOff = true j.Can.RunAsNew = false j.Can.Run, j.Can.StartNow, j.Can.RunAgain, j.Can.Resume = false, false, false, false - j.Can.Approve, j.Can.RetryPublish, j.Can.Merge = false, false, false + j.Can.Approve, j.Can.RetryPublish, j.Can.Merge, j.Can.PRReady = false, false, false, false j.Controls.RerunStep, j.Controls.Sync = false, false } diff --git a/internal/factory/serve/verbs.go b/internal/factory/serve/verbs.go index 2150668159..4e00e7f0f3 100644 --- a/internal/factory/serve/verbs.go +++ b/internal/factory/serve/verbs.go @@ -31,6 +31,7 @@ func (h *Handler) verbs() { h.postJob("approve", true, h.approve) h.postJob("retry_publish", h.config.Store != nil && h.seam.SignOff != nil, h.guarded("retry_publish", h.onlyOwner(h.retryPublish))) h.postJob("merge", h.config.Store != nil, h.onlyOwner(h.mergeJob)) + h.postJob("pr-ready", h.config.Store != nil, h.onlyOwner(h.prReady)) h.postJob("request_changes", s.SendBack != nil, h.recordedWords(s.SendBack, record.RequestChanges)) h.postJob("recheck", s.Reverify != nil, h.guarded("recheck", h.idVerb(s.Reverify))) // Take and release are the lease desk's own doors, not seam fields. The @@ -48,7 +49,7 @@ type jobAction func(http.ResponseWriter, *http.Request, factory.Item) func (h *Handler) postJob(verb string, exists bool, action jobAction) { h.route("/v1/jobs/{id}/"+verb, map[string]http.HandlerFunc{ http.MethodPost: func(w http.ResponseWriter, r *http.Request) { - if verb == "sync" || verb == "approve" || verb == "retry_publish" || verb == "merge" || verb == "answer" || verb == "spend_answer" || verb == "reject_plan" || verb == "undo" || verb == "assign_to_me" || verb == "change_assignee" || verb == "assignee" || verb == "unassign" { + if verb == "sync" || verb == "approve" || verb == "retry_publish" || verb == "merge" || verb == "pr-ready" || verb == "answer" || verb == "spend_answer" || verb == "reject_plan" || verb == "undo" || verb == "assign_to_me" || verb == "change_assignee" || verb == "assignee" || verb == "unassign" { unlock, err := h.lockConsent() if h.failed(w, err) { return diff --git a/internal/manual/chat/factory-stage-conversations.md b/internal/manual/chat/factory-stage-conversations.md index 174a8594a6..c6b2c05516 100644 --- a/internal/manual/chat/factory-stage-conversations.md +++ b/internal/manual/chat/factory-stage-conversations.md @@ -923,3 +923,26 @@ The choice adds no dollar limit and does not interrupt a model call already under way. A leased worker checks the saved choice before each step and landing. If that read fails, it keeps the last saved choice, including an armed hold, and tries again at the next boundary. The read failure does not fail the job. + +## Follow PR — comments that need no code change + +A comment can be considered without producing a new diff. Follow PR reads the +current feedback against the published head and records why no edit is needed. +A successful assessment returns to monitoring; Merge then uses its ordinary +checks, review and approved-head gates. Outstanding requested changes and failing +or pending CI still block Merge. + +A no-change report must include `repair.readiness` with the full current head and +the diff, description, feedback and unfinished-work checks. Missing or incomplete +fields are returned to the model during `stage_result`, so it can correct the +report in the same conversation. Prose alone does not complete the assessment. +The runner still checks that the published head and clean checkout agree. + +If a no-change report was refused against the same PR head, comments and check +state, **Try again** returns without starting a paid round. It says to wait for +a new push or comment, or stop the job. New PR facts permit a fresh attempt. +Attempt exhaustion alone never opens Merge or acknowledges a requested change. + +When the refusal concerns local checkout changes, Try again first rereads the +checkout without calling a model. An unchanged checkout remains refused without +spending; restoring the clean published checkout permits a fresh assessment. diff --git a/internal/manual/chat/factory.md b/internal/manual/chat/factory.md index 7674308e63..cbf6a296e5 100644 --- a/internal/manual/chat/factory.md +++ b/internal/manual/chat/factory.md @@ -281,6 +281,10 @@ The `factory_item` card's thinking (`think strong`, `think cheap`, `default` for After your yes, the answer names the thinking the item was saved with, step by step (`thinking write strong, test strong`; `default` resets the changed steps to usual), and the steps that did not take it: `left as they were: plan cheap`. A budget or note alone says nothing about thinking. If another field changes but no step needs the requested thinking, the answer says `thinking unchanged`. A later turn's opening note lists thinking per running step when they differ (`thinking plan cheap, write strong`), one word when they agree, and nothing when none was set. A level other than cheap, strong or default is refused before any card, as `Invalid arguments: effort is one of cheap, strong, default, or left out.` +## which model a cheap or strong step runs on — small work, checks model, effort models + +A factory step marked `cheap` opens on the **small work** model, and a step marked `strong` on the **checks** model: the same two rows the settings sheet already shows, so changing either changes the factory's cheap or strong steps from their next run. A model picked for the step, or for the whole job, wins over its effort. A step with no effort word runs on the job's model or the factory's default. With one model for every task, every step runs on that one model, whatever its effort. + ## answer the factory_item card in the terminal — yes, keep it, say what to change, esc, the card never appeared In the terminal chat a `factory_item` call puts its card above the message box: a head that asks the question (`#12 · think strong?`), one row for each thing that would change (`thinking write — → strong`, `budget $5 → $8`, `note:`, `why:`), and two answers. @@ -324,6 +328,20 @@ changed code, CI or feedback cannot reuse stale readiness approval. Marking read never grants merge authority. The repository's separate auto-merge setting is the explicit opt-in for automatic merge. +## mark a draft pull request ready for review + +A job's assignee can mark its open draft pull request ready from the job, +using their own GitHub sign-in. A job with no assignee follows the same person +rule as Merge. The PR immediately reads ready after GitHub confirms it; there +is no wait for the next poll. This is an explicit human decision, with no +model review, automatic retry or Undo window. + +A PR that is already ready, closed, or missing cannot be marked ready. If the +job changed since you read it, read it again first. If the outcome is uncertain, +check the PR before trying again. Marking ready removes the draft block from +Merge; checks, approved commits, reviews and every other merge rule still apply. +It does not merge the PR or enable auto-merge. + ## merge a job's pull request — factory auto-merge when checks pass, or why it will not merge A job whose pull request is open has **Merge**. Only its assignee may press it, and pressing waits five seconds with **Undo** before anything is sent. Then the runner that holds the assignee's GitHub account reads the pull request again and merges it, handing GitHub the exact commit, so a branch that moved is refused. The job is done when the pull request reads merged. diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index 6cb3254c3c..16b413faf4 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -1502,9 +1502,12 @@ the copy built into the binary. Your runs can improve the shared measurements. N the machine — what is shared is a measurement of the run, not the work. One setting answers for all of it, `model_pool` in `/settings`, with three values: `on` reads and sends, `read` uses the pool and sends nothing, `off` -does neither. It defaults to `on`. `CODEAF_MODEL_POOL` pins the same word -from the shell, and on a CI machine with neither set codeaf reads but does -not send. **The telemetry off switch stops the pool sending too**: +does neither. It defaults to `off`, including in CI: no pool judge calls, +pool routing or pool records. Each role still follows your task-model and +tier settings, and in Auto the router picks from your allowed models. +`CODEAF_MODEL_POOL=on` or a saved `model_pool = on` enables it; an earlier +explicit choice stays honoured. A missing or blank saved word means you never +chose and stays off. `CODEAF_MODEL_POOL` pins the same word from the shell. **The telemetry off switch stops the pool sending too**: `CODEAF_TELEMETRY=off`, `DO_NOT_TRACK=1` or the `telemetry` switch in `/settings` caps the pool at `read` — it wins over an explicit `on` — and `codeaf pool status` then says `mode read · telemetry`. @@ -1515,7 +1518,7 @@ codeaf pool [show|status|verify] [--json] [--key key] `show` — also what bare `codeaf pool` prints — is the reading form: the mode and the addresses in force with the word saying where each came from -(`default`, `setting`, `env`, `ci`, or `telemetry` when the telemetry off switch capped +(`default`, `setting`, `env`, or `telemetry` when the telemetry off switch capped sending), then what index is cached, how old it is and how many cells it holds, or `no index cached yet · built-in seed of `. The binary carries a seed index of scored runs, shown until a fresher diff --git a/internal/manual/chat/settings.md b/internal/manual/chat/settings.md index 265b534323..8ca259696b 100644 --- a/internal/manual/chat/settings.md +++ b/internal/manual/chat/settings.md @@ -160,7 +160,11 @@ renames this device with `n`, removes another with `r`, and starts `/pair` from **Privacy** separates input history, unfinished drafts, anonymous usage reporting and **shared model recommendations** (formerly Model Pool). Recommendations can -**use and contribute**, **use only**, or remain **off**. Disabling one privacy +**use and contribute**, **use only**, or remain **off** (the default). +With no saved choice, the pool makes no judge calls and writes no pool records; +each role still follows your task-model and tier settings, and in Auto the +router picks from your allowed models. A saved **use and contribute** choice +stays enabled. Disabling one privacy control is not a promise that all the others are disabled. Search accepts old labels as well as the new organization. diff --git a/internal/praf/github/forge.go b/internal/praf/github/forge.go index 81f506d697..f70f199de5 100644 --- a/internal/praf/github/forge.go +++ b/internal/praf/github/forge.go @@ -811,3 +811,9 @@ func (c *client) WithSleep(sleep func(context.Context, time.Duration) error) *cl c.sleepFn = sleep return c } + +// WithHTTPClient uses the supplied transport for this client's GitHub requests. +func (c *client) WithHTTPClient(client *http.Client) *client { + c.httpClient = client + return c +} diff --git a/internal/praf/github/readiness.go b/internal/praf/github/readiness.go index e991c5b031..72904283da 100644 --- a/internal/praf/github/readiness.go +++ b/internal/praf/github/readiness.go @@ -128,6 +128,25 @@ func (c *client) ReadinessMaterial(ctx context.Context, owner, repo, base string // The caller rechecks the complete readiness snapshot first and observes the // result afterward. GitHub supplies no expected-head field for this mutation. func (c *client) MarkPullReady(ctx context.Context, owner, repo string, n int, head string) error { + return c.markPullReady(ctx, owner, repo, n, head, false) +} + +// ReadyRefusal is a definitive refusal before any ready transition was confirmed. +// Permission distinguishes a caller's lack of access from a conflicting PR state. +type ReadyRefusal struct { + Permission bool + Message string +} + +func (e *ReadyRefusal) Error() string { return e.Message } + +// MarkDraftPullReady is the person's explicit action: an already ready PR is +// a conflict, rather than the idempotent success of automatic publication. +func (c *client) MarkDraftPullReady(ctx context.Context, owner, repo string, n int, head string) error { + return c.markPullReady(ctx, owner, repo, n, head, true) +} + +func (c *client) markPullReady(ctx context.Context, owner, repo string, n int, head string, requireDraft bool) error { raw, _, err := c.read(ctx, c.repoURL(owner, repo, "/pulls/"+strconv.Itoa(n)), "") if err != nil { return err @@ -144,9 +163,12 @@ func (c *client) MarkPullReady(ctx context.Context, owner, repo string, n int, h return err } if p.State != "open" || p.Head.SHA != head || p.Draft == nil { - return fmt.Errorf("PR closed, head moved or draft state is unknown before marking ready") + return &ReadyRefusal{Message: "PR closed, head moved or draft state is unknown before marking ready"} } if !*p.Draft { + if requireDraft { + return &ReadyRefusal{Message: "the pull request is already ready for review"} + } return nil } if p.NodeID == "" { @@ -164,6 +186,7 @@ func (c *client) MarkPullReady(ctx context.Context, owner, repo string, n int, h var v struct { Errors []struct { Message string `json:"message"` + Type string `json:"type"` } `json:"errors"` Data struct { Ready struct { @@ -177,6 +200,13 @@ func (c *client) MarkPullReady(ctx context.Context, owner, repo string, n int, h if err = json.Unmarshal(raw, &v); err != nil { return err } + if len(v.Errors) > 0 && v.Data.Ready.PR == nil { + permission := false + for _, failure := range v.Errors { + permission = permission || failure.Type == "FORBIDDEN" || failure.Type == "UNAUTHORIZED" + } + return &ReadyRefusal{Permission: permission, Message: "GitHub refused ready for review: " + v.Errors[0].Message} + } if len(v.Errors) > 0 { return fmt.Errorf("GitHub refused ready for review: %s", v.Errors[0].Message) } From 85679594d232fa6fcf911cded405de4305f1275b Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:50:48 -0500 Subject: [PATCH 03/28] factory: cheap and strong steps run on the small work and checks models (factory.effort_models) (#2148) * factory: a cheap step runs on the small work model, a strong one on the checks model A step's effort word now goes with its conversation, and with no model of the step's or the job's the step opens on the model its effort runs on: models.tiers.low for cheap, models.tiers.high for strong. One model for every task keeps its one model. The app announces factory.effort_models. * changes: name PR 2148 * factory: effort tiers keyed by the one list of effort words * factory: live check of a cheap step on the small work model * factory: a step's effort word comes before the factory's default model Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/factory_effort_live_e2e_test.go | 148 ++++++++++++++++++ cmd/codeaf/factory_effort_model.go | 46 ++++++ cmd/codeaf/factory_effort_model_test.go | 68 ++++++++ cmd/codeaf/factory_stage.go | 2 +- .../unreleased/2148-factory-effort-models.md | 11 ++ internal/appwire/factory_effort_models.go | 25 +++ .../appwire/factory_effort_models_test.go | 24 +++ internal/appwire/server.go | 1 + internal/factory/run/chatstage.go | 5 + internal/factory/run/chatstage_model_test.go | 16 ++ 10 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 cmd/codeaf/factory_effort_live_e2e_test.go create mode 100644 cmd/codeaf/factory_effort_model.go create mode 100644 cmd/codeaf/factory_effort_model_test.go create mode 100644 docs/changes/unreleased/2148-factory-effort-models.md create mode 100644 internal/appwire/factory_effort_models.go create mode 100644 internal/appwire/factory_effort_models_test.go diff --git a/cmd/codeaf/factory_effort_live_e2e_test.go b/cmd/codeaf/factory_effort_live_e2e_test.go new file mode 100644 index 0000000000..90f9477e44 --- /dev/null +++ b/cmd/codeaf/factory_effort_live_e2e_test.go @@ -0,0 +1,148 @@ +//go:build e2e + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/crewroute" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/session" +) + +// THE LIVE CHEAP STEP. The small work model is pinned through the app's own +// settings.models.set (every role on the one live model, [liveRecorder] +// refusing any other), the launch is in auto mode, and one real factory job +// runs a step marked cheap to its approve gate. The step opens on the small +// work model the pin names, and no other model is called. +func TestLiveFactoryCheapStepRunsOnTheSmallWorkModel(t *testing.T) { + if os.Getenv(liveFactoryOptIn) != "1" { + t.Skipf("opt-in live lane: set %s=1 (spends a few cents on OpenRouter)", liveFactoryOptIn) + } + if liveFactoryKey == "" { + t.Fatal("BLOCKED: no OpenRouter key by any road the product reads; not faking a pass") + } + root := resolvedTempDir(t) + t.Setenv("HOME", filepath.Join(root, "home")) + t.Setenv(home.EnvVar, filepath.Join(root, "state")) + t.Setenv(config.ProfileDirEnv, "") + // One profile for the launch and the factory's steps (home.Join). + profile := filepath.Join(root, "state") + t.Setenv("CODEAF_PROFILE_DIR", profile) + t.Setenv("OPENROUTER_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "") + t.Setenv("GH_TOKEN", "") + t.Setenv("GITHUB_TOKEN", "") + t.Setenv(factoryPostureEnv, "") + keys := config.NewAppCredentials() + t.Cleanup(config.UseAppCredentials(keys)) + keys.Set(modelsource.DefaultID, liveFactoryKey) + recorder := newLiveRecorder(t, config.DefaultBaseURL) + t.Setenv("CODEAF_BASE_URL", recorder.server.URL) + + g := newRunRig(t) + proc, err := openV3Process("chat") + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.closeAll) + // The pins are written where settings.models.set writes them; the wire's + // own check asks a live catalog this rig does not serve. + // The router's catalog holds the live model only, so a seat nobody pinned + // routes to it too and every role stays on the one model. + seatCrewCatalog(t, []catalog.Model{{ID: liveFactoryModel, PromptPrice: 0.1e-6, CompletionPrice: 0.4e-6, + CodingIndex: 50, AgenticIndex: 50, IntelligenceIndex: 50, Parameters: []string{"tools"}, ContextLength: 200000, OpenWeights: true}}) + for _, seat := range []crewroute.Seat{crewroute.Worker, crewroute.Planner, crewroute.Checker} { + if err := config.SetCrewPin(profile, seat, liveFactoryModel); err != nil { + t.Fatal(err) + } + } + writeExternalCrewSettings(t, config.BudgetConfigPath(profile), map[string]string{ + config.KeyTierLowModel: liveFactoryModel, config.KeyTierReflexModel: liveFactoryModel, config.KeyTierMastermindModel: liveFactoryModel, + }) + w := newFactoryWire(t, testFactoryHub(t)) + launch, err := openV3Launch(proc, v3Options{Model: liveFactoryModel, Workspace: g.workspace}) + if err != nil { + t.Fatal(err) + } + if launch.Config.OneModel { + t.Fatal("the launch is on one model; a cheap step would keep it") + } + if home.Join() != profile { + t.Fatalf("the factory's profile %q is not the launch's %q", home.Join(), profile) + } + if got := factoryStepModel(profile, factoryrun.ConversationSpec{StepEffort: "cheap"}, false); got != liveFactoryModel { + t.Fatalf("a cheap step would open on %q, want the small work pin %q", got, liveFactoryModel) + } + stageSettle = 300 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + + local := &w.backend.local + local.githubHTTP = (&refusingGitHub{}).client() + local.rules = func(context.Context, string, string) (pipeline.File, error) { return pipeline.File{}, os.ErrNotExist } + local.parent = func() session.Config { return launch.Config } + local.seamHook = func(seam factory.Seam) factory.Seam { + seam.GitHub = func(context.Context) (factory.GitHubLink, error) { return factory.GitHubLink{Login: "owner"}, nil } + return seam + } + var st appwire.FactoryLocalStatus + w.must("factory.local_start", map[string]any{"caller": "live"}, &st) + floor := local.host.floor + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + id, err := floor.Add(context.Background(), factory.Item{ + Title: "write the greeting", Repo: "acme/api", Tier: factory.TierOwner, Cap: liveFactoryBudget, + Stages: []factory.Stage{ + {Name: "write", Kind: factory.StageChat, On: true, Until: "done", Effort: "cheap", + Ask: "Create the file HELLO.txt in the current directory with the write tool, its content exactly `hello factory` and a trailing newline. Then call stage_result with done true, findings 0 and a one-line output. Do nothing else."}, + {Name: factory.ApproveName, Kind: factory.StageGate, On: true}, + }, + }) + if err != nil { + t.Fatal(err) + } + var job api.Job + w.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + if !st.Runner { + var run appwire.FactoryLocalStatus + w.must("factory.local_run", map[string]any{"caller": "live"}, &run) + } + var acted appwire.FactoryJobActed + w.must("factory.job_start", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + ctx, cancel := context.WithTimeout(context.Background(), liveFactoryWall) + defer cancel() + var it factory.Item + for { + it, _ = floor.Get(id) + if it.State == factory.StateLanded || it.Failure != nil || it.Question != "" || ctx.Err() != nil { + break + } + time.Sleep(250 * time.Millisecond) + } + if it.State != factory.StateLanded { + recorder.mu.Lock() + t.Logf("recorder: requested %v answered %v calls %d problems %v", recorder.requests, recorder.replies, recorder.calls, recorder.problems) + recorder.mu.Unlock() + t.Fatalf("job is %s (failure %+v, question %q)", it.State, it.Failure, it.Question) + } + recorder.mu.Lock() + defer recorder.mu.Unlock() + t.Logf("live cheap step: job %d landed at its approve gate · small work pin %s · recorder: requested %v answered %v calls %d", + id, liveFactoryModel, recorder.requests, recorder.replies, recorder.calls) + if len(recorder.problems) > 0 || recorder.calls == 0 || recorder.requests[liveFactoryModel] != recorder.calls || len(recorder.replies) != 1 || recorder.replies[liveFactoryModel] == 0 { + t.Fatalf("model policy: problems %v requested %v answered %v", recorder.problems, recorder.requests, recorder.replies) + } +} diff --git a/cmd/codeaf/factory_effort_model.go b/cmd/codeaf/factory_effort_model.go new file mode 100644 index 0000000000..82ddef4707 --- /dev/null +++ b/cmd/codeaf/factory_effort_model.go @@ -0,0 +1,46 @@ +package main + +import ( + "strings" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "github.com/Agent-Field/codeaf/internal/session" +) + +// factoryEffortTiers is the model tier each step effort word runs on: a cheap +// step on the small-work model (models.tiers.low), a strong step on the +// checks model (models.tiers.high). They are the rows Settings already +// shows and sets, so the app reads and changes them where it does today. +// The words come from factory.EffortWords, cheap first. +var factoryEffortTiers = map[string]string{ + factory.EffortWords[0]: config.ModelTierLow, + factory.EffortWords[1]: config.ModelTierHigh, +} + +// factoryStepModel is the model a step's conversation opens on, "" to keep +// the launch's default: the step's or the job's own pick first, then the +// model its effort runs on. A profile on one model for every task keeps it. +func factoryStepModel(profile string, spec factoryrun.ConversationSpec, oneModel bool) string { + if model := strings.TrimSpace(spec.Model); model != "" { + return model + } + tier, ok := factoryEffortTiers[strings.TrimSpace(spec.StepEffort)] + if !ok || oneModel { + return "" + } + return strings.TrimSpace(config.TierSeatAt(profile, tier).Model) +} + +// factoryOpenModel is the model a step opens on: the one a person picked for the +// step or the job, else the one its effort word runs on, else the factory's +// default, else the launch's. The step's word is the more specific of the last +// two. A launch on one model for every task keeps that model unless a person +// picked one. +func factoryOpenModel(profile string, cfg session.Config, spec factoryrun.ConversationSpec) string { + if model := factoryStepModel(profile, spec, cfg.OneModel); model != "" { + return model + } + return stageOpenModel(cfg, spec) +} diff --git a/cmd/codeaf/factory_effort_model_test.go b/cmd/codeaf/factory_effort_model_test.go new file mode 100644 index 0000000000..f38948c61e --- /dev/null +++ b/cmd/codeaf/factory_effort_model_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/config" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "github.com/Agent-Field/codeaf/internal/session" +) + +// A CHEAP STEP OPENS ON THE SMALL-WORK MODEL AND A STRONG STEP ON THE CHECKS +// MODEL, the two rows Settings already keeps (models.tiers.low and +// models.tiers.high); a model the person picked for the step or the job wins, +// and a profile on one model for everything keeps its one model. +func TestAStepsEffortChoosesItsModel(t *testing.T) { + _, _, _ = useLocalHome(t) + profile := config.ProfileDir() + if err := applyRow(profile, config.KeyTierLowModel, "vendor/small"); err != nil { + t.Fatal(err) + } + if err := applyRow(profile, config.KeyTierHighModel, "vendor/strong"); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + spec factoryrun.ConversationSpec + one bool + want string + }{ + {factoryrun.ConversationSpec{StepEffort: "cheap"}, false, "vendor/small"}, + {factoryrun.ConversationSpec{StepEffort: "strong"}, false, "vendor/strong"}, + {factoryrun.ConversationSpec{}, false, ""}, + {factoryrun.ConversationSpec{StepEffort: "cheap", Model: " vendor/picked "}, false, "vendor/picked"}, + {factoryrun.ConversationSpec{StepEffort: "strong"}, true, ""}, + } { + if got := factoryStepModel(profile, c.spec, c.one); got != c.want { + t.Errorf("%+v one-model %v: %q, want %q", c.spec, c.one, got, c.want) + } + } +} + +// THE STEP'S EFFORT WORD BEATS THE FACTORY'S DEFAULT MODEL: a step that says it +// is cheap work runs on the small-work model on a factory with a default model. +// An effort row with no model leaves the factory's default, and a model picked +// for the step or the job and a launch on one model keep their place above both. +func TestAStepsEffortComesBeforeTheFactorysDefaultModel(t *testing.T) { + _, _, _ = useLocalHome(t) + profile := config.ProfileDir() + if err := applyRow(profile, config.KeyTierLowModel, "vendor/small"); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name string + spec factoryrun.ConversationSpec + one bool + want string + }{ + {"effort over default", factoryrun.ConversationSpec{StepEffort: "cheap", FactoryModel: "vendor/house"}, false, "vendor/small"}, + {"effort row with no model", factoryrun.ConversationSpec{StepEffort: "strong", FactoryModel: "vendor/house"}, false, "vendor/house"}, + {"no effort", factoryrun.ConversationSpec{FactoryModel: "vendor/house"}, false, "vendor/house"}, + {"picked over effort", factoryrun.ConversationSpec{StepEffort: "cheap", FactoryModel: "vendor/house", Model: "vendor/picked"}, false, "vendor/picked"}, + {"one model", factoryrun.ConversationSpec{StepEffort: "cheap", FactoryModel: "vendor/house"}, true, "vendor/launch"}, + } { + cfg := session.Config{Model: "vendor/launch", OneModel: c.one} + if got := factoryOpenModel(profile, cfg, c.spec); got != c.want { + t.Errorf("%s: %q, want %q", c.name, got, c.want) + } + } +} diff --git a/cmd/codeaf/factory_stage.go b/cmd/codeaf/factory_stage.go index fd010d910b..2f475d3eb3 100644 --- a/cmd/codeaf/factory_stage.go +++ b/cmd/codeaf/factory_stage.go @@ -128,7 +128,7 @@ func (m stageMaker) Open(ctx context.Context, spec factoryrun.ConversationSpec) } cfg.Workspace = where cfg.AutomaticModels = spec.AutomaticModels - cfg.Model = stageOpenModel(cfg, spec) + cfg.Model = factoryOpenModel(m.profileDir, cfg, spec) fallback := appDefaultFallback(m.profileDir) if cfg.OneModel || fallback == cfg.Model { fallback = "" diff --git a/docs/changes/unreleased/2148-factory-effort-models.md b/docs/changes/unreleased/2148-factory-effort-models.md new file mode 100644 index 0000000000..133932e33c --- /dev/null +++ b/docs/changes/unreleased/2148-factory-effort-models.md @@ -0,0 +1,11 @@ +--- +kind: changed +title: A cheap factory step runs on the small work model and a strong one on the checks model +pr: 2148 +surface: [engine, chat, docs] +invalidates: + - "Was true: a factory step's effort (`cheap`, `strong`) reached the step only as a caption in its brief (`thinking cheap`); every step without its own model ran on the job's model or the factory's default. Is true now: a step marked cheap opens on the small work model (models.tiers.low, the task models' auxiliary pin) and a step marked strong on the checks model (models.tiers.high, the checker pin), unless the step or the job has its own model; with one model for every task nothing changes. The app announces it with the hello word `factory.effort_models` and reads and sets the two models through settings.models.get and settings.models.set, as it already does." +--- + +Spending's Models rows can say which model cheap and strong factory steps +run on, and change them where Settings already does. diff --git a/internal/appwire/factory_effort_models.go b/internal/appwire/factory_effort_models.go new file mode 100644 index 0000000000..459c38dcec --- /dev/null +++ b/internal/appwire/factory_effort_models.go @@ -0,0 +1,25 @@ +package appwire + +// ── factory.effort_models: which model a cheap or strong step runs on ────── +// +// No method of its own. A factory step marked cheap opens on the small-work +// model (models.tiers.low, the task models' auxiliary pin) and one marked +// strong on the checks model (models.tiers.high, the checker pin), unless the +// step or the job has its own model; on one model for every task, every step +// keeps that one. The app reads both through settings.models.get +// (roles.auxiliary.pin, roles.checker.pin; null is the build's or the +// router's own choice) and sets them through settings.models.set +// (pins.auxiliary, pins.checker), the doors that already change them. + +// FeatureFactoryEffortModels is announced in hello by an engine whose factory +// steps run their effort on those two models and whose settings serve them. +const FeatureFactoryEffortModels = "factory.effort_models" + +func withFactoryEffortModelsFeature(hello Hello, e Engine) Hello { + _, factory := e.(Factory) + _, models := e.(TaskModelsLister) + if factory && models { + hello = withFeature(hello, FeatureFactoryEffortModels) + } + return hello +} diff --git a/internal/appwire/factory_effort_models_test.go b/internal/appwire/factory_effort_models_test.go new file mode 100644 index 0000000000..8b806496bb --- /dev/null +++ b/internal/appwire/factory_effort_models_test.go @@ -0,0 +1,24 @@ +package appwire + +import "testing" + +// factoryTaskModelsEngine is a factory engine whose settings serve the task +// models. +type factoryTaskModelsEngine struct { + factoryEngine + taskModelsEngine +} + +// Only an engine that serves both a factory and the task models announces +// that a factory step's effort runs on the small-work and checks models. +func TestFactoryEffortModelsIsAnnouncedByAFactoryWithTaskModels(t *testing.T) { + for _, e := range []Engine{&fakeEngine{}, &factoryEngine{}, &taskModelsEngine{}} { + if h := withFactoryEffortModelsFeature(Hello{}, e); len(h.Features) != 0 { + t.Errorf("%T announced %v", e, h.Features) + } + } + h := withFactoryEffortModelsFeature(Hello{}, &factoryTaskModelsEngine{}) + if len(h.Features) != 1 || h.Features[0] != FeatureFactoryEffortModels || FeatureFactoryEffortModels != "factory.effort_models" { + t.Fatalf("announced %v", h.Features) + } +} diff --git a/internal/appwire/server.go b/internal/appwire/server.go index 88059931f9..ff641252c6 100644 --- a/internal/appwire/server.go +++ b/internal/appwire/server.go @@ -233,6 +233,7 @@ func (s *Server) Serve(ctx context.Context, in io.Reader, hello Hello, engine En hello = withFactoryJobPRReadyFeature(hello, engine) hello = withFactoryJobBriefSeenFeature(hello, engine) hello = withFactoryJobChatsFeature(hello, engine) + hello = withFactoryEffortModelsFeature(hello, engine) hello = withFactoryUsageFeature(hello, engine) hello = withFactoryEstimateFeature(hello, engine) hello = withFactoryScoutFeature(hello, engine) diff --git a/internal/factory/run/chatstage.go b/internal/factory/run/chatstage.go index 6779f0f397..e9df36fb86 100644 --- a/internal/factory/run/chatstage.go +++ b/internal/factory/run/chatstage.go @@ -91,6 +91,10 @@ type ConversationSpec struct { // is "" and its launch is not pinned to one model for every task. FactoryModel string Effort string + // StepEffort is the step's own effort word (cheap, strong or ""): with + // no model of the step's or the job's, the maker opens the step on the + // model that effort runs on, before the factory's default. + StepEffort string // Resume is the transcript of a conversation to reopen instead of making // one, "" for a new one. The maker then continues it with Brief as a // carry-on note (internal/session's CarryOn), joins no team (it is a @@ -268,6 +272,7 @@ func (e chatExecutor) Run(ctx context.Context, job Job) (factory.StageResult, er AutomaticModels: stageAutomatic(job), FactoryModel: strings.TrimSpace(job.Item.FactoryModel), Effort: job.Item.Effort, + StepEffort: job.Stage.Effort, Stage: door, VerificationStage: job.Stage.Name, VerifyChecks: checksVerification(job.Stage), diff --git a/internal/factory/run/chatstage_model_test.go b/internal/factory/run/chatstage_model_test.go index cc5b0658c4..1c90352899 100644 --- a/internal/factory/run/chatstage_model_test.go +++ b/internal/factory/run/chatstage_model_test.go @@ -26,3 +26,19 @@ func TestAStepWithItsOwnModelOpensOnIt(t *testing.T) { t.Fatalf("opened on %q, automatic %v", spec.Model, spec.AutomaticModels) } } + +// A STEP'S EFFORT WORD GOES WITH ITS CONVERSATION, so the maker can open a +// cheap or strong step on the model that effort runs on. +func TestAStepsEffortWordReachesItsConversation(t *testing.T) { + maker := &FakeMaker{Script: func(ctx context.Context, c *FakeConversation) { + _ = c.Spec.Stage.Report(ctx, factory.StageResult{Done: true}) + }} + job := reviewJob() + job.Stage.Effort = "cheap" + if _, err := NewChatExecutor(maker).Run(context.Background(), job); err != nil { + t.Fatal(err) + } + if got := maker.Opened()[0].Spec.StepEffort; got != "cheap" { + t.Fatalf("step effort %q", got) + } +} From a3357f3077828f8182a8f61d987c81c8564e379d Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:51:02 -0500 Subject: [PATCH 04/28] factory: Backlog jobs say which protected paths they name (factory.job_protected) (#2145) * factory: a Backlog job says which protected paths it names The queue judges every Backlog job against the rules it would start under, read once per repository and show-only, and carries the protected paths its issue, comments, plan or files name. Absent is not judged. The app announces factory.job_protected. * changes: name PR 2145 * factory: live check of the protected key against a real repository * changes: name PR 2145 in the entry * factory: a cheap step runs on the small work model, a strong one on the checks model A step's effort word now goes with its conversation, and with no model of the step's or the job's the step opens on the model its effort runs on: models.tiers.low for cheap, models.tiers.high for strong. One model for every task keeps its one model. The app announces factory.effort_models. * changes: name PR 2148 * factory: judge protected paths only for jobs the canvas shows in Backlog * factory: effort tiers keyed by the one list of effort words * factory: live check of a cheap step on the small work model * factory: a step's effort word comes before the factory's default model Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../appfactory_protected_live_e2e_test.go | 100 ++++++++++++++++++ .../unreleased/2145-factory-job-protected.md | 11 ++ internal/appwire/factory_job_protected.go | 21 ++++ .../appwire/factory_job_protected_test.go | 14 +++ internal/appwire/server.go | 1 + internal/factory/api/wire.go | 14 +++ internal/factory/pipeline/house.go | 14 +++ internal/factory/serve/protected.go | 61 +++++++++++ internal/factory/serve/protected_test.go | 75 +++++++++++++ internal/factory/serve/serve.go | 1 + 10 files changed, 312 insertions(+) create mode 100644 cmd/codeaf/appfactory_protected_live_e2e_test.go create mode 100644 docs/changes/unreleased/2145-factory-job-protected.md create mode 100644 internal/appwire/factory_job_protected.go create mode 100644 internal/appwire/factory_job_protected_test.go create mode 100644 internal/factory/serve/protected.go create mode 100644 internal/factory/serve/protected_test.go diff --git a/cmd/codeaf/appfactory_protected_live_e2e_test.go b/cmd/codeaf/appfactory_protected_live_e2e_test.go new file mode 100644 index 0000000000..c92b368e7f --- /dev/null +++ b/cmd/codeaf/appfactory_protected_live_e2e_test.go @@ -0,0 +1,100 @@ +//go:build e2e + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory/api" +) + +// THE LIVE PROTECTED KEY. Against a real repository read through the signed-in +// gh, read-only: the factory reads its open issues into Backlog, the factory's +// own rules ask first on one path, and the queue says, per Backlog job, which +// protected paths its issue names. One job must name the path and one must +// name none. Nothing is written to GitHub and no model is asked. +// +// Opt-in: CODEAF_LIVE_FACTORY_SETUP=1, CODEAF_LIVE_FACTORY_REPO= +// and CODEAF_LIVE_PROTECTED=, such as `touches auth/`, that some +// open issue names and some does not. +func TestLiveBacklogJobsSayWhichProtectedPathsTheyName(t *testing.T) { + rule := strings.TrimSpace(os.Getenv("CODEAF_LIVE_PROTECTED")) + if os.Getenv(liveSetupOptIn) != "1" || rule == "" { + t.Skipf("opt-in live lane: set %s=1, %s= and CODEAF_LIVE_PROTECTED=", liveSetupOptIn, liveSetupRepoEnv) + } + if !strings.Contains(liveSetupRepo, "/") || liveSetupGH == "" || liveSetupGHConfig == "" { + t.Fatal("BLOCKED: no repository, gh or gh configuration; not faking a pass") + } + appHome, userHome, _ := useLocalHome(t) + var dirs []string + for _, tool := range []string{"git", "gh"} { + if p, err := exec.LookPath(tool); err == nil { + dirs = append(dirs, filepath.Dir(p)) + } + } + dirs = append(dirs, filepath.Dir(liveSetupGH)) + t.Setenv("PATH", strings.Join(dirs, string(os.PathListSeparator))) + t.Setenv("GH_CONFIG_DIR", liveSetupGHConfig) + t.Setenv("GH_PROMPT_DISABLED", "1") + t.Setenv(factoryPostureEnv, "") + t.Setenv("OPENROUTER_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "") + model := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "this lane makes no model call", http.StatusServiceUnavailable) + })) + t.Cleanup(model.Close) + t.Setenv("CODEAF_BASE_URL", model.URL) + + w := newFactoryWire(t, testFactoryHub(t)) + gh := &readOnlyGitHub{} + w.backend.local.githubHTTP = gh.client() + s := &localSetup{factoryWire: w, home: appHome, userHm: userHome} + st := s.start() + if v := s.setup("factory.setup_connect", map[string]any{"source": "gh"}); !v.GitHub.Connected { + t.Fatalf("connect: %+v", v.GitHub) + } + s.setup("factory.setup_select", map[string]any{"repos": []string{liveSetupRepo}}) + deadline := time.Now().Add(liveSetupWait) + for { + if v := s.setup("factory.setup_read", map[string]any{}); repoOf(v, liveSetupRepo).Read == "done" { + break + } + if time.Now().After(deadline) { + t.Fatal("the repository's issues were never read") + } + time.Sleep(200 * time.Millisecond) + } + var rules appwire.FactoryRules + empty := []string{} + s.must("factory.rules_set", map[string]any{"factory": st.Factory, "rules": empty, "ask_first": []string{rule}, "never": empty}, &rules) + + var q api.Queue + s.must("factory.queue", map[string]any{"factory": st.Factory}, &q) + named, none, unjudged := 0, 0, 0 + for _, j := range q.Jobs { + line, _ := json.Marshal(j.Protected) + t.Logf("live protected: %s %s %s", j.Ref, j.State, line) + switch { + case j.Protected == nil: + unjudged++ + case len(j.Protected.Paths) > 0: + named++ + default: + none++ + } + } + t.Logf("live protected summary: jobs %d, naming a protected path %d, naming none %d, not judged %d, GitHub writes refused %d", + len(q.Jobs), named, none, unjudged, len(gh.refused)) + if named == 0 || none == 0 || unjudged != 0 || len(gh.refused) != 0 { + t.Fatalf("named %d, none %d, unjudged %d, refused %v", named, none, unjudged, gh.refused) + } +} diff --git a/docs/changes/unreleased/2145-factory-job-protected.md b/docs/changes/unreleased/2145-factory-job-protected.md new file mode 100644 index 0000000000..f1576546d5 --- /dev/null +++ b/docs/changes/unreleased/2145-factory-job-protected.md @@ -0,0 +1,11 @@ +--- +kind: added +title: A Backlog job says which protected paths it names +pr: 2145 +surface: [engine, docs] +invalidates: + - "Was true: nothing on a job said whether it touches a protected path before it started; only a job the start rules judged carried a mixed reason, and only a started job carried a plan's hold. Is true now: every Backlog job in the queue carries `protected {paths}`, the protected paths of the rules it would start under (ask-first path lines and never paths, as written) that its issue, comments, plan or files name, matched as a start matches them; an empty list names none, and absent is not judged. The app announces it with the hello word `factory.job_protected`." +--- + +A view such as "small and safe" can leave out Backlog jobs that touch a +protected path, by the same rules a start would hold them to. diff --git a/internal/appwire/factory_job_protected.go b/internal/appwire/factory_job_protected.go new file mode 100644 index 0000000000..0df7e69216 --- /dev/null +++ b/internal/appwire/factory_job_protected.go @@ -0,0 +1,21 @@ +package appwire + +// ── factory.job_protected: which protected paths a Backlog job names ─────── +// +// No method: factory.queue's jobs carry `protected {paths}` on each Backlog +// job (internal/factory/api's JobProtected), the protected paths of the +// rules it would start under that its issue, comments, plan or files name. +// Absent on a job is not judged (past the Backlog, or rules nobody could +// read), never "touches none"; an empty list is a job that names none. A +// factory older than this word sends nothing, which reads the same. + +// FeatureFactoryJobProtected is announced in hello by an engine whose +// factory's queue carries [api.JobProtected] on Backlog jobs. +const FeatureFactoryJobProtected = "factory.job_protected" + +func withFactoryJobProtectedFeature(hello Hello, e Engine) Hello { + if _, ok := e.(Factory); ok { + hello = withFeature(hello, FeatureFactoryJobProtected) + } + return hello +} diff --git a/internal/appwire/factory_job_protected_test.go b/internal/appwire/factory_job_protected_test.go new file mode 100644 index 0000000000..212f51cd95 --- /dev/null +++ b/internal/appwire/factory_job_protected_test.go @@ -0,0 +1,14 @@ +package appwire + +import "testing" + +// A factory engine announces that its queue's Backlog jobs say which +// protected paths they name; an engine with no factory does not. +func TestFactoryJobProtectedIsAnnouncedByAFactoryEngine(t *testing.T) { + if h := withFactoryJobProtectedFeature(Hello{}, &fakeEngine{}); len(h.Features) != 0 { + t.Fatalf("announced %v", h.Features) + } + if h := withFactoryJobProtectedFeature(Hello{}, &factoryEngine{}); len(h.Features) != 1 || h.Features[0] != FeatureFactoryJobProtected || FeatureFactoryJobProtected != "factory.job_protected" { + t.Fatalf("announced %v", h.Features) + } +} diff --git a/internal/appwire/server.go b/internal/appwire/server.go index ff641252c6..b520e64fdf 100644 --- a/internal/appwire/server.go +++ b/internal/appwire/server.go @@ -233,6 +233,7 @@ func (s *Server) Serve(ctx context.Context, in io.Reader, hello Hello, engine En hello = withFactoryJobPRReadyFeature(hello, engine) hello = withFactoryJobBriefSeenFeature(hello, engine) hello = withFactoryJobChatsFeature(hello, engine) + hello = withFactoryJobProtectedFeature(hello, engine) hello = withFactoryEffortModelsFeature(hello, engine) hello = withFactoryUsageFeature(hello, engine) hello = withFactoryEstimateFeature(hello, engine) diff --git a/internal/factory/api/wire.go b/internal/factory/api/wire.go index 38b4f0cf4e..76aba5bbcc 100644 --- a/internal/factory/api/wire.go +++ b/internal/factory/api/wire.go @@ -525,6 +525,11 @@ type Job struct { // (factory.RuleChecks), for its two ◆ rows; absent before either was // judged. RuleChecks *RuleChecks `json:"rule_checks,omitempty"` + // Protected is which protected paths a Backlog job names, judged on every + // queue read by the start rules it would start under (JobProtected). + // Absent is not judged: a job past the Backlog, or rules nobody could + // read; the hello word `factory.job_protected` says a client may read it. + Protected *JobProtected `json:"protected,omitempty"` // Brief is the job's brief mark: the rev of its newest kept version and // the brief's status, so a reader that polls the job reads the brief // itself (GET /v1/jobs/{id}/brief) only when one of them moves. Absent @@ -963,6 +968,15 @@ type RuleChecks struct { PublishHead string `json:"publish_head,omitempty"` } +// JobProtected is the protected paths (the rules' ask-first path lines and +// never paths, as written) that the paths a job names touch: its issue, its +// comments, its plan and its pull request's files, matched as a start +// matches them. Empty paths is a job that names none, which is not proof its +// change will touch none: a job's plan is checked again before it goes on. +type JobProtected struct { + Paths []string `json:"paths"` +} + // PRChecks is the check-run summary. A zero count is omitted. type PRChecks struct { Passed int `json:"passed,omitempty"` diff --git a/internal/factory/pipeline/house.go b/internal/factory/pipeline/house.go index 5949e8a41d..7a42449dcf 100644 --- a/internal/factory/pipeline/house.go +++ b/internal/factory/pipeline/house.go @@ -233,3 +233,17 @@ func touchesAny(globs []string, path string) bool { } return false } + +// ProtectedNamed is every protected path of f's start rules (its ask-first +// path lines and its never paths, [ProtectedPaths]) that one of paths +// touches, by the matcher a start uses, in the rules' order; empty, never +// nil, when none is touched. +func (f File) ProtectedNamed(paths []string) []string { + out := []string{} + for _, rule := range ProtectedPaths(f.Start.AskFirst, f.Start.Never) { + if protectedWhy([]string{rule}, paths) != "" { + out = append(out, rule) + } + } + return out +} diff --git a/internal/factory/serve/protected.go b/internal/factory/serve/protected.go new file mode 100644 index 0000000000..cf15d1ba6d --- /dev/null +++ b/internal/factory/serve/protected.go @@ -0,0 +1,61 @@ +package serve + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + factorygithub "github.com/Agent-Field/codeaf/internal/factory/github" + "github.com/Agent-Field/codeaf/internal/factory/intake" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" +) + +// judgeProtected says which protected paths each Backlog job names +// ([api.JobProtected]): the paths intake would hand a start, against the +// rules the job would start under. jobs are the issue jobs of items, in +// order, as [api.FromSnapshot] makes them. Each +// repository's rules are read once, show-only, so a queue never waits on +// GitHub for them; a repository whose rules cannot be read leaves its jobs +// unjudged rather than safe. +func (h *Handler) judgeProtected(ctx context.Context, items []factory.Item, jobs []api.Job) { + if h.config.Pipeline == nil { + return + } + ctx = factorygithub.ShowOnly(ctx) + rules := map[string]*pipeline.File{} + for i, it := range factory.IssueJobs(items) { + if factory.JobState(it) != "backlog" { + continue + } + repo := factory.RepoOf(it) + f, read := rules[repo] + if !read { + f = h.showRules(ctx, repo) + rules[repo] = f + } + if f != nil { + jobs[i].Protected = &api.JobProtected{Paths: f.ProtectedNamed(intake.Job(it).Paths)} + } + } +} + +// showRules is repo's rules as the factory runs them, or nil when they +// cannot be read. +func (h *Handler) showRules(ctx context.Context, repo string) *pipeline.File { + if repo == "" { + return nil + } + dir := "" + if h.seam.RecipeAt != nil { + _, _, at, err := h.seam.RecipeAt(repo) + if err != nil { + return nil + } + dir = at + } + f, err := h.config.Pipeline(ctx, repo, dir) + if err != nil { + return nil + } + return &f +} diff --git a/internal/factory/serve/protected_test.go b/internal/factory/serve/protected_test.go new file mode 100644 index 0000000000..f36562b07b --- /dev/null +++ b/internal/factory/serve/protected_test.go @@ -0,0 +1,75 @@ +package serve_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" + "github.com/Agent-Field/codeaf/internal/factory/serve" +) + +// A BACKLOG JOB SAYS WHICH PROTECTED PATHS IT NAMES: the ask-first path lines +// and never paths of the rules it would start under, matched against the +// paths its issue, comments and files name, as a start matches them. An empty +// list is a job that names none; absent is a job nobody judged (not in the +// backlog, or rules nobody could read). +func TestTheQueueSaysWhichProtectedPathsABacklogJobNames(t *testing.T) { + seam := factory.FixtureSeam(now) + seam.RecipeAt = func(repo string) (factory.Recipe, []factory.Problem, string, error) { + return factory.DefaultRecipe(), nil, "", nil + } + seam.Load = func() (factory.Snapshot, error) { + return factory.Snapshot{Items: []factory.Item{ + {ID: 1, Repo: "acme/web", State: factory.StateNew, Kind: factory.KindIssue, Title: "Rotate keys", Body: "The bug is in infra/terraform/main.tf and docs/a.md"}, + {ID: 2, Repo: "acme/web", State: factory.StateNew, Kind: factory.KindIssue, Title: "Typo", Body: "README.md says teh"}, + {ID: 3, Repo: "acme/web", State: factory.StateNew, Kind: factory.KindPR, Body: "infra/x.tf"}, + {ID: 6, Repo: "acme/web", State: factory.StateNew, Kind: factory.KindIssue, Title: "CI", Comments: []factory.Comment{{Body: "see .github/workflows/ci.yml"}}}, + {ID: 4, Repo: "acme/web", State: factory.StateRunning, Kind: factory.KindIssue, Body: "infra/x.tf"}, + {ID: 5, Repo: "acme/api", State: factory.StateNew, Kind: factory.KindIssue, Body: "infra/x.tf"}, + {ID: 7, Repo: "acme/web", State: factory.StateNew, Kind: factory.KindIssue, Body: "infra/x.tf", PublishError: "push refused"}, + }}, nil + } + cfg := config + cfg.Pipeline = func(_ context.Context, repo, _ string) (pipeline.File, error) { + if repo == "acme/api" { + return pipeline.File{}, context.DeadlineExceeded + } + f := pipeline.Defaults() + f.Start.AskFirst = []string{"touches infra/", "path: billing/", "Ask before deleting files"} + f.Start.Never = []string{".github/workflows/"} + return f, nil + } + r := httptest.NewRequest(http.MethodGet, "/v1/queue", nil) + r.RemoteAddr = "127.0.0.1:1234" + w := httptest.NewRecorder() + serve.New(seam, cfg).ServeHTTP(w, r) + checkJSON(t, w, http.StatusOK) + var q api.Queue + if err := json.Unmarshal(w.Body.Bytes(), &q); err != nil { + t.Fatal(err) + } + if len(q.Jobs) != 6 { + t.Fatalf("jobs = %d", len(q.Jobs)) + } + got := map[string]*api.JobProtected{} + for _, j := range q.Jobs { + got[j.JobID] = j.Protected + } + want := map[string][]string{"web#1": {"infra/"}, "web#2": {}, "web#6": {".github/workflows/"}} + for id, paths := range want { + p := got[id] + if p == nil || len(p.Paths) != len(paths) || (len(paths) > 0 && p.Paths[0] != paths[0]) { + t.Errorf("job %s protected = %+v, want %v", id, p, paths) + } + } + for _, id := range []string{"web#4", "api#5", "web#7"} { + if got[id] != nil { + t.Errorf("job %s was judged: %+v", id, got[id]) + } + } +} diff --git a/internal/factory/serve/serve.go b/internal/factory/serve/serve.go index 8e18faee35..4936f3d5e1 100644 --- a/internal/factory/serve/serve.go +++ b/internal/factory/serve/serve.go @@ -502,6 +502,7 @@ func (h *Handler) queue(w http.ResponseWriter, r *http.Request) { h.soloJob(&q.Jobs[i], s.Items[i], handle) } } + h.judgeProtected(r.Context(), s.Items, q.Jobs) q.Reason = reason if reason != "" { if q.Jobs == nil { From 4bc08508d042f6fca8925221d5aba6ad01c34065 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:51:14 -0500 Subject: [PATCH 05/28] factory: a no-change comment finishes Follow PR, and a Try again that can't succeed doesn't spend (#2256) * factory: a raised budget resumes an Auto job its runner parked at the cap A worker's runner parks at the budget through its ask, and its report puts that question on the desk copy as Asking. raise_budget resumed only when Asking was nil, so for a held job it never resumed. Its Yes on the budget signal was a second answer path that the gate never reached. The raise now treats the open ask as the cap question when QKind is cap and any Asking is that same question. When the new cap clears the spend on an Auto job, the raise answers "continue" through the answer door a person's answer uses (continueCap, taken out of the answer handler). The ask comes down, the ledger keeps budget_changed then answered, and the timeline says the budget was raised. The budget signal now carries only the cap and goes first, so the runner never doubles the old cap. An unheld cap continue had the same gap: KeepAnswer cleared Asking but left the job needing you. AnswerUnheld now queues it through run.ReleaseCap, which the answer handler's own fallback uses as well. Manual jobs, any other open ask, and a raise to the spend or lower stay parked. The new worker tests run against the rig's worker-backed desk. The old test ran on a Manual job through a second handler. Co-Authored-By: Claude Opus 5.5 * factory: say the raise resumed only once the continue is answered Review of the raise found two problems. First, the timeline said "carrying on" in the same write as the cap, before the answer door ran. A refused answer then left the job parked with a timeline that said otherwise. The note is now written after the continue is accepted. Second, the answer door's fallback for a cap no runner waits on queued the job whatever it waited on at that moment. A job that moved on to another question between the read and the release lost that question. The release now takes down only a cap question. This commit also adds the change entry and removes a stray test comment. Co-Authored-By: Claude Opus 5.5 * changes: the raised budget entry carries its PR number Co-Authored-By: Claude Opus 5.5 * factory: a person's continue at the cap queues an unheld job again (test) Pins the second half of the fix: on an unheld Auto job whose copy still carries the cap question as an open ask, a person's continue takes the ask down and queues the job. On the base (39cec149fb) the job was left needing you with its question; on this branch it is queued. Co-Authored-By: Claude Opus 5.5 * test(factory): preserve ordered budget answers and spending stops Co-Authored-By: Claude Opus 5.5 * test(factory): a Manual budget raise reaches the next approval Co-Authored-By: Claude Opus 5.5 * fix(factory): a budget raise answers Manual's cap question The Auto-only resume predicate stranded Manual jobs after a raise above spend. Keep Take over and other decisions parked while allowing Manual to reach its next step approval. Co-Authored-By: Claude Opus 5.5 * test: reproduce no-change feedback report escaping correction Co-Authored-By: Claude Opus 5.5 * fix: correct no-change feedback reports in the same conversation Tracked feedback requires a structured current-head assessment. Return omissions at the reporting door while the model can correct them, retaining the clean published-head verifier. Co-Authored-By: Claude Opus 5.5 * docs: use the required change entry schema Co-Authored-By: Claude Opus 5.5 * test: refuse unchanged no-change feedback retries before spending Co-Authored-By: Claude Opus 5.5 * fix: refuse unchanged feedback verification retries before spending Bind the no-change verification refusal to its PR observations in the saved result. Answer and rerun paths retain the question and budget until those facts change; Stop and fresh feedback remain available. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: exercise no-change feedback with a live factory model Co-Authored-By: Claude Opus 5.5 * test: preserve unpaid retry when a refused checkout is restored Co-Authored-By: Claude Opus 5.5 * fix: recheck refused checkout state before another model call A PR snapshot alone cannot tell whether a rejected checkout was restored. Preserve its content receipt and validate it on the worker without spending; changed contents permit a fresh assessment. Co-Authored-By: Claude Opus 5.5 * test: retain current-head feedback review fences Co-Authored-By: Claude Opus 5.5 * test: preserve the running state in the live feedback fixture Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ...actory_no_change_feedback_live_e2e_test.go | 109 ++++++++++ .../unreleased/2256-follow-pr-no-change.md | 20 ++ internal/factory/run/chatstage.go | 11 +- internal/factory/run/follow_retry.go | 122 +++++++++++ internal/factory/run/inbox.go | 3 + internal/factory/run/loop.go | 19 +- .../factory/run/no_change_feedback_test.go | 192 ++++++++++++++++++ internal/factory/run/person.go | 10 + internal/factory/run/step_control.go | 6 + 9 files changed, 488 insertions(+), 4 deletions(-) create mode 100644 cmd/codeaf/factory_no_change_feedback_live_e2e_test.go create mode 100644 docs/changes/unreleased/2256-follow-pr-no-change.md create mode 100644 internal/factory/run/follow_retry.go create mode 100644 internal/factory/run/no_change_feedback_test.go diff --git a/cmd/codeaf/factory_no_change_feedback_live_e2e_test.go b/cmd/codeaf/factory_no_change_feedback_live_e2e_test.go new file mode 100644 index 0000000000..d6630052ce --- /dev/null +++ b/cmd/codeaf/factory_no_change_feedback_live_e2e_test.go @@ -0,0 +1,109 @@ +//go:build e2e + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "github.com/Agent-Field/codeaf/internal/home" +) + +// A captured local PR snapshot carries the same comment revisions as GitHub. +// The runner, conversation, tools and model are real; nothing writes to GitHub. +func TestLiveFollowPRConsidersNothingToChangeWithoutDiff(t *testing.T) { + if os.Getenv(liveFactoryOptIn) != "1" { + t.Skip("opt-in live factory lane") + } + if liveFactoryKey == "" { + t.Fatal("BLOCKED: no OpenRouter key") + } + root := resolvedTempDir(t) + t.Setenv("HOME", filepath.Join(root, "home")) + t.Setenv(home.EnvVar, filepath.Join(root, "state")) + t.Setenv("CODEAF_MODEL_POOL", "off") + t.Setenv("OPENROUTER_API_KEY", liveFactoryKey) + t.Setenv("OPENAI_API_KEY", "") + t.Setenv(factoryPostureEnv, "") + profile := v3Profile(t, map[string]any{ + "models.roles": "", "models.tiers.low": liveFactoryModel, "models.tiers.reflex": liveFactoryModel, + "models.tiers.high": liveFactoryModel, "models.tiers.worker": liveFactoryModel, "models.tiers.mastermind": liveFactoryModel, + config.KeyTaskModelsMode: "one", config.KeyTaskModelsOne: liveFactoryModel, "models.fallbacks": "", + }) + t.Setenv(config.ProfileDirEnv, profile) + recorder := newLiveRecorder(t, config.DefaultBaseURL) + t.Setenv("CODEAF_BASE_URL", recorder.server.URL) + g := newRunRig(t) + base := strings.TrimSpace(gitT(t, g.workspace, "rev-parse", "HEAD")) + if err := os.WriteFile(filepath.Join(g.workspace, "README.md"), []byte("hello\n"), 0600); err != nil { + t.Fatal(err) + } + gitT(t, g.workspace, "add", "README.md") + gitT(t, g.workspace, "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", "commit", "-qm", "Add greeting") + head := strings.TrimSpace(gitT(t, g.workspace, "rev-parse", "HEAD")) + freshEngineProcess(t) + proc, err := openV3Process("chat") + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.closeAll) + launch, err := openV3Launch(proc, v3Options{Model: liveFactoryModel, OneModel: true, Workspace: g.workspace}) + if err != nil { + t.Fatal(err) + } + assertFactoryPin(t, launch.Config, liveFactoryModel) + stageSettle = 300 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + item := factory.Item{Title: "Add hello to README", Body: "README.md contains hello on its own line.", Repo: "api", Kind: factory.KindIssue, Cap: .50, Autonomy: factory.AutonomyAuto, Model: liveFactoryModel, HeadCommit: head, ChangeBase: base, Publish: "pr", State: factory.StateRunning, + PullRequest: &factory.PullRequest{Number: 11, State: factory.PROpen, HeadSHA: head, BaseSHA: base, ChecksRead: true, ReviewsRead: true, Mergeable: factory.MergeableYes, FeedbackTracked: true, Feedback: "PR conversation comment 1: Nothing to change.", FeedbackItems: []factory.PRFeedback{{ID: "conversation:1", Revision: "v1"}}}, + FollowUp: &factory.PRFollowUp{Kind: "pr", State: "queued", HeadSHA: head, Attempt: 1, Limit: 3, Inspecting: map[string]string{"conversation:1": "v1"}, Details: "Current PR description: Add hello to README. PR conversation comment 1: Nothing to change.", Request: "Read the current checkout and comment. Assess it against the diff; do not manufacture an edit. No GitHub access is needed: the captured PR snapshot is authoritative."}, + Stream: &factory.Stream{Paused: true}, + } + factory.EnsureFollowPR(&item) + made, err := g.st.Create(item) + if err != nil { + t.Fatal(err) + } + id := made.ID + maker := factoryStageMaker(g.st, g.workspace, profile, launch.Config) + r := factoryrun.New(factoryrun.Options{Store: g.st, Workdir: func(factory.Item) (string, error) { return g.workspace, nil }, Exec: map[factory.StageKind]factoryrun.Executor{factory.StageChat: factoryChatStage(g.st, profile, factoryrun.NewChatExecutor(maker))}, Push: func(string, string) error { t.Error("no-change review tried to push"); return factory.ErrReadOnly }}) + stopAll(t, g.st, r) + if err := r.Hold(id, false); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(liveFactoryWall) + var got factory.Item + for time.Now().Before(deadline) { + got, _ = g.st.Get(id) + if got.State == factory.StateShipped || got.Question != "" || got.Failure != nil { + break + } + time.Sleep(100 * time.Millisecond) + } + recorder.mu.Lock() + t.Logf("LIVE calls=%d cost_usd=%.6f requests=%v recorded_models=%v problems=%v", recorder.calls, recorder.cost, recorder.requests, recorder.replies, recorder.problems) + if recorder.calls == 0 || len(recorder.problems) != 0 || len(recorder.replies) != 1 || recorder.replies[liveFactoryModel] == 0 || recorder.cost > liveFactoryBudget { + t.Error("live model evidence or budget failed") + } + recorder.mu.Unlock() + if got.State != factory.StateShipped || got.FollowUp.State != "watching" || got.FollowUp.Reviewed["conversation:1"] != "v1" { + t.Fatalf("no-change feedback did not finish: state=%s question=%q follow=%+v", got.State, got.Question, got.FollowUp) + } + ph := got.Stream.Phases[len(got.Stream.Phases)-1] + if ph.Result == nil || ph.Result.Model != liveFactoryModel || ph.Result.Repair == nil || ph.Result.Repair.Decision != "no_change" { + t.Fatalf("wrong recorded result: %+v", ph.Result) + } + if strings.TrimSpace(gitT(t, g.workspace, "rev-parse", "HEAD")) != head || strings.TrimSpace(gitT(t, g.workspace, "status", "--porcelain", "--untracked-files=all")) != "" { + t.Fatal("no-change review produced a diff") + } + if v := factory.MergeGate(got, factory.MergeAsk{By: "sam"}); !v.OK { + t.Fatalf("Merge remains blocked: %s", v.Why) + } + t.Logf("LIVE finished=true no_new_diff=true considered=%s model=%s spent_usd=%.6f", got.FollowUp.Reviewed["conversation:1"], ph.Result.Model, got.Stream.Spent) +} diff --git a/docs/changes/unreleased/2256-follow-pr-no-change.md b/docs/changes/unreleased/2256-follow-pr-no-change.md new file mode 100644 index 0000000000..1bf3f3a044 --- /dev/null +++ b/docs/changes/unreleased/2256-follow-pr-no-change.md @@ -0,0 +1,20 @@ +--- +kind: fixed +title: Follow PR considers comments without manufacturing changes +pr: 2256 +surface: [engine] +invalidates: + - "No-change feedback reports with missing checklist fields previously ended a paid round before receiving a correction; they now receive the correction during the report call." +--- + +Follow PR can finish a current-head review of a comment that requests no edit +without making a new diff. Missing review checklist fields now reach the model +when it reports, allowing correction in the same conversation instead of spending +another round to discover the omission. Merge still requires its existing gates. + +Trying the same refused no-change review again with unchanged PR observations +now returns a recovery explanation without starting a paid round. New pushes, +comments or check results permit another attempt. Stop remains available, and +attempt exhaustion never bypasses Merge's gates. +A checkout refusal is reread locally before another model call: unchanged files +spend nothing, while restoring the published checkout permits another assessment. diff --git a/internal/factory/run/chatstage.go b/internal/factory/run/chatstage.go index e9df36fb86..1fe9203b78 100644 --- a/internal/factory/run/chatstage.go +++ b/internal/factory/run/chatstage.go @@ -256,6 +256,8 @@ func (e chatExecutor) Run(ctx context.Context, job Job) (factory.StageResult, er door.ask, door.stage = job.Ask, job.Stage if f := job.Item.FollowUp; f != nil && f.Kind == "readiness" && f.Readiness != nil { door.readinessHead = f.Readiness.Head + } else if p := job.Item.PullRequest; factory.IsCIRepairStage(job.Stage) && p != nil && p.FeedbackTracked { + door.feedbackHead = p.HeadSHA } log := job.Log if log == nil { @@ -896,6 +898,7 @@ type stageDoor struct { // until done, which no report contradicts. stage factory.Stage readinessHead string + feedbackHead string // ask is the round's road to the manager (Job.Ask), nil for none. ask func(ctx context.Context, q factory.Asked) (string, error) } @@ -935,8 +938,12 @@ func newStageDoor() *stageDoor { return &stageDoor{} } // claim marked ok: false, is held like any report and judged into another // round. A refused call is not counted in Reports: nothing was reported. func (d *stageDoor) Report(_ context.Context, result factory.StageResult) error { - if !result.Checkpoint && d.readinessHead != "" && result.Done && (result.Repair == nil || (result.Repair.Decision != "needs_repair" && result.Repair.Decision != "needs_you")) { - if why := factory.ReadinessReportFault(d.readinessHead, result); why != "" { + head := d.readinessHead + if head == "" && result.Repair != nil && result.Repair.Decision == "no_change" { + head = d.feedbackHead + } + if !result.Checkpoint && head != "" && result.Done && (result.Repair == nil || (result.Repair.Decision != "needs_repair" && result.Repair.Decision != "needs_you")) { + if why := factory.ReadinessReportFault(head, result); why != "" { d.RejectReport() return errors.New(why) } diff --git a/internal/factory/run/follow_retry.go b/internal/factory/run/follow_retry.go new file mode 100644 index 0000000000..5fdc9219b9 --- /dev/null +++ b/internal/factory/run/follow_retry.go @@ -0,0 +1,122 @@ +package run + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "strings" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/verification" +) + +const noChangeReviewRefusal = "current-head feedback review is incomplete or the checkout changed" +const followRefusalReceipt = "Follow PR refusal: " +const followCheckoutReceipt = "Follow PR refused checkout: " + +// The receipt binds a refused no-change report to the observations it read. +// It travels with the existing result notes across worker reports and restarts. +func followRefusalKey(it factory.Item, reason string) string { + p := it.PullRequest + if p == nil { + return "" + } + raw, _ := json.Marshal(struct { + Head, Feedback, Decision, Reason string + Versions map[string]string + Checks factory.PRChecks + Conflict bool + }{p.HeadSHA, p.Feedback, p.ReviewDecision, reason, factory.ReviewVersions(p), p.Checks, p.Conflict}) + return fmt.Sprintf("%x", sha256.Sum256(raw)) +} + +func unchangedFollowRefusal(it factory.Item, ph factory.Phase) error { + if ph.Name != factory.FollowPRStep || ph.Result == nil { + return nil + } + r := *ph.Result + // A checkout refusal is revalidated by its worker before any model call. + if refusedCheckout(r) != nil { + return nil + } + if verificationReason(r) != noChangeReviewRefusal { + return nil + } + key := followRefusalReceipt + followRefusalKey(it, noChangeReviewRefusal) + for _, note := range r.Notes { + if note == key { + return StepControlRefusal("This PR has the same head and comments, and its current-head review is still incomplete or the checkout changed. Wait for a new push or comment before trying again, or stop the job. No paid round was started.") + } + } + return nil +} + +func followAnswerRefusal(it factory.Item, yes bool, words string) error { + if it.QKind != factory.QKindOutOfAttempts || factory.OutOfAttemptsChoice(yes, strings.TrimSpace(words)) != factory.ChoiceTryOnceMore || it.Stream == nil { + return nil + } + i := it.Stream.Cur + if i < 0 || i >= len(it.Stream.Phases) { + return nil + } + return unchangedFollowRefusal(it, it.Stream.Phases[i]) +} + +// A complete assessment can be refused because the checkout changed. Only the +// worker can reread that checkout, so server answers admit an unpaid recheck. +func refusedCheckout(r factory.StageResult) *verification.Snapshot { + for _, note := range r.Notes { + if raw, ok := strings.CutPrefix(note, followCheckoutReceipt); ok { + var at verification.Snapshot + if json.Unmarshal([]byte(raw), &at) == nil && at.Head != "" && at.Fingerprint != "" { + return &at + } + } + } + return nil +} + +func followRefusalNotes(job Job, res factory.StageResult) []string { + notes := []string{followRefusalReceipt + followRefusalKey(job.Item, noChangeReviewRefusal)} + if factory.ReadinessReportFault(job.Item.PullRequest.HeadSHA, res) != "" { + return notes + } + at, err := verification.SnapshotAt(job.Dir) + if err != nil || at.Head == "" || at.Fingerprint == "" { + return nil + } + raw, _ := json.Marshal(at) + return append(notes, followCheckoutReceipt+string(raw)) +} + +func cachedCheckoutRefusal(job Job) *factory.StageResult { + if job.Stage.Name != factory.FollowPRStep || job.Item.Stream == nil { + return nil + } + for _, ph := range job.Item.Stream.Phases { + if ph.Name != job.Stage.Name || ph.Result == nil { + continue + } + r := *ph.Result + before := refusedCheckout(r) + if before == nil || verificationReason(r) != noChangeReviewRefusal { + return nil + } + receipt := followRefusalReceipt + followRefusalKey(job.Item, noChangeReviewRefusal) + same := false + for _, note := range r.Notes { + same = same || note == receipt + } + if !same { + return nil + } + now, err := verification.SnapshotAt(job.Dir) + if err != nil || now != *before { + return nil + } + r.Spent, r.Chat, r.Model = 0, "", "" + r.Output = "The checkout still has the same unfinished changes or does not match the published head. Restore the clean published checkout, wait for a new push or comment, or stop the job. No paid round was started." + return &r + } + return nil +} diff --git a/internal/factory/run/inbox.go b/internal/factory/run/inbox.go index 310d30a475..cc41b4a898 100644 --- a/internal/factory/run/inbox.go +++ b/internal/factory/run/inbox.go @@ -219,6 +219,9 @@ func (lp *floorLoop) keepAnswer(c *loopCtl, id int, a loopAnswer) error { // keepStored puts the person's answer, said as word, on the item for the // step's next round, and takes the question down. func keepStored(it *factory.Item, word string, now time.Time) error { + if err := followAnswerRefusal(*it, false, word); err != nil { + return err + } q := it.Asking if q == nil { return fmt.Errorf("%s is not waiting on you", it.Ref()) diff --git a/internal/factory/run/loop.go b/internal/factory/run/loop.go index a33fdd0d37..c047ee9440 100644 --- a/internal/factory/run/loop.go +++ b/internal/factory/run/loop.go @@ -961,7 +961,14 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { if job.Progress != nil { job.Progress(followProgressStart(job)) } - res, again, err := lp.round(c, exec, job) + var res factory.StageResult + var again bool + var err error + if cached := cachedCheckoutRefusal(job); cached != nil { + res = *cached + } else { + res, again, err = lp.round(c, exec, job) + } if c.ctx.Err() != nil { return false } @@ -986,9 +993,11 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { repair.Decision = "fixed" candidate.Repair = &repair if checked := lp.completedPublishedRepair(job.Item, candidate); checked.Repair.Decision != "no_change" { + receipts := followRefusalNotes(job, res) res.Done, res.Exit = false, 1 res.Output = "The no-change review needs a complete assessment of current feedback on the clean published head. " + res.Output - res.Notes = append(res.Notes, verification.FailurePrefix+" current-head feedback review is incomplete or the checkout changed") + res.Notes = append(res.Notes, verification.FailurePrefix+" "+noChangeReviewRefusal) + res.Notes = append(res.Notes, receipts...) } } if readiness && (res.Repair == nil || (res.Repair.Decision != "needs_repair" && res.Repair.Decision != "needs_you")) && verificationReason(res) == "" { @@ -2225,6 +2234,9 @@ func (r *Runner) Answer(id int, yes bool, words string) error { if err != nil { return err } + if err := followAnswerRefusal(it, yes, words); err != nil { + return err + } if it.State == factory.StateNeedsYou && it.QKind == factory.QKindAskFirst { if yes { return r.Launch(id) @@ -2257,6 +2269,9 @@ func (r *Runner) Answer(id int, yes bool, words string) error { c.pending = "" lp.mu.Unlock() if err := r.opts.Store.Update(id, func(it *factory.Item) error { + if err := followAnswerRefusal(*it, yes, words); err != nil { + return err + } grantAdditionalRound(it, yes, words) return nil }); err != nil { diff --git a/internal/factory/run/no_change_feedback_test.go b/internal/factory/run/no_change_feedback_test.go new file mode 100644 index 0000000000..e783da83ff --- /dev/null +++ b/internal/factory/run/no_change_feedback_test.go @@ -0,0 +1,192 @@ +package run + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" +) + +func TestFollowPRNoChangeFeedbackCorrectsReportWithoutAnotherRound(t *testing.T) { + dir := gitRig(t) + head := mustGit(t, dir, "rev-parse", "HEAD") + maker := &FakeMaker{Script: func(ctx context.Context, c *FakeConversation) { + r := factory.StageResult{Done: true, Repair: &factory.CIRepairResult{Decision: "no_change", Reason: "Read the current comment: Nothing to change. It requests no edit."}} + if err := c.Spec.Stage.Report(ctx, r); err == nil || !strings.Contains(err.Error(), "repair.readiness") { + t.Errorf("incomplete feedback review was accepted instead of corrected in the same round: %v", err) + } + r.Repair.Readiness = &factory.ReadinessReview{Head: "stale", DiffReviewed: true, DescriptionMatches: true, FeedbackAddressed: true, NoUnfinishedWork: true} + if err := c.Spec.Stage.Report(ctx, r); err == nil { + t.Error("stale-head review was accepted") + } + r.Repair.Readiness.Head = head + r.Repair.Readiness.FeedbackAddressed = false + if err := c.Spec.Stage.Report(ctx, r); err == nil { + t.Error("unreviewed current-head feedback was accepted") + } + r.Repair.Readiness.FeedbackAddressed = true + if err := c.Spec.Stage.Report(ctx, r); err != nil { + t.Errorf("complete no-change assessment refused: %v", err) + } + }} + g := newRig(t, map[factory.StageKind]Executor{factory.StageChat: NewChatExecutor(maker)}, func(o *Options) { + o.Workdir = func(factory.Item) (string, error) { return dir, nil } + o.Push = func(string, string) error { t.Error("no-change review pushed a commit"); return nil } + }) + id := g.add("read harmless comment", chat("proof")) + if err := g.st.Update(id, func(it *factory.Item) error { + it.Cap = 10 + it.State, it.Publish, it.HeadCommit = factory.StateRunning, "pr", head + it.PullRequest = &factory.PullRequest{Number: 11, State: factory.PROpen, HeadSHA: head, ChecksRead: true, ReviewsRead: true, Mergeable: factory.MergeableYes, FeedbackTracked: true, Feedback: "Nothing to change", FeedbackItems: []factory.PRFeedback{{ID: "conversation:1", Revision: "v1"}}} + it.FollowUp = &factory.PRFollowUp{Kind: "pr", State: "queued", HeadSHA: head, Attempt: 1, Limit: 3, Inspecting: map[string]string{"conversation:1": "v1"}} + it.Stream = &factory.Stream{Phases: []factory.Phase{{Name: "proof", Kind: factory.StageChat, State: factory.PhaseDone}}, Paused: true} + factory.EnsureFollowPR(it) + return nil + }); err != nil { + t.Fatal(err) + } + if err := g.r.Hold(id, false); err != nil { + t.Fatal(err) + } + it := g.waitState(id, factory.StateShipped) + if it.FollowUp.State != "watching" || it.FollowUp.Reviewed["conversation:1"] != "v1" || !strings.Contains(it.FollowUp.Reason, "requests no edit") || len(maker.Opened()) != 1 { + t.Fatalf("feedback not considered in one round: %+v", it.FollowUp) + } + if v := factory.MergeGate(it, factory.MergeAsk{By: "sam"}); !v.OK { + t.Fatalf("harmless feedback still blocks Merge: %s", v.Why) + } + for _, ci := range []factory.PRChecks{{Pending: 1}, {Failed: 1}} { + it.PullRequest.Checks = ci + if v := factory.MergeGate(it, factory.MergeAsk{By: "sam"}); v.OK { + t.Fatal("CI blocker opened Merge") + } + } + if got := mustGit(t, dir, "rev-parse", "HEAD"); got != head { + t.Fatal("no-change review changed HEAD") + } +} + +func TestFollowPRNoChangeRefusalDoesNotPayForSameSnapshot(t *testing.T) { + dir := gitRig(t) + head := mustGit(t, dir, "rev-parse", "HEAD") + var calls atomic.Int32 + g := newRig(t, map[factory.StageKind]Executor{factory.StageChat: ExecutorFunc(func(context.Context, Job) (factory.StageResult, error) { + calls.Add(1) + return factory.StageResult{Done: true, Spent: .03, Repair: &factory.CIRepairResult{Decision: "no_change", Reason: "Nothing to change"}}, nil + })}, func(o *Options) { o.Workdir = func(factory.Item) (string, error) { return dir, nil } }) + id := g.add("incomplete feedback review", chat("proof")) + if err := g.st.Update(id, func(it *factory.Item) error { + it.Cap = 10 + it.State, it.Publish, it.HeadCommit = factory.StateRunning, "pr", head + it.PullRequest = &factory.PullRequest{Number: 11, State: factory.PROpen, HeadSHA: head, FeedbackTracked: true, Feedback: "Please fix the error", FeedbackItems: []factory.PRFeedback{{ID: "conversation:1", Revision: "v1"}}} + it.FollowUp = &factory.PRFollowUp{Kind: "pr", State: "queued", HeadSHA: head, Attempt: 1, Limit: 3, Inspecting: map[string]string{"conversation:1": "v1"}} + it.Stream = &factory.Stream{Phases: []factory.Phase{{Name: "proof", Kind: factory.StageChat, State: factory.PhaseDone}}, Paused: true} + factory.EnsureFollowPR(it) + return nil + }); err != nil { + t.Fatal(err) + } + if err := g.r.Hold(id, false); err != nil { + t.Fatal(err) + } + it := g.waitState(id, factory.StateNeedsYou) + before := calls.Load() + if err := g.r.Answer(id, true, factory.ChoiceTryOnceMore); err == nil || !strings.Contains(err.Error(), "new push or comment") { + t.Fatalf("unchanged refusal started paid retry instead of explaining recovery: %v", err) + } + for _, action := range []struct { + name string + run func(*factory.Item) error + }{ + {"held answer", func(i *factory.Item) error { return AnswerHeld(i, true, time.Now(), factory.ChoiceTryOnceMore) }}, + {"cold answer", func(i *factory.Item) error { return AnswerUnheld(i, true, factory.ChoiceTryOnceMore, time.Now()) }}, + {"step rerun", func(i *factory.Item) error { return StepControl(i, factory.FollowPRStep, true, "", time.Now()) }}, + } { + t.Run(action.name, func(t *testing.T) { + current, _ := g.st.Get(id) + if err := action.run(¤t); err == nil { + t.Fatal("unchanged deterministic refusal allowed retry") + } + for _, change := range []string{"head", "comment"} { + fresh, _ := g.st.Get(id) + if change == "head" { + fresh.PullRequest.HeadSHA = strings.Repeat("f", 40) + } else { + fresh.PullRequest.FeedbackItems = append(fresh.PullRequest.FeedbackItems, factory.PRFeedback{ID: "conversation:2", Revision: "v1"}) + fresh.PullRequest.Feedback += "\nNew comment" + } + if err := action.run(&fresh); err != nil { + t.Fatalf("%s change still refused: %v", change, err) + } + } + }) + } + current, _ := g.st.Get(id) + if calls.Load() != before || current.Stream.Spent != it.Stream.Spent || current.Question != it.Question { + t.Fatal("refused retry changed spend or question") + } + if v := factory.MergeGate(current, factory.MergeAsk{By: "sam"}); v.OK { + t.Fatal("attempt exhaustion opened Merge with unassessed requested change") + } + if err := AnswerUnheld(¤t, false, factory.ChoiceCancel, time.Now()); err != nil { + t.Fatalf("Stop refused: %v", err) + } +} + +func TestFollowPRCheckoutRefusalRetriesWithoutPayingUntilCheckoutChanges(t *testing.T) { + dir := gitRig(t) + head := mustGit(t, dir, "rev-parse", "HEAD") + dirty := filepath.Join(dir, "untracked.txt") + if err := os.WriteFile(dirty, []byte("unfinished work"), 0600); err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + g := newRig(t, map[factory.StageKind]Executor{factory.StageChat: ExecutorFunc(func(context.Context, Job) (factory.StageResult, error) { + calls.Add(1) + return factory.StageResult{Done: true, Repair: &factory.CIRepairResult{Decision: "no_change", Reason: "Comment requests no change", Readiness: &factory.ReadinessReview{Head: head, DiffReviewed: true, DescriptionMatches: true, FeedbackAddressed: true, NoUnfinishedWork: true}}}, nil + })}, func(o *Options) { o.Workdir = func(factory.Item) (string, error) { return dir, nil } }) + id := g.add("checkout refusal", chat("proof")) + if err := g.st.Update(id, func(it *factory.Item) error { + it.Cap = 10 + it.State, it.Publish, it.HeadCommit = factory.StateRunning, "pr", head + it.PullRequest = &factory.PullRequest{Number: 11, State: factory.PROpen, HeadSHA: head, FeedbackTracked: true, Feedback: "Nothing to change", FeedbackItems: []factory.PRFeedback{{ID: "conversation:1", Revision: "v1"}}} + it.FollowUp = &factory.PRFollowUp{Kind: "pr", State: "queued", HeadSHA: head, Attempt: 1, Limit: 3, Inspecting: map[string]string{"conversation:1": "v1"}} + it.Stream = &factory.Stream{Phases: []factory.Phase{{Name: "proof", Kind: factory.StageChat, State: factory.PhaseDone}}, Paused: true} + factory.EnsureFollowPR(it) + return nil + }); err != nil { + t.Fatal(err) + } + if err := g.r.Hold(id, false); err != nil { + t.Fatal(err) + } + it := g.waitState(id, factory.StateNeedsYou) + if it.FollowUp.Reviewed["conversation:1"] != "" { + t.Fatal("dirty checkout acknowledged feedback") + } + before := calls.Load() + if err := g.r.Answer(id, true, factory.ChoiceTryOnceMore); err != nil { + t.Fatalf("checkout retry must reach unpaid checkout validation: %v", err) + } + g.wait(id, "new unpaid refusal", func(i factory.Item) bool { + return i.State == factory.StateNeedsYou && i.Stream.Phases[len(i.Stream.Phases)-1].Round > it.Stream.Phases[len(it.Stream.Phases)-1].Round + }) + if calls.Load() != before { + t.Fatal("unchanged dirty checkout started another paid model round") + } + if err := os.Remove(dirty); err != nil { + t.Fatal(err) + } + if err := g.r.Answer(id, true, factory.ChoiceTryOnceMore); err != nil { + t.Fatalf("restored checkout still refused: %v", err) + } + got := g.waitState(id, factory.StateShipped) + if calls.Load() != before+1 || got.FollowUp.Reviewed["conversation:1"] != "v1" { + t.Fatal("restored checkout did not receive one fresh review") + } +} diff --git a/internal/factory/run/person.go b/internal/factory/run/person.go index 9cbb11279f..b0e3621d70 100644 --- a/internal/factory/run/person.go +++ b/internal/factory/run/person.go @@ -112,6 +112,13 @@ func HoldQueued(it *factory.Item, now time.Time) error { // passed (yes) or sent back to wait (no) here, because only a person's door // moves it. Passing it stamps the step's finish, which the brief's clock reads. func AnswerHeld(it *factory.Item, yes bool, now time.Time, words ...string) error { + answerWords := "" + if len(words) > 0 { + answerWords = words[0] + } + if err := followAnswerRefusal(*it, yes, answerWords); err != nil { + return err + } if it.State != factory.StateNeedsYou || strings.TrimSpace(it.Question) == "" { return fmt.Errorf("%s is not waiting on you", it.Ref()) } @@ -183,6 +190,9 @@ func grantFreshLoop(it *factory.Item, ph *factory.Phase) { // nothing here starts a run: the job is queued for a worker to pull. Every // other question is kept for the step's next round, as KeepAnswer does. func AnswerUnheld(it *factory.Item, yes bool, words string, now time.Time) error { + if err := followAnswerRefusal(*it, yes, words); err != nil { + return err + } if it.QKind == "cap" && factory.IsContinue(yes, words) { ReleaseCap(it, now) return nil diff --git a/internal/factory/run/step_control.go b/internal/factory/run/step_control.go index a29563bf7e..54948aac15 100644 --- a/internal/factory/run/step_control.go +++ b/internal/factory/run/step_control.go @@ -56,6 +56,9 @@ func stepControl(it *factory.Item, name string, rerun bool, words, model string, return StepControlRefusal("job budget reached; raise Budget before trying again") } if rerun { + if err := unchangedFollowRefusal(*it, s.Phases[index]); err != nil { + return err + } it.Recovery = nil previous := s.Phases[index] for i := index; i < len(s.Phases); i++ { @@ -66,6 +69,9 @@ func stepControl(it *factory.Item, name string, rerun bool, words, model string, // round marks the stream begun, even when retrying the first step, so // worker admission revives it without erasing spend or earlier results. target := &s.Phases[index] + if previous.Result != nil && refusedCheckout(*previous.Result) != nil { + target.Result = previous.Result + } target.Round = max(previous.Round+1, 1) grantFreshLoop(it, target) target.Rounds = append([]factory.RoundLine(nil), previous.Rounds...) From 585067133ab99fe10decf9aeb459a83ff6eb2ac0 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:51:28 -0500 Subject: [PATCH 06/28] config: Model Pool stays off until someone turns it on (#2257) * test: require explicit Model Pool opt-in Co-Authored-By: Claude Opus 5.5 * fix: keep Model Pool off until explicitly enabled Default the shared resolver to off, including CI, while preserving saved choices and environment overrides. Align settings and manual copy and opt pool fixtures in explicitly. Co-Authored-By: Claude Opus 5.5 * test: exercise pool default through production app boot Use the production app opener so the provider and record assertions cover the real pool startup sweep without one-model mode masking it. Explicitly enable the two pool push fixtures. Co-Authored-By: Claude Opus 5.5 * test: explicitly enable pool for exec pending records The restart-sweep fixture intentionally tests pool records; opt it in now that untouched profiles stay off. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * config: Model Pool off says only what the switch does Off stops pool judging and sharing. It does not hold task roles to the chat model: in Auto the router still picks from the allowed models. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- README.md | 5 +- cmd/codeaf/appfactory_pool_default_test.go | 128 ++++++++++++++++++ cmd/codeaf/exec_pending_test.go | 1 + cmd/codeaf/pool_test.go | 42 +++--- cmd/codeaf/poolinstall_test.go | 4 +- cmd/codeaf/poolleak_test.go | 1 + docs/TELEMETRY.md | 2 +- .../unreleased/2257-model-pool-default-off.md | 14 ++ internal/config/settings.go | 6 +- internal/config/settings_presentation.go | 2 +- internal/config/settings_test.go | 12 +- internal/pool/poolcfg/poolcfg.go | 24 +--- internal/pool/poolcfg/poolcfg_test.go | 65 +++++---- 13 files changed, 219 insertions(+), 87 deletions(-) create mode 100644 cmd/codeaf/appfactory_pool_default_test.go create mode 100644 docs/changes/unreleased/2257-model-pool-default-off.md diff --git a/README.md b/README.md index a4718822b7..18e45bfa64 100644 --- a/README.md +++ b/README.md @@ -188,8 +188,9 @@ ChatGPT plan, Ollama and any OpenAI-compatible endpoint. ## Model Pool -Installs can pool what they have measured about models. It is on by -default: what an install sends is computed, text-free numbers about the models +Installs can pool what they have measured about models. It is off by +default. Turn it on with `model_pool = on` in settings or +`CODEAF_MODEL_POOL=on`. When enabled, what an install sends is computed, text-free numbers about the models it ran (role, model, a number, which model judged, door, size bucket, day) under a per-install nonce, never code, prompts, paths or an identity, and `codeaf pool status` shows exactly what is diff --git a/cmd/codeaf/appfactory_pool_default_test.go b/cmd/codeaf/appfactory_pool_default_test.go new file mode 100644 index 0000000000..b4fff82d04 --- /dev/null +++ b/cmd/codeaf/appfactory_pool_default_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestAppFactoryModelPoolRequiresOptIn(t *testing.T) { + for _, tc := range []struct { + name, saved, env, ci string + want poolcfg.Mode + source string + }{ + {name: "never chose", want: poolcfg.Off, source: "default"}, + {name: "CI never chose", ci: "true", want: poolcfg.Off, source: "default"}, + {name: "saved on", saved: "on", want: poolcfg.On, source: "setting"}, + {name: "saved off", saved: "off", want: poolcfg.Off, source: "setting"}, + {name: "env on", env: "on", saved: "off", want: poolcfg.On, source: "env"}, + {name: "env off", env: "off", saved: "on", want: poolcfg.Off, source: "env"}, + } { + t.Run(tc.name, func(t *testing.T) { + profile := t.TempDir() + if tc.saved != "" { + raw, _ := json.Marshal(map[string]string{config.KeyModelPool: tc.saved}) + if err := os.WriteFile(filepath.Join(profile, "config.json"), raw, 0600); err != nil { + t.Fatal(err) + } + } + got := config.ModelPoolResolved(profile, func(key string) (string, bool) { + value, ok := map[string]string{"CODEAF_MODEL_POOL": tc.env, "CI": tc.ci}[key] + return value, ok + }) + if got.Mode != tc.want || got.Source.Mode != tc.source { + t.Fatalf("pool mode=%s source=%s; want %s from %s", got.Mode, got.Source.Mode, tc.want, tc.source) + } + if tc.want == poolcfg.Off && (got.CanRead() || got.CanSend()) { + t.Fatal("off pool permits routing, judging or records") + } + }) + } + t.Run("configured provider answers and is recorded", func(t *testing.T) { + s := newLocalSetup(t) + t.Setenv("CODEAF_MODEL_POOL", "") + t.Setenv("CI", "") + const model = "deepseek/deepseek-v4.1-flash" + var calls atomic.Int64 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + fmt.Fprint(w, `{"data":[]}`) + return + } + var req struct { + Model string `json:"model"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + if req.Model != model { + t.Errorf("requested model=%q, want %q; pool must not swap vendors", req.Model, model) + } + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"model": model, "choices": []any{map[string]any{"message": map[string]any{"content": briefDraft}, "finish_reason": "stop"}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + })) + t.Cleanup(endpoint.Close) + profile := v3Profile(t, map[string]any{ + "model.talk": model, "models.tiers.low": model, + "models.tiers.reflex": model, "models.tiers.worker": model, + "models.tiers.high": model, "models.tiers.mastermind": model, + }) + t.Setenv(config.ProfileDirEnv, profile) + t.Setenv("OPENROUTER_API_KEY", "fixture") + t.Setenv("CODEAF_BASE_URL", endpoint.URL) + t.Setenv("CODEAF_MODEL_POOL_URL", endpoint.URL+"/index.json") + t.Setenv("CODEAF_MODEL_POOL_MIRROR_URL", "") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", endpoint.URL+"/rows") + freshEngineProcess(t) + s.backend.open = appOpenInProcess + st := s.start() + floor := s.backend.local.host.floor + it, err := floor.Create(factory.Item{Model: model, Repo: "acme/api", Num: 7, Title: "Fix retries", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: 1, Stream: &factory.Stream{Phases: []factory.Phase{{Name: "test", Kind: factory.StageCheck, State: factory.PhaseDone, Round: 1, Result: &factory.StageResult{Done: true, Tests: &factory.TestCounts{Passed: 3}}}}}}) + if err != nil { + t.Fatal(err) + } + var answer appwire.FactoryJobBriefWritten + s.must("factory.job_brief_write", map[string]any{"factory": st.Factory, "job": it.ID}, &answer) + if !answer.Kept || answer.Model != model || calls.Load() != 1 { + t.Fatalf("answer=%+v provider calls=%d", answer, calls.Load()) + } + if !session.FlushUsage() { + t.Fatal("usage did not flush") + } + rows, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + recorded := 0 + for _, row := range rows { + if row.FactoryJob == factory.LedgerName(it) { + recorded += row.Calls + if row.Model != model || row.Seat == session.SeatJudge { + t.Errorf("unexpected recorded model or judge: %+v", row) + } + } + } + if recorded != 1 { + t.Fatalf("recorded calls=%d, want 1", recorded) + } + // Join the production startup sweep before checking that it wrote nothing. + closeEngineProcess() + if _, err := os.Stat(config.ProfilePath(profile, "pool")); !os.IsNotExist(err) { + t.Fatalf("off pool created records: %v", err) + } + }) +} diff --git a/cmd/codeaf/exec_pending_test.go b/cmd/codeaf/exec_pending_test.go index 84ea994067..51f3025155 100644 --- a/cmd/codeaf/exec_pending_test.go +++ b/cmd/codeaf/exec_pending_test.go @@ -110,6 +110,7 @@ func readPendingRows(t *testing.T, profileDir string) []pendingLanding { // unverified state a judge scores; a pool that forbids reading leaves none; and // two runs never share an id, because the restart sweep dedups on it. func TestExecLeavesAPendingJudgeRecordForTheRestartSweep(t *testing.T) { + t.Setenv("CODEAF_MODEL_POOL", "on") model, profileDir := execPendingHome(t) envelope := runOnePendingExec(t, model) diff --git a/cmd/codeaf/pool_test.go b/cmd/codeaf/pool_test.go index b9469e1538..86775036ec 100644 --- a/cmd/codeaf/pool_test.go +++ b/cmd/codeaf/pool_test.go @@ -183,7 +183,7 @@ func TestPoolShowPrintsTheConfigAndSaysWhenNoIndexIsCached(t *testing.T) { } body := out.String() for _, want := range []string{ - "mode on · default", + "mode off · default", "relay https://codeaf.agentfield.ai/pool · default", "index https://codeaf.agentfield.ai/pool/index.json · default", "mirror https://raw.githubusercontent.com/Agent-Field/CodeAF/model-pool/pool/index.json · default", @@ -239,7 +239,7 @@ func TestPoolShowJSONWithASeededIndexReportsTheDocument(t *testing.T) { if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { t.Fatalf("--json did not parse: %v\n%s", err, out.String()) } - if answer.Mode != "on" || answer.ModeSource != "default" { + if answer.Mode != "off" || answer.ModeSource != "default" { t.Fatalf("mode = %q from %q", answer.Mode, answer.ModeSource) } if answer.IndexURL == "" || answer.SubmitURL == "" || answer.TTLSeconds != 86400 { @@ -852,7 +852,7 @@ func TestPoolPublicKeysCarryTheIndexSignersKey(t *testing.T) { prev := poolRefreshGo poolRefreshGo = func(scope string, fn func()) { started++ } t.Cleanup(func() { poolRefreshGo = prev }) - startPoolIndexRefresh(context.Background(), t.TempDir(), poolcfg.Resolve("", "", noEnv), poolPublicKeys) + startPoolIndexRefresh(context.Background(), t.TempDir(), poolcfg.Resolve("on", "", noEnv), poolPublicKeys) if started != 1 { t.Fatalf("the build's key started %d refresh(es) under mode on, want 1", started) } @@ -880,7 +880,7 @@ func TestPoolTrustedKeysFollowTheStoredKey(t *testing.T) { } } - built, err := poolTrustedKeysErr(poolcfg.Resolve("", "", noEnv)) + built, err := poolTrustedKeysErr(poolcfg.Resolve("on", "", noEnv)) if err != nil || len(built) != 1 || !bytes.Equal(built[0], poolPublicKeys[0]) { t.Fatalf("with nothing stored, the build's own key is the key (err %v)", err) } @@ -893,7 +893,7 @@ func TestPoolVerifyFetchesUnderTheBuiltInKeyAndNamesWhatFailed(t *testing.T) { dir := t.TempDir() // A dead port is the fetch that would have happened: if the door let one // through, the failure below would be a fetch error, not the sentence. - lookup := oneEnv("CODEAF_MODEL_POOL_URL", "http://127.0.0.1:1/index.json") + lookup := poolEnv(map[string]string{"CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": "http://127.0.0.1:1/index.json"}) var out strings.Builder err := runPoolWith([]string{"verify"}, &out, dir, poolClock(t), lookup) if err == nil { @@ -930,7 +930,7 @@ func TestPoolVerifyRefusesWhenThePoolIsOff(t *testing.T) { func TestPoolVerifyRefusesAKeyItCannotRead(t *testing.T) { for _, bad := range []string{"not base64!!", "abcd"} { var out strings.Builder - err := runPoolWith([]string{"verify", "--key", bad}, &out, t.TempDir(), poolClock(t), noEnv) + err := runPoolWith([]string{"verify", "--key", bad}, &out, t.TempDir(), poolClock(t), oneEnv("CODEAF_MODEL_POOL", "on")) if !errors.Is(err, exitCannotRun) { t.Fatalf("a key %q is a door refusal, not a fetch: %v", bad, err) } @@ -957,6 +957,7 @@ func TestPoolVerifyChecksOnlyUnderTheKeyItIsGiven(t *testing.T) { t.Fatal(err) } lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -1008,7 +1009,7 @@ func TestPoolVerifyNamesAStoredKeyThatDoesNotDecode(t *testing.T) { func TestPoolVerifyNamesTheEnvPinWhenItsKeyDoesNotDecode(t *testing.T) { var out strings.Builder err := runPoolWith([]string{"verify"}, &out, t.TempDir(), poolClock(t), - oneEnv("CODEAF_MODEL_POOL_PUBLIC_KEY", "not-a-key")) + poolEnv(map[string]string{"CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_PUBLIC_KEY": "not-a-key"})) if !errors.Is(err, exitIncomplete) { t.Fatalf("a pin that does not decode is a door refusal, not a fetch: %v", err) } @@ -1035,7 +1036,7 @@ func TestPoolVerifyFetchesAndChecksASignedIndex(t *testing.T) { []byte(base64.StdEncoding.EncodeToString(sig)), 0o644); err != nil { t.Fatal(err) } - lookup := oneEnv("CODEAF_MODEL_POOL_URL", filepath.Join(src, "index.json")) + lookup := poolEnv(map[string]string{"CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": filepath.Join(src, "index.json")}) dir := t.TempDir() var out strings.Builder if err := runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(pub)}, @@ -1073,7 +1074,7 @@ func TestPoolVerifyNamesTheMetricsItVerified(t *testing.T) { []byte(base64.StdEncoding.EncodeToString(sig)), 0o644); err != nil { t.Fatal(err) } - lookup := oneEnv("CODEAF_MODEL_POOL_URL", filepath.Join(src, "index.json")) + lookup := poolEnv(map[string]string{"CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": filepath.Join(src, "index.json")}) dir := t.TempDir() var out strings.Builder if err := runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(pub)}, @@ -1140,7 +1141,7 @@ func TestPoolVerifyRefusesADocumentItsKeyDoesNotTrust(t *testing.T) { []byte(base64.StdEncoding.EncodeToString(sig)), 0o644); err != nil { t.Fatal(err) } - lookup := oneEnv("CODEAF_MODEL_POOL_URL", filepath.Join(src, "index.json")) + lookup := poolEnv(map[string]string{"CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": filepath.Join(src, "index.json")}) dir := t.TempDir() var out strings.Builder err = runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(pub)}, @@ -1176,7 +1177,7 @@ func TestPoolRefreshStartsNoGoroutineWithoutAKey(t *testing.T) { t.Cleanup(func() { poolRefreshGo = prev }) key := []ed25519.PublicKey{make(ed25519.PublicKey, ed25519.PublicKeySize)} - on := poolcfg.Resolve("", "", noEnv) + on := poolcfg.Resolve("on", "", noEnv) off := poolcfg.Resolve("off", "", noEnv) startPoolIndexRefresh(context.Background(), t.TempDir(), on, nil) @@ -1212,15 +1213,8 @@ func stubPoolRefresh(t *testing.T) *int { func TestWirePoolIndexStartsTheRefreshAndThePush(t *testing.T) { started := stubPoolRefresh(t) t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/v1/rows") - // A CI environment answers the mode when neither the environment word nor - // the stored setting does: GitHub Actions exports CI=true, the resolver reads - // that as read-only, the push never starts, and the count below read one on - // every pull-request run while passing on every laptop. This test is about - // the DEFAULT mode, so the CI word is emptied for its duration; t.Setenv - // restores the runner's own value afterwards. - t.Setenv("CI", "") - - // The default mode is on: the refresh and the push both start. + // This test exercises both errands of an explicitly enabled pool. + t.Setenv("CODEAF_MODEL_POOL", "on") wirePoolIndex(t.TempDir()) if *started != 2 { t.Fatalf("a sending pool started %d errand(s) at start-up, want the refresh and the push", *started) @@ -1670,6 +1664,7 @@ func poolEnv(pairs map[string]string) func(string) (string, bool) { // at once and reaches no network. func deadEnv() func(string) (string, bool) { return poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": "http://127.0.0.1:1/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "http://127.0.0.1:1/index.json", }) @@ -1792,6 +1787,7 @@ func TestPoolStatusSaysTheRelayAnswered(t *testing.T) { } server := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -1829,6 +1825,7 @@ func TestPoolStatusSaysWhenThisRunCachedTheIndex(t *testing.T) { } server := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -1872,6 +1869,7 @@ func TestPoolStatusSaysWhenThisRunReplacedTheCachedIndex(t *testing.T) { } server := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -1901,6 +1899,7 @@ func TestPoolStatusJSONSaysWhenThisRunCachedTheIndex(t *testing.T) { } server := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -1969,6 +1968,7 @@ func TestPoolStatusFallsToTheMirrorWhenTheRelayIsDown(t *testing.T) { } mirror := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": "http://127.0.0.1:1/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": mirror.URL + "/index.json", }) @@ -1999,6 +1999,7 @@ func TestPoolStatusChecksOnlyUnderTheKeyItIsGiven(t *testing.T) { t.Fatal(err) } lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) @@ -2087,6 +2088,7 @@ func TestPoolStatusJSONCarriesTheRelayAndMirror(t *testing.T) { } server := poolServer(t, priv, signedPoolDoc(7)) lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL": "on", "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", "CODEAF_MODEL_POOL_MIRROR_URL": "", }) diff --git a/cmd/codeaf/poolinstall_test.go b/cmd/codeaf/poolinstall_test.go index 1b36ff06e2..44b65d6d9c 100644 --- a/cmd/codeaf/poolinstall_test.go +++ b/cmd/codeaf/poolinstall_test.go @@ -126,7 +126,7 @@ func TestPoolPushPostsPendingRowsUnderTheInstallNonce(t *testing.T) { dir := seedOutbox(t) submit := oneEnv("CODEAF_MODEL_POOL_SUBMIT_URL", srv.URL) - cfg := poolcfg.Resolve("", "", submit) + cfg := poolcfg.Resolve("on", "", submit) poolPush(context.Background(), dir, cfg, 2*time.Second) if len(installs) != 1 { @@ -162,7 +162,7 @@ func TestPoolPushLeavesTheRowsPendingWhenTheRelayDoesNotAnswer(t *testing.T) { // A dead loopback port: the address the relay would have answered at, if // the relay were there. submit := oneEnv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/v1/rows") - cfg := poolcfg.Resolve("", "", submit) + cfg := poolcfg.Resolve("on", "", submit) said, err := captureStdout(t, func() error { poolPush(context.Background(), dir, cfg, time.Second) diff --git a/cmd/codeaf/poolleak_test.go b/cmd/codeaf/poolleak_test.go index a226747c86..fef6408e51 100644 --- a/cmd/codeaf/poolleak_test.go +++ b/cmd/codeaf/poolleak_test.go @@ -32,6 +32,7 @@ import ( // fails here rather than passing a second later and racing the profile's own // clean-up. func TestTheLaunchesPoolWritersStopWhenTheProcessCloses(t *testing.T) { + t.Setenv("CODEAF_MODEL_POOL", "on") released := make(chan struct{}) var release sync.Once free := func() { release.Do(func() { close(released) }) } diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index 6312116e08..2721d3f419 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -133,7 +133,7 @@ and neither does a test binary. ## The Model Pool is a second stream, under its own switch The usage counts are not the only thing this binary sends to AgentField. With -`model_pool` set to `on` — the default — a judge scores each crew seat after a +`model_pool` explicitly set to `on` — it defaults to `off` — a judge scores each crew seat after a task lands, and one row per seat leaves for `https://codeaf.agentfield.ai/pool/v1/rows`: the model slug that held the seat, the judge's slug, the seat (the worker, checker or planner, spelled on the wire as `worker`, diff --git a/docs/changes/unreleased/2257-model-pool-default-off.md b/docs/changes/unreleased/2257-model-pool-default-off.md new file mode 100644 index 0000000000..61bf1475be --- /dev/null +++ b/docs/changes/unreleased/2257-model-pool-default-off.md @@ -0,0 +1,14 @@ +--- +kind: changed +title: Model Pool stays off until you choose it +pr: 2257 +surface: [engine, chat, app] +invalidates: + - "The Model Pool previously enabled itself on fresh profiles and used read mode in CI." +--- + +With no saved choice, the Model Pool is off, including in CI. It makes no +pool judge calls, routing or records. Each role still follows your task-model +and tier settings, and in Auto the router picks from your allowed models. +Explicit saved on, read and off choices remain honoured; CODEAF_MODEL_POOL +still overrides the saved choice. Settings and the manual now say off by default. diff --git a/internal/config/settings.go b/internal/config/settings.go index 40757c1704..a7dd84bd15 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -113,7 +113,7 @@ const ( KeyDocumentEngine = "document_engine" KeyVisionModel = "vision_model" // KeyModelPool is the stored word the pool's resolver takes: the same three - // answers the CODEAF_MODEL_POOL pin and a CI environment may give it + // answers the CODEAF_MODEL_POOL pin may give it // (internal/pool/poolcfg). The resolver beside the search rows is the one // place the process environment is read for the pool. KeyModelPool = "model_pool" @@ -2027,8 +2027,8 @@ func (s *Settings) build() []Setting { Setting{ Key: KeyModelPool, Category: CategoryModels, Kind: SettingChoice, Label: "model pool", Env: "CODEAF_MODEL_POOL", Choices: ModelPoolChoices, - Hint: "codeaf picks your models from the public Model Pool, and your runs improve it. " + - "On by default: what leaves is computed, text-free numbers under a per-install nonce, " + + Hint: "Off by default: no pool judging, recommendations or sharing. Task roles still follow your task-model settings. " + + "on: use and contribute computed, text-free numbers under a per-install nonce, " + "never code, prompts or paths. read: use the pool, send nothing. off: neither " + "(CODEAF_MODEL_POOL=off for one run).", read: func() string { return ModelPoolAt(dir).Mode.String() }, diff --git a/internal/config/settings_presentation.go b/internal/config/settings_presentation.go index 46bf026bc9..57b94e83d8 100644 --- a/internal/config/settings_presentation.go +++ b/internal/config/settings_presentation.go @@ -76,7 +76,7 @@ func (s Setting) ChatPresentation() SettingPresentation { p.Category, p.Label = "Privacy", "share anonymous usage reports" case KeyModelPool: p.Category, p.Label = "Privacy", "shared model recommendations" - p.Description = "Use and contribute receives model recommendations and shares text-free performance measurements. Use only sends nothing. Off does neither. Code, prompts and file paths are never included." + p.Description = "Off by default. Use and contribute receives model recommendations and shares text-free performance measurements. Use only sends nothing. Off does neither. Code, prompts and file paths are never included." case KeyModelPoolPublicKey: p.Category, p.Label, p.Advanced = "Privacy", "model pool signing key", true case KeyToolApprovalMode: diff --git a/internal/config/settings_test.go b/internal/config/settings_test.go index 498f41dacc..1df8a780b8 100644 --- a/internal/config/settings_test.go +++ b/internal/config/settings_test.go @@ -1331,7 +1331,7 @@ func TestThePromptProfileRowRoundTripsAndItsPinWinsForOneLaunch(t *testing.T) { // three words poolcfg.Resolve accepts from the environment, so a row and a // shell spell one thing — and the resolver in this package is the one place // the process environment is read for the pool. -func TestTheModelPoolRowDefaultsToOnAndFollowsItsStoredWordAndItsPin(t *testing.T) { +func TestTheModelPoolRowDefaultsToOffAndFollowsItsStoredWordAndItsPin(t *testing.T) { dir := t.TempDir() t.Setenv("CODEAF_MODEL_POOL", "") t.Setenv("CI", "") @@ -1348,10 +1348,10 @@ func TestTheModelPoolRowDefaultsToOnAndFollowsItsStoredWordAndItsPin(t *testing. if row.Category != CategoryModels || row.Kind != SettingChoice || row.Label != "model pool" { t.Fatalf("model pool row = %+v", row) } - if row.Value() != "on" { - t.Fatalf("the pool does not default to on: %q", row.Value()) + if row.Value() != "off" { + t.Fatalf("the pool does not default to off: %q", row.Value()) } - if got := ModelPoolAt(dir); got.Mode != poolcfg.On || got.Source.Mode != "default" { + if got := ModelPoolAt(dir); got.Mode != poolcfg.Off || got.Source.Mode != "default" { t.Fatalf("an untouched profile resolved %+v", got) } @@ -1393,8 +1393,8 @@ func TestTheModelPoolRowDefaultsToOnAndFollowsItsStoredWordAndItsPin(t *testing. if err := writeProfileValue(dir, KeyModelPool, "maybe"); err != nil { t.Fatal(err) } - if got := ModelPoolAt(dir); got.Mode != poolcfg.On { - t.Fatalf("a stale word did not fall back to on: %+v", got) + if got := ModelPoolAt(dir); got.Mode != poolcfg.Off { + t.Fatalf("a stale word did not fall back to off: %+v", got) } } diff --git a/internal/pool/poolcfg/poolcfg.go b/internal/pool/poolcfg/poolcfg.go index 4886d71c25..2a5d26b452 100644 --- a/internal/pool/poolcfg/poolcfg.go +++ b/internal/pool/poolcfg/poolcfg.go @@ -54,8 +54,8 @@ func (m Mode) String() string { } // Sources records where each resolved value came from. Every field holds one -// of four words: "default", "setting", "env" or "ci" — and Mode alone may hold -// a fifth, "telemetry", when the telemetry off switch capped it at Read. +// of three words: "default", "setting" or "env" — and Mode alone may hold +// "telemetry", when the telemetry off switch capped it at Read. type Sources struct { Mode string RelayURL string @@ -87,7 +87,7 @@ type Config struct { Source Sources } -// The eleven names Resolve reads, and no others. The first three are the +// The ten names Resolve reads, and no others. The first three are the // telemetry off switch's environment rungs, spelled exactly as // internal/telemetry's ladder spells them — the variable, the ecosystem's // word, and an endpoint set to nothing — so the one switch the notice names @@ -97,7 +97,6 @@ const ( envDoNotTrack = "DO_NOT_TRACK" envEndpoint = "CODEAF_TELEMETRY_ENDPOINT" envMode = "CODEAF_MODEL_POOL" - envCI = "CI" envRelay = "CODEAF_MODEL_POOL_RELAY_URL" envIndex = "CODEAF_MODEL_POOL_URL" envSubmit = "CODEAF_MODEL_POOL_SUBMIT_URL" @@ -111,7 +110,6 @@ const ( srcDefault = "default" srcSetting = "setting" srcEnv = "env" - srcCI = "ci" // srcTelemetry is the Mode source when the telemetry off switch capped // an On at Read: the pool still reads, and sends nothing. srcTelemetry = "telemetry" @@ -148,7 +146,6 @@ func Resolve(setting, publicKey string, lookup func(name string) (value string, return strings.TrimSpace(value), true } modeWord, _ := get(envMode) - ciWord, _ := get(envCI) telemetryWord, _ := get(envTelemetry) doNotTrackWord, _ := get(envDoNotTrack) endpointWord, endpointSet := get(envEndpoint) @@ -160,16 +157,14 @@ func Resolve(setting, publicKey string, lookup func(name string) (value string, keyWord, _ := get(envKey) // The mode is the first of these that answers, and that answer is its - // source: the environment, the stored setting, CI, then the default. A + // source: the environment, the stored setting, then off. A // word neither the environment nor the setting knows is simply not an // answer and the next line gets its turn. - mode, modeSrc := On, srcDefault + mode, modeSrc := Off, srcDefault if m, ok := parseMode(modeWord); ok { mode, modeSrc = m, srcEnv } else if m, ok := parseMode(strings.TrimSpace(setting)); ok { mode, modeSrc = m, srcSetting - } else if ciSaysYes(ciWord) { - mode, modeSrc = Read, srcCI } // THE TELEMETRY OFF SWITCH WINS OVER EVERY ANSWER ABOVE, an explicit // `on` included. The notice says "Turn off: CODEAF_TELEMETRY=off" without @@ -314,15 +309,6 @@ func (c Config) Quieted() Config { return c } -// ciSaysYes reads the CI word: one of true, 1 or yes, without regard to case. -func ciSaysYes(word string) bool { - switch strings.ToLower(word) { - case "true", "1", "yes": - return true - } - return false -} - // acceptURL reads one of the four shapes a pool address may take, and nothing // else: an "https://" URL, an "http://" URL whose host is localhost or // 127.0.0.1 with or without a port, a "file://" URL, or an absolute filesystem diff --git a/internal/pool/poolcfg/poolcfg_test.go b/internal/pool/poolcfg/poolcfg_test.go index 285728f851..4d5e9808e3 100644 --- a/internal/pool/poolcfg/poolcfg_test.go +++ b/internal/pool/poolcfg/poolcfg_test.go @@ -36,7 +36,7 @@ func TestModeString(t *testing.T) { func TestResolveDefaults(t *testing.T) { want := Config{ - Mode: On, + Mode: Off, RelayURL: DefaultRelayURL, IndexURL: DefaultIndexURL, SubmitURL: DefaultSubmitURL, @@ -85,8 +85,8 @@ func TestResolveMode(t *testing.T) { {"setting on", "on", envOf(nil), On, "setting"}, {"setting read", "read", envOf(nil), Read, "setting"}, {"setting padded and mixed case", " Off\t", envOf(nil), Off, "setting"}, - {"setting unknown word falls through", "sometimes", envOf(nil), On, "default"}, - {"setting blank falls through", " ", envOf(nil), On, "default"}, + {"setting unknown word falls through", "sometimes", envOf(nil), Off, "default"}, + {"setting blank falls through", " ", envOf(nil), Off, "default"}, {"env beats setting", "off", envOf(map[string]string{"CODEAF_MODEL_POOL": "on"}), On, "env"}, } for _, c := range cases { @@ -105,20 +105,20 @@ func TestResolveModeFromCI(t *testing.T) { want Mode wantSrc string }{ - {"ci true", "", envOf(map[string]string{"CI": "true"}), Read, "ci"}, - {"ci one", "", envOf(map[string]string{"CI": "1"}), Read, "ci"}, - {"ci yes", "", envOf(map[string]string{"CI": "yes"}), Read, "ci"}, - {"ci padded and mixed case", "", envOf(map[string]string{"CI": " YES "}), Read, "ci"}, - {"ci false is not an answer", "", envOf(map[string]string{"CI": "false"}), On, "default"}, - {"ci zero is not an answer", "", envOf(map[string]string{"CI": "0"}), On, "default"}, - {"ci no is not an answer", "", envOf(map[string]string{"CI": "no"}), On, "default"}, - {"ci set and empty is not an answer", "", envOf(map[string]string{"CI": ""}), On, "default"}, - {"ci reads when the setting is unknown", "maybe", envOf(map[string]string{"CI": "true"}), Read, "ci"}, - {"ci reads when the setting is blank", " ", envOf(map[string]string{"CI": "1"}), Read, "ci"}, + {"ci true", "", envOf(map[string]string{"CI": "true"}), Off, "default"}, + {"ci one", "", envOf(map[string]string{"CI": "1"}), Off, "default"}, + {"ci yes", "", envOf(map[string]string{"CI": "yes"}), Off, "default"}, + {"ci padded and mixed case", "", envOf(map[string]string{"CI": " YES "}), Off, "default"}, + {"ci false is not an answer", "", envOf(map[string]string{"CI": "false"}), Off, "default"}, + {"ci zero is not an answer", "", envOf(map[string]string{"CI": "0"}), Off, "default"}, + {"ci no is not an answer", "", envOf(map[string]string{"CI": "no"}), Off, "default"}, + {"ci set and empty is not an answer", "", envOf(map[string]string{"CI": ""}), Off, "default"}, + {"ci stays off when the setting is unknown", "maybe", envOf(map[string]string{"CI": "true"}), Off, "default"}, + {"ci stays off when the setting is blank", " ", envOf(map[string]string{"CI": "1"}), Off, "default"}, {"setting beats ci", "off", envOf(map[string]string{"CI": "true"}), Off, "setting"}, {"env on beats ci", "off", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "on"}), On, "env"}, {"env off beats ci", "on", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "off"}), Off, "env"}, - {"env unknown and setting unknown leave ci", "maybe", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "maybe"}), Read, "ci"}, + {"unknown words stay off in ci", "maybe", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "maybe"}), Off, "default"}, } for _, c := range cases { got := Resolve(c.setting, "", c.lookup) @@ -130,8 +130,7 @@ func TestResolveModeFromCI(t *testing.T) { // TestResolveModeUnderTheTelemetryOffSwitch is the law the notice rests on: // "Turn off: CODEAF_TELEMETRY=off" is true of everything that leaves, so the -// switch caps the pool at Read — over the default, over CI, and over an -// explicit `on` from the setting or the environment — and leaves Read and Off +// switch caps the pool at Read for an explicit `on` from the setting or the environment — and leaves Read and Off // as they were, because they already send nothing. func TestResolveModeUnderTheTelemetryOffSwitch(t *testing.T) { cases := []struct { @@ -141,22 +140,22 @@ func TestResolveModeUnderTheTelemetryOffSwitch(t *testing.T) { want Mode wantSrc string }{ - {"telemetry off caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "telemetry"}, - {"telemetry 0 caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY": "0"}), Read, "telemetry"}, - {"telemetry false, padded and mixed case", "", envOf(map[string]string{"CODEAF_TELEMETRY": " False "}), Read, "telemetry"}, - {"do not track 1 caps the default", "", envOf(map[string]string{"DO_NOT_TRACK": "1"}), Read, "telemetry"}, - {"do not track true caps the default", "", envOf(map[string]string{"DO_NOT_TRACK": "true"}), Read, "telemetry"}, - {"an empty endpoint caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": ""}), Read, "telemetry"}, + {"telemetry off leaves the default off", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Off, "default"}, + {"telemetry 0 leaves the default off", "", envOf(map[string]string{"CODEAF_TELEMETRY": "0"}), Off, "default"}, + {"telemetry false, padded and mixed case", "", envOf(map[string]string{"CODEAF_TELEMETRY": " False "}), Off, "default"}, + {"do not track 1 leaves the default off", "", envOf(map[string]string{"DO_NOT_TRACK": "1"}), Off, "default"}, + {"do not track true leaves the default off", "", envOf(map[string]string{"DO_NOT_TRACK": "true"}), Off, "default"}, + {"an empty endpoint leaves the default off", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": ""}), Off, "default"}, {"a blank endpoint caps an explicit env on", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": " ", "CODEAF_MODEL_POOL": "on"}), Read, "telemetry"}, - {"a set endpoint does not cap", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": "https://example.test/t"}), On, "default"}, + {"a set endpoint does not cap", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": "https://example.test/t"}), Off, "default"}, {"telemetry off caps an explicit setting on", "on", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "telemetry"}, {"telemetry off caps an explicit env on", "off", envOf(map[string]string{"CODEAF_TELEMETRY": "off", "CODEAF_MODEL_POOL": "on"}), Read, "telemetry"}, {"telemetry off leaves read as read", "read", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "setting"}, {"telemetry off leaves off as off", "off", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Off, "setting"}, - {"telemetry off leaves ci as ci", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off", "CI": "true"}), Read, "ci"}, - {"telemetry on is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": "on"}), On, "default"}, - {"telemetry set and empty is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": ""}), On, "default"}, - {"do not track 0 is not an answer", "", envOf(map[string]string{"DO_NOT_TRACK": "0"}), On, "default"}, + {"telemetry off leaves ci off", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off", "CI": "true"}), Off, "default"}, + {"telemetry on is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": "on"}), Off, "default"}, + {"telemetry set and empty is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": ""}), Off, "default"}, + {"do not track 0 is not an answer", "", envOf(map[string]string{"DO_NOT_TRACK": "0"}), Off, "default"}, } for _, c := range cases { got := Resolve(c.setting, "", c.lookup) @@ -514,7 +513,7 @@ func TestCanSendAndCanRead(t *testing.T) { func TestResolvedConfigCanSendAndCanRead(t *testing.T) { // An empty submit address turns sending off while everything else stays on. - got := Resolve("", "", envOf(map[string]string{"CODEAF_MODEL_POOL_SUBMIT_URL": ""})) + got := Resolve("on", "", envOf(map[string]string{"CODEAF_MODEL_POOL_SUBMIT_URL": ""})) if got.SubmitURL != "" || got.Source.SubmitURL != "env" { t.Fatalf("SubmitURL = %q from %q, want %q from %q", got.SubmitURL, got.Source.SubmitURL, "", "env") } @@ -525,8 +524,8 @@ func TestResolvedConfigCanSendAndCanRead(t *testing.T) { t.Error("an empty submit address must still read") } - if got := Resolve("", "", envOf(nil)); !got.CanSend() || !got.CanRead() { - t.Errorf("the defaults must send and read: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) + if got := Resolve("on", "", envOf(nil)); !got.CanSend() || !got.CanRead() { + t.Errorf("explicit on must send and read: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) } if got := Resolve("read", "", envOf(nil)); got.CanSend() || !got.CanRead() { t.Errorf("read must read and not send: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) @@ -546,18 +545,18 @@ func TestResolveNilLookupUsesSetting(t *testing.T) { } } -// TestResolveAsksElevenNames pins the whole of what Resolve reads: the eight +// TestResolveAsksTenNames pins the whole of what Resolve reads: the seven // pool names, and the three environment rungs of the telemetry off switch, // which the pool obeys so the one switch the notice names stops everything // that leaves. -func TestResolveAsksElevenNames(t *testing.T) { +func TestResolveAsksTenNames(t *testing.T) { var asked []string Resolve("", "", func(name string) (string, bool) { asked = append(asked, name) return "", false }) sort.Strings(asked) - want := []string{envMode, envCI, envRelay, envIndex, envSubmit, envMirror, envTTL, envKey, envTelemetry, envDoNotTrack, envEndpoint} + want := []string{envMode, envRelay, envIndex, envSubmit, envMirror, envTTL, envKey, envTelemetry, envDoNotTrack, envEndpoint} sort.Strings(want) if !reflect.DeepEqual(asked, want) { t.Errorf("Resolve asked lookup for %v, want %v", asked, want) From 76c968146920f5aa5c064f5de63eef1f92fd449d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 2 Nov 2026 20:00:43 -0500 Subject: [PATCH 07/28] perf(knowledge): capture wakes on the chat's own writes, not a per-chat 1 s ticker (#2255) * knowledge: capture wakes on the chat's own journal writes After #2042 an idle chat's capture tick opened no database, but it still woke every second and opened, read and closed the chat's journal. It was the largest per-chat timer left in an idle engine. The journal writer's hook now wakes capture after each line reaches the file (an in-process signal; no file watch). Each write is followed by one trailing look a second later, a look that moves the cursor is followed by another, so a backlog still moves one batch a second, and a five-second safety tick covers lines another process writes to the same journal and consent changed elsewhere. A look a write woke uses a consent answer at most a second old (CaptureConsentWithin), so a turn written just after consent changed elsewhere is captured on the trailing look. An idle chat now runs one five-second tick and nothing else. Co-Authored-By: Claude Opus 5.5 * docs: change entry for #2048 Co-Authored-By: Claude Opus 5.5 * knowledge: a new capture scanner always takes one trailing look Review found that a rebind could leave a write without its trailing look: the retired scanner may consume the wake and answer from a consent view under a second old, and the replacement's first look moved nothing, so it armed no trailing look and capture waited for the safety tick. Every scanner now looks once when it starts and once more a second later with fresh consent. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../2048-knowledge-capture-on-write.md | 12 ++ internal/knowledge/store.go | 18 ++- internal/session/agent.go | 2 +- internal/session/knowledge.go | 96 ++++++++++++++-- internal/session/knowledge_consent_test.go | 6 +- internal/session/knowledge_wake_test.go | 106 ++++++++++++++++++ internal/session/session.go | 4 + 7 files changed, 227 insertions(+), 17 deletions(-) create mode 100644 docs/changes/unreleased/2048-knowledge-capture-on-write.md create mode 100644 internal/session/knowledge_wake_test.go diff --git a/docs/changes/unreleased/2048-knowledge-capture-on-write.md b/docs/changes/unreleased/2048-knowledge-capture-on-write.md new file mode 100644 index 0000000000..c4aaa49c62 --- /dev/null +++ b/docs/changes/unreleased/2048-knowledge-capture-on-write.md @@ -0,0 +1,12 @@ +--- +kind: changed +title: Knowledge capture wakes on the chat's own writes, not every second +pr: 2048 +surface: [chat, engine] +invalidates: + - "Each live chat's knowledge capture woke every second and read its journal, even while idle. It now wakes when the chat writes its journal, looks once more a second later, and otherwise runs one five-second safety tick." +--- + +Lines this process writes are captured within about a second, as before. Lines +another process writes to the same journal are captured within five seconds, +and consent changed in another process with no write here within ten. diff --git a/internal/knowledge/store.go b/internal/knowledge/store.go index 383a4c644f..5c83485010 100644 --- a/internal/knowledge/store.go +++ b/internal/knowledge/store.go @@ -68,23 +68,33 @@ func (v *Vault) captureConsentView() *consentView { // It answers whether a tick has anything to look at; a tick that finds sources // to admit asks [Vault.ConfirmConsent] before it admits them or moves past them. func (v *Vault) CaptureConsent() (enabled bool, since time.Time) { - return v.consent(false) + return v.consent(consentRecheck) +} + +// CaptureConsentWithin is CaptureConsent for a look that has a reason to be +// prompt, such as a chat that has just written: the shared answer is used only +// while it is younger than maxAge. The view is shared, so however many chats +// write, this opens the database at most once per maxAge. +func (v *Vault) CaptureConsentWithin(maxAge time.Duration) (enabled bool, since time.Time) { + return v.consent(min(maxAge, consentRecheck)) } // ConfirmConsent reads consent now and shares the answer with the other // tickers. Admission uses it, so a source is judged against the database's own // opt-in time, never a remembered one. func (v *Vault) ConfirmConsent() (enabled bool, since time.Time) { - return v.consent(true) + return v.consent(0) } -func (v *Vault) consent(fresh bool) (enabled bool, since time.Time) { +// consent answers from the shared view while it is younger than maxAge and no +// consent write of this process came after it; maxAge 0 always reads. +func (v *Vault) consent(maxAge time.Duration) (enabled bool, since time.Time) { view := v.captureConsentView() view.mu.Lock() defer view.mu.Unlock() now := consentNow() generation := view.generation.Load() - if !fresh && !view.checked.IsZero() && view.readGeneration == generation && now.Sub(view.checked) < consentRecheck { + if maxAge > 0 && !view.checked.IsZero() && view.readGeneration == generation && now.Sub(view.checked) < maxAge { return view.enabled, view.since } // Keep the lock across the read so simultaneous tickers share failures too. diff --git a/internal/session/agent.go b/internal/session/agent.go index e9959c8a7a..131113cba1 100644 --- a/internal/session/agent.go +++ b/internal/session/agent.go @@ -317,7 +317,7 @@ func newAgent(config Config, client Completer) (*Agent, error) { } restored := replayed.messages agent.file = file - file.onWrite = agent.tellWrote + file.onWrite = func() { agent.tellWrote(); agent.wakeKnowledge() } agent.presentation = file.presentation agent.restoreProgramHold() // AND WHAT AN EARLIER PROCESS OF THIS SESSION MADE. It is the one thing diff --git a/internal/session/knowledge.go b/internal/session/knowledge.go index cc28a53dee..c4ead8753e 100644 --- a/internal/session/knowledge.go +++ b/internal/session/knowledge.go @@ -6,12 +6,34 @@ import ( "log" "strings" "sync" + "sync/atomic" "time" "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/knowledge" ) +// The test hook counts scans without changing the journal or the capture path. +var knowledgeScanHook atomic.Pointer[func(string)] + +// knowledgeSafetyTick bounds discovery of journal lines this process did not +// write, such as a session host's or terminal's writes. With consentRecheck, +// lines covered by consent changed elsewhere wait at most +// consentRecheck + knowledgeSafetyTick. +const knowledgeSafetyTick = 5 * time.Second + +// knowledgeCatchUp is the former capture cadence. A look follows a write burst +// by this long, and follows any look that moved the cursor, so a backlog keeps +// moving one batch a second and the timer stops once a look finds nothing new. +const knowledgeCatchUp = time.Second + +// knowledgeWroteConsentAge is how old the shared consent answer may be for a +// look a write woke: a turn written just after consent changed elsewhere is +// captured on the trailing look, a second later, not at the next safety tick. +// The answer is shared, so writing chats open the database at most once a +// second between them, and idle chats never. +const knowledgeWroteConsentAge = time.Second + // Knowledge is the profile-wide vault the conversation's tools read. // Capture is opt-in and only persisted, human-facing conversations write it. func (a *Agent) Knowledge() *knowledge.Vault { @@ -47,6 +69,7 @@ type knowledgeJournal struct { } func (a *Agent) initKnowledge() { + a.knowledgeWake = make(chan struct{}, 1) if a.config.knowledgeReadVault != nil { a.knowledgeVault = a.config.knowledgeReadVault return @@ -56,6 +79,16 @@ func (a *Agent) initKnowledge() { } a.knowledgeVault = knowledge.Open(config.ProfilePath(a.config.ProfileDir, "")) } + +// wakeKnowledge is called by the journal writer's hook. It never blocks or +// takes a lock, because writers call it with locks held. +func (a *Agent) wakeKnowledge() { + select { + case a.knowledgeWake <- struct{}{}: + default: + } +} + func (a *Agent) startKnowledge() { vault := a.Knowledge() if vault == nil || a.config.InTask || a.config.taskID != 0 || a.config.knowledgeReadVault != nil || a.file == nil { @@ -73,19 +106,56 @@ func (a *Agent) startKnowledge() { sessionID := a.id go func() { defer close(j.done) - ticker := time.NewTicker(time.Second) - defer ticker.Stop() + safety := time.NewTicker(knowledgeSafetyTick) + defer safety.Stop() + catchUpTimer := time.NewTimer(knowledgeCatchUp) + catchUpTimer.Stop() + defer catchUpTimer.Stop() + var catchUp <-chan time.Time + // look scans once and says whether it moved the cursor; wrote says a + // write woke it, or it trails one, so its consent answer is fresh. + look := func(wrote bool) bool { + j.mu.Lock() + before := j.offset + j.mu.Unlock() + a.scanKnowledge(ctx, j, path, sessionID, wrote) + j.mu.Lock() + defer j.mu.Unlock() + return j.offset != before + } + // trail asks for one more look knowledgeCatchUp from now. + trail := func() { + catchUpTimer.Reset(knowledgeCatchUp) + catchUp = catchUpTimer.C + } + // A resumed chat may already have durable lines waiting for capture, + // and a rebind may have taken over a write whose wake the retired + // scanner consumed: look now, and once more with fresh consent. + look(false) + trail() for { select { case <-ctx.Done(): // A bounded final copy helps small chats close immediately. The // process worker recovers the remaining durable journal later. finalCtx, finish := context.WithTimeout(context.Background(), time.Second) - a.scanKnowledge(finalCtx, j, path, sessionID) + a.scanKnowledge(finalCtx, j, path, sessionID, true) finish() return - case <-ticker.C: - a.scanKnowledge(ctx, j, path, sessionID) + case <-a.knowledgeWake: + // Every write is followed by one trailing look, which catches + // lines still being written and consent that changed elsewhere. + look(true) + trail() + case <-safety.C: + if look(false) { + trail() + } + case <-catchUp: + catchUp = nil + if look(true) { + trail() + } } } }() @@ -131,7 +201,10 @@ func (a *Agent) stopKnowledge() { // scanKnowledge reads complete journal lines by byte offset. Re-journaled // compaction windows are skipped, so stubs and summaries never become new // claims. IDs survive reopen/retry and do not conflate equal human messages. -func (a *Agent) scanKnowledge(ctx context.Context, j *knowledgeJournal, path, sessionID string) (err error) { +func (a *Agent) scanKnowledge(ctx context.Context, j *knowledgeJournal, path, sessionID string, wrote bool) (err error) { + if hook := knowledgeScanHook.Load(); hook != nil && *hook != nil { + (*hook)(path) + } j.mu.Lock() defer j.mu.Unlock() defer func() { @@ -142,10 +215,15 @@ func (a *Agent) scanKnowledge(ctx context.Context, j *knowledgeJournal, path, se j.lastError = "" } }() - // Idle chat tickers share consent reads instead of reopening SQLite each - // second. The shared answer only decides whether to look; sources found + // Idle chats share consent reads instead of each reopening SQLite. The + // shared answer only decides whether to look; sources found // below are admitted, or passed over, on consent read afresh. - enabled, since := j.vault.CaptureConsent() + enabled, since := false, time.Time{} + if wrote { + enabled, since = j.vault.CaptureConsentWithin(knowledgeWroteConsentAge) + } else { + enabled, since = j.vault.CaptureConsent() + } if !enabled { return nil } diff --git a/internal/session/knowledge_consent_test.go b/internal/session/knowledge_consent_test.go index e137536f9c..f3791b3300 100644 --- a/internal/session/knowledge_consent_test.go +++ b/internal/session/knowledge_consent_test.go @@ -184,6 +184,6 @@ func TestKnowledgeKeepsSourcesWhileTheVaultIsGone(t *testing.T) { } } -// consentWait is one consent recheck plus a capture tick and room for a loaded -// machine. -const consentWait = 8 * time.Second +// consentWait allows the documented bound for consent changed elsewhere: the +// vault's five-second consentRecheck plus knowledgeSafetyTick, with room for load. +const consentWait = 5*time.Second + knowledgeSafetyTick + 3*time.Second diff --git a/internal/session/knowledge_wake_test.go b/internal/session/knowledge_wake_test.go new file mode 100644 index 0000000000..a3a5a285f4 --- /dev/null +++ b/internal/session/knowledge_wake_test.go @@ -0,0 +1,106 @@ +package session + +import ( + "encoding/json" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/knowledge" +) + +func waitForKnowledgePending(t *testing.T, vault *knowledge.Vault, want int, within time.Duration) { + t.Helper() + deadline := time.Now().Add(within) + for { + snapshot, err := vault.Snapshot() + if err != nil { + t.Fatal(err) + } + if snapshot.Pending == want { + return + } + if time.Now().After(deadline) { + t.Fatalf("wanted %d pending sources within %s, got %+v", want, within, snapshot) + } + time.Sleep(20 * time.Millisecond) + } +} + +func TestKnowledgeCaptureFollowsTheChatsOwnWrites(t *testing.T) { + agent := consentChat(t, t.TempDir()) + vault := agent.Knowledge() + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + collect(t, mustSubmit(t, agent, "Explain how the queue keeps work durable")) + waitForKnowledgePending(t, vault, 2, time.Second) +} + +func TestAnIdleChatsCaptureRunsOnlyTheSafetyTick(t *testing.T) { + for _, enabled := range []bool{true, false} { + name := "capture off" + if enabled { + name = "capture on" + } + t.Run(name, func(t *testing.T) { + agent := consentChat(t, t.TempDir()) + path := agent.file.journalName() + var scans atomic.Int64 + hook := func(scanned string) { + if scanned == path { + scans.Add(1) + } + } + previous := knowledgeScanHook.Swap(&hook) + t.Cleanup(func() { knowledgeScanHook.Store(previous) }) + if enabled { + if err := agent.Knowledge().StartBuild(); err != nil { + t.Fatal(err) + } + } + collect(t, mustSubmit(t, agent, "Explain how the queue keeps work durable")) + if enabled { + waitForKnowledgePending(t, agent.Knowledge(), 2, 2*time.Second) + } + // Let the completed turn's catch-up finish before measuring idle scans. + time.Sleep(2200 * time.Millisecond) + scans.Store(0) + time.Sleep(6 * time.Second) + count := scans.Load() + t.Logf("idle chat (%s): %d scans in 6s", name, count) + if count > 2 { + t.Fatalf("idle chat scanned %d times; want <= 2", count) + } + }) + } +} + +func TestKnowledgeCapturesAnotherWritersLinesWithinTheSafetyTick(t *testing.T) { + agent := consentChat(t, t.TempDir()) + vault := agent.Knowledge() + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + collect(t, mustSubmit(t, agent, "Explain how the queue keeps work durable")) + waitForKnowledgePending(t, vault, 2, 2*time.Second) + // No turn or catch-up wake should be left to discover the external append. + time.Sleep(2200 * time.Millisecond) + file, err := os.OpenFile(agent.file.journalName(), os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + err = json.NewEncoder(file).Encode(map[string]any{ + "type": "message", "role": "user", "content": "Another writer keeps this source durable.", + "timestamp": time.Now().UTC().Format(time.RFC3339Nano), + }) + closeErr := file.Close() + if err != nil { + t.Fatal(err) + } + if closeErr != nil { + t.Fatal(closeErr) + } + waitForKnowledgePending(t, vault, 3, knowledgeSafetyTick+2*time.Second) +} diff --git a/internal/session/session.go b/internal/session/session.go index 23b1a00a3b..70e8b020c2 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -2765,6 +2765,10 @@ type Agent struct { knowledgeJournal *knowledgeJournal taskComputerPrompt bool remoteAckJobs sync.WaitGroup + // knowledgeWake is signalled after each journal line reaches the file. + // initKnowledge creates it once before the agent is reachable and never + // replaces it, so writers read it without a lock. + knowledgeWake chan struct{} // jailNotice keeps a host limit from becoming noise on every tool call. jailNotice sync.Once From e1a36924b006c76ee39b87abe93c701f0e0fc5ec Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 2 Nov 2026 20:01:22 -0500 Subject: [PATCH 08/28] tests: sync fixtures keep their socket homes short on macOS and wait for branch publication (#2032) Rebased onto app/v3, where #2178 and #2183 already cover most of this branch: the engine-backed fixtures and the hosted take-back's resumed-hold proof. What is left: - the hosted move rig and the drive rig's default daemon keep their homes under /tmp on Unix, because macOS's temporary folder leaves a workspace socket only a few bytes under the address limit; - the L12 death-branches test waits for A's branch row to hold its three orphan turns before it reads them. Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/hosted_takeback_test.go | 10 +++++++++- docs/changes/unreleased/2032-sync-engine-fixtures.md | 5 +++++ internal/syncsetup/drive_test.go | 10 ++++++++++ internal/syncsetup/twohomes_test.go | 4 ++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 docs/changes/unreleased/2032-sync-engine-fixtures.md diff --git a/cmd/codeaf/hosted_takeback_test.go b/cmd/codeaf/hosted_takeback_test.go index 993a2cef0b..3eef5ae626 100644 --- a/cmd/codeaf/hosted_takeback_test.go +++ b/cmd/codeaf/hosted_takeback_test.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -67,7 +68,14 @@ func (m moveRenewals) RenewalChanges() <-chan struct{} { return m.changes } func newHostedMoveRig(t *testing.T, wrap ...func(http.Handler) http.Handler) *hostedMoveRig { t.Helper() cfg, c := doorChat(t) - short, err := os.MkdirTemp("", "cf-move-") + // The workspace host's socket lives under this home. macOS's temporary + // folder leaves it only a few bytes under the socket limit, and none once + // that folder is spelled through /private, so Unix homes start at /tmp. + base := "" + if runtime.GOOS != "windows" { + base = "/tmp" + } + short, err := os.MkdirTemp(base, "cf-move-") if err != nil { t.Fatal(err) } diff --git a/docs/changes/unreleased/2032-sync-engine-fixtures.md b/docs/changes/unreleased/2032-sync-engine-fixtures.md new file mode 100644 index 0000000000..49cdb18f55 --- /dev/null +++ b/docs/changes/unreleased/2032-sync-engine-fixtures.md @@ -0,0 +1,5 @@ +--- +kind: internal +title: Sync fixtures keep their socket homes short on macOS and wait for branch publication +pr: 2032 +--- diff --git a/internal/syncsetup/drive_test.go b/internal/syncsetup/drive_test.go index 195bb77478..8cab8d45a4 100644 --- a/internal/syncsetup/drive_test.go +++ b/internal/syncsetup/drive_test.go @@ -498,6 +498,16 @@ func TestDriveSideStartsAsViewerWhenHeld(t *testing.T) { // daemon's socket appearing shows the daemon, and not the spawn, served them. func TestDriveSidePublishesOverTheDefaultDaemon(t *testing.T) { r := newDriveRig(t) + if runtime.GOOS != "windows" { + // The rig has not opened an engine transport yet. A short home keeps + // its socket inside the home even with macOS's small address limit. + short, err := os.MkdirTemp("/tmp", "cds") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(short) }) + t.Setenv(home.EnvVar, short) + } t.Setenv("CODEAF_ENGINE_DAEMON", "") r.engine.Transport = nil t.Cleanup(func() { stopDaemons(t) }) diff --git a/internal/syncsetup/twohomes_test.go b/internal/syncsetup/twohomes_test.go index e7c33fa0a3..082cae3369 100644 --- a/internal/syncsetup/twohomes_test.go +++ b/internal/syncsetup/twohomes_test.go @@ -493,6 +493,10 @@ func TestTwoHomesL12DeathBranches(t *testing.T) { h.block.set(false) waitFor(t, "A to learn it was superseded", func() bool { _, viewer := a.drive.Viewer(); return viewer }) + waitFor(t, "A's branch row to hold three orphan turns", func() bool { + branch, ok := branchRowOf(t, h.a, h.cell.ID) + return ok && branch.OrphanTurns == 3 + }) v, err := h.a.Dir.Cell(ctx, h.cell.ID) if err != nil || v.Cell.Head != parentHead || v.Cell.Lease.Device != h.b.Device.ID() { From a4d4a56c29a7899c318f8c233300024dd26bed1a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 2 Nov 2026 20:28:09 -0500 Subject: [PATCH 09/28] remote: an app chat's streamed text is folded before it is framed (#2043) * remote: an app chat's streamed text is folded before it is framed Every streamed delta of an in-process app chat crossed the remote framing, the in-memory pipe and about eight goroutine handoffs on its own. At 40 tok/s the deltas are 25 ms apart, so the app outbox's 16 ms window never folded anything. Engine.DeltaWindow (opt-in, set by the app's in-process and lent conversations to DefaultDeltaWindow, 33 ms) lets each stream's pump send the first chunk at once and fold the text that follows until the window ends or 4 KiB. Every non-text event sends the held text first and goes out at once; the channel closing flushes before "closed". Co-Authored-By: Claude Opus 5.5 * remote: the delta window's tests cover swap, a real reconnect and a steady stream A swap while text is held lets none of the old conversation through; a real client cut mid-burst and resumed sees each character once; with the production window a quiet tail arrives and a steady producer is never starved. The 4 KiB value is documented as the flush threshold it is, and the latency check keeps its median bound without a per-sample maximum that scheduler pauses could break. Co-Authored-By: Claude Opus 5.5 * docs: change entry for #2043 Co-Authored-By: Claude Opus 5.5 * remote: the delta window's swap and steady-stream tests stand on their own The swap test drives its own pump and joins only that one, so it no longer depends on the fake agent opening no other lanes. The steady-producer test bounds frames by the time the producer actually took and checks that text reaches the surface before the producer finishes, keeping the largest gap as a diagnostic. The change entry names the conversations it applies to: the ones the desktop's engine holds in process. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/app.go | 3 + cmd/codeaf/app_lend.go | 3 + .../unreleased/2043-app-delta-window.md | 18 + internal/remote/deltawindow.go | 137 +++ internal/remote/deltawindow_test.go | 805 ++++++++++++++++++ internal/remote/server.go | 21 +- 6 files changed, 985 insertions(+), 2 deletions(-) create mode 100644 docs/changes/unreleased/2043-app-delta-window.md create mode 100644 internal/remote/deltawindow.go create mode 100644 internal/remote/deltawindow_test.go diff --git a/cmd/codeaf/app.go b/cmd/codeaf/app.go index 64b3af9959..435f88a5b4 100644 --- a/cmd/codeaf/app.go +++ b/cmd/codeaf/app.go @@ -368,6 +368,9 @@ func appOpenInProcess(ask appOpenAsk) (*appConn, error) { } eng, err := assembleEngine(h, workspace, transcript, workspace, ask.IntentSiblingNames) if err == nil { + // In-process chats are the desktop's hot path. Remote surfaces keep + // one frame per event until someone measures them. + eng.DeltaWindow = remote.DefaultDeltaWindow err = enableAppConversationModels(eng.Agent, ask.Models) } if err == nil && ask.AutomaticModels != nil { diff --git a/cmd/codeaf/app_lend.go b/cmd/codeaf/app_lend.go index 353490cb26..af21a3518a 100644 --- a/cmd/codeaf/app_lend.go +++ b/cmd/codeaf/app_lend.go @@ -200,6 +200,9 @@ func appLendLive(agent *session.Agent, ask appOpenAsk, hello remote.Hello) (*app SessionFile: ask.Transcript, Resumed: true, } + // In-process chats are the desktop's hot path. Remote surfaces keep + // one frame per event until someone measures them. + engine.DeltaWindow = remote.DefaultDeltaWindow sess := remote.NewSession(engine, true) loop, err := remote.LoopbackAttach(hello, remote.AttachOptions{ Open: func(remote.Hello) (*remote.Session, error) { return sess, nil }, diff --git a/docs/changes/unreleased/2043-app-delta-window.md b/docs/changes/unreleased/2043-app-delta-window.md new file mode 100644 index 0000000000..c13fabab80 --- /dev/null +++ b/docs/changes/unreleased/2043-app-delta-window.md @@ -0,0 +1,18 @@ +--- +kind: changed +title: In-process desktop chats fold streamed text before it is framed +pr: 2043 +surface: [engine] +invalidates: + - "Previously every streamed chunk of a desktop chat crossed the remote framing as its own event frame; now a chat the desktop's engine holds in process folds text for up to 33 ms before framing it, and other surfaces keep one frame per event." +--- + +A conversation the desktop's engine holds in process (one it opens or lends) +sends the first chunk of streamed text at once +and folds the chunks that follow for up to 33 ms, or until about 4 KiB, before +it frames them. Tool calls, questions, stops, errors and the turn's end send the +held text first and are never delayed. The text, its order and replay are +unchanged; at 40 tokens a second a stream reaches the window about 30 times a +second instead of 40, at a lower engine cost per stream. Terminal and remote +surfaces, and desktop chats joined through another process's session host, are +unchanged. diff --git a/internal/remote/deltawindow.go b/internal/remote/deltawindow.go new file mode 100644 index 0000000000..0fb3f193cd --- /dev/null +++ b/internal/remote/deltawindow.go @@ -0,0 +1,137 @@ +package remote + +import ( + "reflect" + "time" + + "github.com/Agent-Field/codeaf/internal/session" +) + +// A STREAM'S TEXT IS FOLDED BEFORE IT IS FRAMED, AND ONLY ITS TEXT. When a +// conversation's engine sets [Engine.DeltaWindow], each stream's pump sends a +// text or reasoning delta at once when none went out within the window, and +// otherwise holds it, folding the chunks that follow with identical metadata, +// until the window ends or the run reaches DeltaBytes. Anything that is not +// streamed text first sends the held text and then goes out itself, at once, +// and ends the window, so a tool call, a question, a stop or the turn's end is +// never late and the words after it start at once. The channel closing sends +// the held text before the stream's "closed" frame. A folded run is one event, +// one seq and one ring entry, so replay and resume see exactly what was sent. +// +// DefaultDeltaWindow spans two frames at 60 Hz because a one-frame window +// cannot fold the desktop's text deltas arriving 25 milliseconds apart. +const DefaultDeltaWindow = 33 * time.Millisecond + +// DeltaBytes is the flush threshold for a pending run, not a frame size cap. +// Folding a chunk can cross it; a chunk already at or above it travels alone. +const DeltaBytes = 4 << 10 + +func streamedText(kind session.EventKind) bool { + return kind == session.EventTextDelta || kind == session.EventReasoning +} + +func textContinues(prev, next session.Event) bool { + if prev.Kind != next.Kind || !streamedText(prev.Kind) { + return false + } + // Metadata belongs to the run, including fields added to Event later. Only + // the text may differ without changing what the surface was told. + prev.Text, next.Text = "", "" + return reflect.DeepEqual(prev, next) +} + +// deltaWindow belongs to one pump so independent streams never hold each +// other's text or share a timer. +type deltaWindow struct { + window time.Duration + emit func(session.Event) + pending *session.Event + until time.Time +} + +func (w *deltaWindow) send(ev session.Event) { + w.emit(ev) + w.until = time.Now().Add(w.window) +} + +func (w *deltaWindow) flush() { + if w.pending == nil { + return + } + ev := *w.pending + w.pending = nil + w.send(ev) +} + +func (w *deltaWindow) add(ev session.Event) { + if !streamedText(ev.Kind) { + w.flush() + w.emit(ev) + // Tool calls, questions and every other event end the burst, so the next + // text reaches the person at once rather than inheriting the old window. + w.until = time.Time{} + return + } + if w.pending != nil && !time.Now().Before(w.until) { + w.flush() + } + // A chunk at or above the flush threshold travels alone. Smaller chunks + // fold before the threshold is checked, so their frame may exceed it. + if len(ev.Text) >= DeltaBytes { + w.flush() + w.send(ev) + return + } + if w.pending != nil && textContinues(*w.pending, ev) { + w.pending.Text += ev.Text + if len(w.pending.Text) >= DeltaBytes { + w.flush() + } + return + } + w.flush() + if !time.Now().Before(w.until) { + w.send(ev) + return + } + w.pending = &ev +} + +func pumpDeltaWindow(window time.Duration, events <-chan session.Event, emit func(session.Event)) { + w := deltaWindow{window: window, emit: emit} + var timer *time.Timer + var armedUntil time.Time + defer func() { + if timer != nil { + timer.Stop() + } + }() + for { + var fire <-chan time.Time + if w.pending != nil { + // Reset only when the deadline changes. In particular, flushing a run for + // changed metadata must move the timer too, rather than wake at its old end. + if timer == nil { + timer = time.NewTimer(time.Until(w.until)) + } else if !armedUntil.Equal(w.until) { + timer.Reset(time.Until(w.until)) + } + armedUntil = w.until + fire = timer.C + } else if !armedUntil.IsZero() { + timer.Stop() + armedUntil = time.Time{} + } + select { + case ev, ok := <-events: + if !ok { + w.flush() + return + } + w.add(ev) + case <-fire: + armedUntil = time.Time{} + w.flush() + } + } +} diff --git a/internal/remote/deltawindow_test.go b/internal/remote/deltawindow_test.go new file mode 100644 index 0000000000..38d63a466a --- /dev/null +++ b/internal/remote/deltawindow_test.go @@ -0,0 +1,805 @@ +package remote + +import ( + "encoding/json" + "errors" + "io" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/session" +) + +// These tests read frames at the surface so coalescing cannot hide a lost event, +// a replay gap or a close sent before the last text. +func deltaSession(t *testing.T, window time.Duration) (*fakeAgent, *Session, *link, uint64, chan session.Event) { + t.Helper() + agent := &fakeAgent{} + engine := engineOn(agent) + engine.DeltaWindow = window + sess := NewSession(engine, true) + t.Cleanup(func() { _ = sess.Close() }) + l := dialSession(t, sess) + l.hello(Hello{Version: Version}) + ref := decode[StreamRef](t, l.ok(1, MethodSubmit, SubmitArgs{Text: "go"}).Payload) + return agent, sess, l, ref.Stream, agent.stream(0) +} + +// Driver updates describe the connection, not the ordered turn stream. A +// second surface can cause one without adding an event to the replay ring. +func deltaFrame(l *link) Frame { + for { + frame := l.recv() + if frame.Kind != "driver" { + return frame + } + } +} + +func deltaEvent(t *testing.T, l *link, id uint64, seq *uint64) session.Event { + t.Helper() + frame := deltaFrame(l) + (*seq)++ + if frame.Kind != "event" || frame.ID != id || frame.Seq != *seq { + t.Fatalf("frame = %+v, want event on stream %d seq %d", frame, id, *seq) + } + return decode[EventWire](t, frame.Payload).Unwire() +} + +func deltaClose(t *testing.T, l *link, id, seq uint64) { + t.Helper() + if frame := deltaFrame(l); frame.Kind != "closed" || frame.ID != id || frame.Seq != seq { + t.Fatalf("close = %+v, want stream %d seq %d", frame, id, seq) + } +} + +// Only adjacent text with identical metadata is one observable run. This is +// deliberately independent of the coalescer's comparison helper. +func deltaRuns(events []session.Event) []session.Event { + var runs []session.Event + for _, ev := range events { + if n := len(runs); n > 0 && (ev.Kind == session.EventTextDelta || ev.Kind == session.EventReasoning) { + prev, next := runs[n-1], ev + prev.Text, next.Text = "", "" + if reflect.DeepEqual(prev, next) { + runs[n-1].Text += ev.Text + continue + } + } + runs = append(runs, ev) + } + return runs +} + +func TestDeltaWindowEquivalence(t *testing.T) { + script := []session.Event{ + {Kind: session.EventTextDelta, Text: "hel"}, + {Kind: session.EventTextDelta, Text: "lo"}, + {Kind: session.EventTextDelta, Text: " there"}, + {Kind: session.EventReasoning, Text: "think"}, + {Kind: session.EventReasoning, Text: "ing"}, + {Kind: session.EventToolBegin, Tool: "read", CallID: "c", Args: "{}"}, + {Kind: session.EventToolOutput, Tool: "read", CallID: "c", Text: "output"}, + {Kind: session.EventToolEnd, Tool: "read", CallID: "c", Text: "done"}, + {Kind: session.EventConsentRequest, ID: 7, Tool: "bash", Hint: "permission"}, + {Kind: session.EventTextDelta, Text: "after"}, + {Kind: session.EventTextDelta, Text: " tool"}, + {Kind: session.EventTextDelta, Text: "addressed", Addressed: true}, + {Kind: session.EventTextDelta, Text: " again", Addressed: true}, + {Kind: session.EventTextDelta, Text: "run", Addressed: true, RunID: "r2"}, + {Kind: session.EventTextDelta, Text: " two", Addressed: true, RunID: "r2"}, + {Kind: session.EventTextDelta, Text: "hint", Hint: "different", Addressed: true, RunID: "r2"}, + {Kind: session.EventTurnDone, Usage: session.Usage{Output: 17}}, + } + var baseline []session.Event + for _, window := range []time.Duration{0, 2 * time.Second} { + agent, _, l, id, stream := deltaSession(t, window) + for _, ev := range script { + stream <- ev + } + agent.finish(stream) + var got []session.Event + var seq uint64 + for { + frame := deltaFrame(l) + if frame.Kind == "closed" { + if frame.ID != id || frame.Seq != seq { + t.Fatalf("closed = %+v, last seq %d", frame, seq) + } + break + } + seq++ + if frame.Kind != "event" || frame.ID != id || frame.Seq != seq { + t.Fatalf("frame = %+v, want seq %d", frame, seq) + } + got = append(got, decode[EventWire](t, frame.Payload).Unwire()) + } + if window == 0 { + if len(got) != len(script) { + t.Fatalf("disabled window sent %d frames, want %d", len(got), len(script)) + } + baseline = deltaRuns(got) + } else if !reflect.DeepEqual(deltaRuns(got), baseline) { + t.Fatalf("coalescing changed ordered runs:\ngot %+v\nwant %+v", deltaRuns(got), baseline) + } + } +} + +func TestDeltaWindowFirstDeltaIsImmediate(t *testing.T) { + agent, _, l, id, stream := deltaSession(t, 2*time.Second) + var seq uint64 + for _, text := range []string{"first", "after tool"} { + start := time.Now() + stream <- session.Event{Kind: session.EventTextDelta, Text: text} + if ev := deltaEvent(t, l, id, &seq); ev.Text != text { + t.Fatalf("delta = %+v", ev) + } + if elapsed := time.Since(start); elapsed >= 500*time.Millisecond { + t.Fatalf("first delta waited %v", elapsed) + } + stream <- session.Event{Kind: session.EventToolBegin, Tool: "read"} + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventToolBegin { + t.Fatalf("tool = %+v", ev) + } + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +// The scripted stop emits the same done event as a cancelled turn, through the +// real interrupt method, so this exercises the stop door as well as the pump. +type deltaStoppedAgent struct{ *fakeAgent } + +func (a *deltaStoppedAgent) Interrupt() { a.InterruptFor(session.StopByPerson) } +func (a *deltaStoppedAgent) InterruptFor(door session.StopDoor) { + a.fakeAgent.InterruptFor(door) + if stream := a.stream(0); door == session.StopByPerson && stream != nil { + stream <- session.Event{Kind: session.EventTurnDone} + } +} + +func deltaStop(t *testing.T, l *link, agent *fakeAgent) { + t.Helper() + l.ok(100, MethodInterrupt, InterruptArgs{Door: string(session.StopByPerson)}) + agent.mu.Lock() + door := agent.stopDoor + agent.mu.Unlock() + if door != session.StopByPerson { + t.Fatalf("stop door = %q", door) + } +} + +func deltaInterruptSession(t *testing.T, window time.Duration) (*fakeAgent, *link, uint64, chan session.Event) { + t.Helper() + agent := &fakeAgent{} + engine := engineOn(agent) + engine.Agent = &deltaStoppedAgent{agent} + engine.DeltaWindow = window + sess := NewSession(engine, true) + t.Cleanup(func() { _ = sess.Close() }) + l := dialSession(t, sess) + l.hello(Hello{Version: Version}) + ref := decode[StreamRef](t, l.ok(1, MethodSubmit, SubmitArgs{Text: "go"}).Payload) + return agent, l, ref.Stream, agent.stream(0) +} + +func TestDeltaWindowNonTextFlushesAndClosesWindow(t *testing.T) { + agent, l, id, stream := deltaInterruptSession(t, 2*time.Second) + var seq uint64 + for _, kind := range []session.EventKind{session.EventToolBegin, session.EventConsentRequest, session.EventTurnDone} { + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "a" { + t.Fatalf("leading delta = %+v", ev) + } + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + stream <- session.Event{Kind: session.EventTextDelta, Text: "c"} + start := time.Now() + if kind == session.EventTurnDone { + deltaStop(t, l, agent) + } else { + stream <- session.Event{Kind: kind, ID: 7, Tool: "read"} + } + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventTextDelta || ev.Text != "bc" { + t.Fatalf("pending delta = %+v", ev) + } + if ev := deltaEvent(t, l, id, &seq); ev.Kind != kind { + t.Fatalf("barrier = %+v, want %v", ev, kind) + } + stream <- session.Event{Kind: session.EventTextDelta, Text: "after"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "after" { + t.Fatalf("after barrier = %+v", ev) + } + if elapsed := time.Since(start); elapsed >= 500*time.Millisecond { + t.Fatalf("barrier waited %v", elapsed) + } + stream <- session.Event{Kind: session.EventToolEnd} + deltaEvent(t, l, id, &seq) + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +func TestDeltaWindowTurnEndFlushes(t *testing.T) { + for _, done := range []bool{false, true} { + agent, _, l, id, stream := deltaSession(t, 2*time.Second) + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + var seq uint64 + deltaEvent(t, l, id, &seq) + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + stream <- session.Event{Kind: session.EventTextDelta, Text: "c"} + start := time.Now() + if done { + stream <- session.Event{Kind: session.EventTurnDone} + } + agent.finish(stream) + if ev := deltaEvent(t, l, id, &seq); ev.Text != "bc" { + t.Fatalf("tail = %+v", ev) + } + if done { + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventTurnDone { + t.Fatalf("done = %+v", ev) + } + } + deltaClose(t, l, id, seq) + if elapsed := time.Since(start); elapsed >= 500*time.Millisecond { + t.Fatalf("close waited %v", elapsed) + } + } +} + +func TestDeltaWindowCoalescesOnlyWhenEnabled(t *testing.T) { + for _, window := range []time.Duration{0, time.Second} { + agent, _, l, id, stream := deltaSession(t, window) + for range 50 { + stream <- session.Event{Kind: session.EventTextDelta, Text: "x"} + } + agent.finish(stream) + var seq uint64 + var text string + for { + frame := deltaFrame(l) + if frame.Kind == "closed" { + if frame.ID != id || frame.Seq != seq { + t.Fatalf("close = %+v, last seq %d", frame, seq) + } + break + } + seq++ + if frame.Kind != "event" || frame.ID != id || frame.Seq != seq { + t.Fatalf("frame = %+v", frame) + } + ev := decode[EventWire](t, frame.Payload).Unwire() + if ev.Kind != session.EventTextDelta { + t.Fatalf("delta = %+v", ev) + } + text += ev.Text + } + if text != strings.Repeat("x", 50) { + t.Fatalf("text = %q", text) + } + if window == 0 && seq != 50 || window > 0 && seq > 3 { + t.Fatalf("window %v sent %d frames", window, seq) + } + } +} + +func TestDeltaWindowByteCapSendsImmediately(t *testing.T) { + agent, _, l, id, stream := deltaSession(t, 2*time.Second) + var seq uint64 + stream <- session.Event{Kind: session.EventTextDelta, Text: "first"} + deltaEvent(t, l, id, &seq) + start := time.Now() + stream <- session.Event{Kind: session.EventTextDelta, Text: strings.Repeat("x", DeltaBytes-1)} + stream <- session.Event{Kind: session.EventTextDelta, Text: "y"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != strings.Repeat("x", DeltaBytes-1)+"y" { + t.Fatalf("capped delta = %d bytes", len(ev.Text)) + } + // The threshold is checked after folding, so two smaller chunks can overshoot. + chunk := strings.Repeat("o", DeltaBytes-1) + stream <- session.Event{Kind: session.EventTextDelta, Text: chunk} + stream <- session.Event{Kind: session.EventTextDelta, Text: chunk} + if ev := deltaEvent(t, l, id, &seq); ev.Text != chunk+chunk { + t.Fatalf("overshoot delta = %d bytes, want %d", len(ev.Text), 2*(DeltaBytes-1)) + } + // A large delta is its own frame, even when a small run is already pending. + stream <- session.Event{Kind: session.EventTextDelta, Text: "small"} + stream <- session.Event{Kind: session.EventTextDelta, Text: strings.Repeat("z", DeltaBytes+1)} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "small" { + t.Fatalf("pending before large delta = %d bytes", len(ev.Text)) + } + if ev := deltaEvent(t, l, id, &seq); ev.Text != strings.Repeat("z", DeltaBytes+1) { + t.Fatalf("large delta = %d bytes", len(ev.Text)) + } + if elapsed := time.Since(start); elapsed >= 500*time.Millisecond { + t.Fatalf("byte cap waited %v", elapsed) + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +func TestDeltaWindowReplayWhileTextIsPending(t *testing.T) { + for _, cursor := range []bool{true, false} { + agent, sess, first, id, stream := deltaSession(t, 2*time.Second) + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + var seq uint64 + seen := deltaEvent(t, first, id, &seq).Text + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + stream <- session.Event{Kind: session.EventTextDelta, Text: "c"} + // Once the pump has taken both chunks, at least the earlier one is pending. + waitForEngine(t, func() bool { return len(stream) == 0 }) + if cursor { + if err := first.end(); err != nil { + t.Fatal(err) + } + } + // Replaying a frame emitted while nobody watched tests both the ring and the + // pending tail. The metadata change flushes "bc" and leaves "d" pending. + stream <- session.Event{Kind: session.EventTextDelta, Text: "d", Addressed: true} + stream <- session.Event{Kind: session.EventTextDelta, Text: "e", Addressed: true} + waitForEngine(t, func() bool { return len(stream) == 0 }) + if !cursor { + // The original surface stays attached in the no-cursor case; both + // surfaces must see each emitted frame once, including the pending tail. + seen += deltaEvent(t, first, id, &seq).Text + } + second := dialSession(t, sess) + hello := Hello{Version: Version, Watch: !cursor} + if cursor { + hello.Resume = []StreamCursor{{Stream: id, Seq: seq}} + } else { + seq, seen = 0, "" + } + second.hello(hello) + stream <- session.Event{Kind: session.EventTextDelta, Text: "f", Addressed: true} + agent.finish(stream) + for { + frame := deltaFrame(second) + if frame.Kind == "closed" { + if frame.ID != id || frame.Seq != seq { + t.Fatalf("closed = %+v, last seq %d", frame, seq) + } + break + } + seq++ + if frame.Kind != "event" || frame.ID != id || frame.Seq != seq { + t.Fatalf("replay = %+v, want seq %d", frame, seq) + } + seen += decode[EventWire](t, frame.Payload).Unwire().Text + } + if seen != "abcdef" { + t.Fatalf("cursor %v received %q", cursor, seen) + } + if !cursor { + firstSeq := uint64(2) + if ev := deltaEvent(t, first, id, &firstSeq); ev.Text != "def" { + t.Fatalf("original surface tail = %q", ev.Text) + } + deltaClose(t, first, id, firstSeq) + } + } +} + +func TestDeltaWindowSwapWhileTextIsPending(t *testing.T) { + for _, method := range []string{MethodSessionNew, MethodSessionOpen} { + t.Run(method, func(t *testing.T) { + first, second := &fakeAgent{}, &fakeAgent{title: "the new conversation"} + engine := engineOn(first) + engine.DeltaWindow = 2 * time.Second + const file = "/sessions/next.jsonl" + engine.Fresh = func() (WrappedAgent, string, error) { return second, file, nil } + engine.Open = func(string) (WrappedAgent, bool, error) { return second, true, nil } + sess := NewSession(engine, true) + t.Cleanup(func() { _ = sess.Close() }) + l := dialSession(t, sess) + l.hello(Hello{Version: Version}) + id, generation := sess.mint() + stream := make(chan session.Event, 64) + closeStream := sync.OnceFunc(func() { close(stream) }) + t.Cleanup(closeStream) + sess.tellTurn(Turn{Stream: id, Said: "old"}, nil) + if frame := deltaFrame(l); frame.Kind != "turn" || decode[Turn](t, frame.Payload).Stream != id { + t.Fatalf("turn = %+v, want stream %d", frame, id) + } + drained := make(chan struct{}) + sess.pumps.Add(1) + go func() { + sess.pump(id, generation, stream) + close(drained) + }() + var seq uint64 + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + deltaEvent(t, l, id, &seq) + stream <- session.Event{Kind: session.EventTextDelta, Text: "old pending b"} + stream <- session.Event{Kind: session.EventTextDelta, Text: "old pending c"} + // Taking both chunks means the pump has folded at least the first one. + waitForEngine(t, func() bool { return len(stream) == 0 }) + welcome := decode[Welcome](t, l.ok(2, method, file).Payload) + if welcome.SessionFile != file || welcome.Title != second.title || welcome.Resumed != (method == MethodSessionOpen) { + t.Fatalf("swap welcome = %+v", welcome) + } + // Owning the channel lets us join just this retired pump after its + // close-time flush, even if other session pumps are still running. + // The real swap still invalidates its generation while text is pending. + closeStream() + select { + case <-drained: + case <-time.After(5 * time.Second): + t.Fatal("the retired pump did not finish") + } + id = decode[StreamRef](t, l.ok(3, MethodSubmit, SubmitArgs{Text: "new"}).Payload).Stream + nextStream := second.stream(0) + nextStream <- session.Event{Kind: session.EventTextDelta, Text: "new text"} + second.finish(nextStream) + seq = 0 + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventTextDelta || ev.Text != "new text" { + t.Fatalf("text after swap = %+v", ev) + } + deltaClose(t, l, id, seq) + }) + } +} + +// Duplicate numbered frames on the repaired transport so the real client's +// sequence check must discard replay overlap, including folded text frames. +type deltaReplayWriter struct { + io.Writer + duplicated chan struct{} +} + +func (w deltaReplayWriter) Write(p []byte) (int, error) { + n, err := w.Writer.Write(p) + if err != nil { + return n, err + } + var frame Frame + if json.Unmarshal(p, &frame) == nil && frame.Kind == "event" && frame.Seq != 0 { + if _, err := w.Writer.Write(p); err != nil { + return n, err + } + select { + case w.duplicated <- struct{}{}: + default: + } + } + return n, nil +} + +func TestDeltaWindowRealClientReconnectWhileTextIsPending(t *testing.T) { + agent := &fakeAgent{} + engine := engineOn(agent) + engine.DeltaWindow = 5 * time.Second + sess := NewSession(engine, true) + t.Cleanup(func() { _ = sess.Close() }) + var first io.ReadWriteCloser + dials := 0 + ready, cancel := make(chan struct{}), make(chan struct{}) + t.Cleanup(func() { close(cancel) }) + resumed := make(chan Hello, 1) + duplicated := make(chan struct{}, 1) + dialer := func() (io.ReadWriteCloser, error) { + dials++ + repair := dials > 1 + if repair { + select { + case <-ready: + case <-cancel: + return nil, io.ErrClosedPipe + } + } + surface, engine := Pipe() + if !repair { + first = surface + } + go func() { + defer engine.Close() + var out io.Writer = engine + if repair { + out = deltaReplayWriter{Writer: engine, duplicated: duplicated} + } + _ = ServeAttach(engine, out, AttachOptions{Open: func(hello Hello) (*Session, error) { + if repair { + resumed <- hello + } + return sess, nil + }}) + }() + return surface, nil + } + client, err := Roam("devbox", Hello{Version: Version}, Roaming{Dial: dialer, Window: 5 * time.Second}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + events, err := client.Agent().Submit(nil, "go") + if err != nil { + t.Fatal(err) + } + stream := agent.stream(0) + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + leading, ok := nextEvent(t, events) + if !ok || leading.Kind != session.EventTextDelta || leading.Text != "a" { + t.Fatalf("leading event = %+v, open %v", leading, ok) + } + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + stream <- session.Event{Kind: session.EventTextDelta, Text: "c"} + waitForEngine(t, func() bool { return len(stream) == 0 }) + if err := first.Close(); err != nil { + t.Fatal(err) + } + // Changed metadata flushes "bc" into the ring while disconnected and + // leaves the following text pending through the real resume handshake. + stream <- session.Event{Kind: session.EventTextDelta, Text: "d", Addressed: true} + stream <- session.Event{Kind: session.EventTextDelta, Text: "e", Addressed: true} + waitForEngine(t, func() bool { return len(stream) == 0 }) + close(ready) + replayed, ok := nextEvent(t, events) + if !ok || replayed.Kind != session.EventTextDelta || replayed.Text != "bc" { + t.Fatalf("replayed event = %+v, open %v", replayed, ok) + } + hello := <-resumed + if len(hello.Resume) != 1 || hello.Resume[0].Stream == 0 || hello.Resume[0].Seq != 1 || hello.Session != engine.SessionFile { + t.Fatalf("resume hello = %+v", hello) + } + stream <- session.Event{Kind: session.EventTextDelta, Text: "f", Addressed: true} + agent.finish(stream) + text := leading.Text + replayed.Text + for { + ev, ok := nextEvent(t, events) + if !ok { + break + } + if ev.Kind != session.EventTextDelta { + t.Fatalf("resumed event = %+v", ev) + } + text += ev.Text + } + if text != "abcdef" { + t.Fatalf("client drew %q, want every character exactly once in order", text) + } + select { + case <-duplicated: + default: + t.Fatal("the repaired transport supplied no replay overlap") + } +} + +func TestDeltaWindowDefaultQuietTail(t *testing.T) { + agent, _, l, id, stream := deltaSession(t, DefaultDeltaWindow) + var seq uint64 + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "a" { + t.Fatalf("leading delta = %+v", ev) + } + start := time.Now() + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventTextDelta || ev.Text != "b" { + t.Fatalf("quiet tail = %+v", ev) + } + elapsed := time.Since(start) + t.Logf("default-window quiet tail: %v", elapsed) + if elapsed >= 500*time.Millisecond { + t.Fatalf("quiet tail waited %v", elapsed) + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +func TestDeltaWindowDefaultSteadyProducer(t *testing.T) { + const deltas, interval = 80, 5 * time.Millisecond + agent, _, l, id, stream := deltaSession(t, DefaultDeltaWindow) + var want strings.Builder + for i := range deltas { + want.WriteByte(byte('a' + i%26)) + } + text := want.String() + twoFrames := make(chan struct{}) + produced := make(chan struct { + duration time.Duration + streamedBeforeEnd bool + }, 1) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + var firstPush time.Time + var streamedBeforeEnd bool + for i, char := range text { + <-ticker.C + if i == 0 { + firstPush = time.Now() + } + if i == len(text)-1 { + // Keep the final push behind two surface receipts so a delayed + // reader cannot turn successful streaming into a starvation failure. + select { + case <-twoFrames: + streamedBeforeEnd = true + case <-time.After(2 * time.Second): + } + } + stream <- session.Event{Kind: session.EventTextDelta, Text: string(char)} + } + produced <- struct { + duration time.Duration + streamedBeforeEnd bool + }{time.Since(firstPush), streamedBeforeEnd} + agent.finish(stream) + }() + var seq uint64 + var got strings.Builder + var last time.Time + var maxGap time.Duration + for { + frame := deltaFrame(l) + if frame.Kind == "closed" { + if frame.ID != id || frame.Seq != seq { + t.Fatalf("close = %+v, last seq %d", frame, seq) + } + break + } + now := time.Now() + if !last.IsZero() && now.Sub(last) > maxGap { + maxGap = now.Sub(last) + } + last = now + seq++ + if frame.Kind != "event" || frame.ID != id || frame.Seq != seq { + t.Fatalf("frame = %+v, want stream %d seq %d", frame, id, seq) + } + ev := decode[EventWire](t, frame.Payload).Unwire() + if ev.Kind != session.EventTextDelta { + t.Fatalf("steady event = %+v", ev) + } + got.WriteString(ev.Text) + if seq == 2 { + close(twoFrames) + } + } + production := <-produced + t.Logf("default-window steady producer: %d deltas over %v, %d frames, max gap %v", deltas, production.duration, seq, maxGap) + if got.String() != text { + t.Fatalf("steady text = %q, want %q", got.String(), text) + } + if limit := uint64((production.duration+DefaultDeltaWindow-1)/DefaultDeltaWindow) + 3; seq > limit { + t.Fatalf("steady producer sent %d frames, want at most %d", seq, limit) + } + if !production.streamedBeforeEnd { + t.Fatal("fewer than two text frames reached the surface before the last push") + } +} + +// The median measures ordinary non-text latency without treating a scheduler +// pause as timer waiting. TestDeltaWindowNonTextFlushesAndClosesWindow uses a +// long window to prove that pending text and non-text never wait for the timer. +func TestDeltaWindowNonTextLatency(t *testing.T) { + for _, tc := range []struct { + name string + kind session.EventKind + }{ + {"stop", session.EventTurnDone}, + {"consent", session.EventConsentRequest}, + {"tool", session.EventToolBegin}, + {"error", session.EventError}, + } { + t.Run(tc.name, func(t *testing.T) { + kind := tc.kind + agent, l, id, stream := deltaInterruptSession(t, DefaultDeltaWindow) + var seq uint64 + var samples []time.Duration + for range 20 { + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + deltaEvent(t, l, id, &seq) + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + start := time.Now() + if kind == session.EventTurnDone { + deltaStop(t, l, agent) + } else { + ev := session.Event{Kind: kind, ID: 7, Tool: "read"} + if kind == session.EventError { + ev.Err = errors.New("scripted failure") + } + stream <- ev + } + if ev := deltaEvent(t, l, id, &seq); ev.Kind != session.EventTextDelta || ev.Text != "b" { + t.Fatalf("pending = %+v", ev) + } + if ev := deltaEvent(t, l, id, &seq); ev.Kind != kind { + t.Fatalf("barrier = %+v", ev) + } + samples = append(samples, time.Since(start)) + } + sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] }) + median, maximum := samples[len(samples)/2], samples[len(samples)-1] + t.Logf("non-text push-to-receipt: median %v, max %v (window %v)", median, maximum, DefaultDeltaWindow) + if median >= DefaultDeltaWindow/2 { + t.Fatalf("non-text waited: median %v, max %v", median, maximum) + } + agent.finish(stream) + deltaClose(t, l, id, seq) + }) + } +} + +func TestDeltaWindowTimerFlushesAndIdleStartsImmediately(t *testing.T) { + const window = 400 * time.Millisecond + agent, _, l, id, stream := deltaSession(t, window) + var seq uint64 + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + deltaEvent(t, l, id, &seq) + for _, text := range []string{"bc", "de"} { + start := time.Now() + for _, char := range text { + stream <- session.Event{Kind: session.EventTextDelta, Text: string(char)} + } + if ev := deltaEvent(t, l, id, &seq); ev.Text != text { + t.Fatalf("timed tail = %q, want %q", ev.Text, text) + } + if elapsed := time.Since(start); elapsed < window/2 || elapsed > window+500*time.Millisecond { + t.Fatalf("tail arrived after %v", elapsed) + } + } + // An empty window needs no timer wakeup. Let it expire before the next run. + time.Sleep(window + 100*time.Millisecond) + start := time.Now() + stream <- session.Event{Kind: session.EventTextDelta, Text: "f"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "f" { + t.Fatalf("idle first delta = %q", ev.Text) + } + if elapsed := time.Since(start); elapsed >= window/4 { + t.Fatalf("idle first delta waited %v", elapsed) + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +func TestDeltaWindowMetadataFlushRestartsTimer(t *testing.T) { + const window = time.Second + agent, _, l, id, stream := deltaSession(t, window) + var seq uint64 + stream <- session.Event{Kind: session.EventTextDelta, Text: "a"} + deltaEvent(t, l, id, &seq) + stream <- session.Event{Kind: session.EventTextDelta, Text: "b"} + time.Sleep(300 * time.Millisecond) + start := time.Now() + stream <- session.Event{Kind: session.EventTextDelta, Text: "c", RunID: "next"} + if ev := deltaEvent(t, l, id, &seq); ev.Text != "b" { + t.Fatalf("old run = %q", ev.Text) + } + if ev := deltaEvent(t, l, id, &seq); ev.Text != "c" || ev.RunID != "next" { + t.Fatalf("new run = %+v", ev) + } + if elapsed := time.Since(start); elapsed < 900*time.Millisecond || elapsed > window+500*time.Millisecond { + t.Fatalf("restarted window lasted %v", elapsed) + } + agent.finish(stream) + deltaClose(t, l, id, seq) +} + +func TestDeltaWindowStreamsAreIndependent(t *testing.T) { + agent, _, l, firstID, first := deltaSession(t, 2*time.Second) + var firstSeq uint64 + first <- session.Event{Kind: session.EventTextDelta, Text: "a"} + deltaEvent(t, l, firstID, &firstSeq) + first <- session.Event{Kind: session.EventTextDelta, Text: "b"} + secondID := decode[StreamRef](t, l.ok(2, MethodSubmit, SubmitArgs{Text: "another turn"}).Payload).Stream + second := agent.stream(1) + var secondSeq uint64 + start := time.Now() + second <- session.Event{Kind: session.EventReasoning, Text: "thinking"} + if ev := deltaEvent(t, l, secondID, &secondSeq); ev.Kind != session.EventReasoning || ev.Text != "thinking" { + t.Fatalf("second stream = %+v", ev) + } + if elapsed := time.Since(start); elapsed >= 500*time.Millisecond { + t.Fatalf("second stream inherited the first's window: %v", elapsed) + } + agent.finish(second) + deltaClose(t, l, secondID, secondSeq) + agent.finish(first) + if ev := deltaEvent(t, l, firstID, &firstSeq); ev.Text != "b" { + t.Fatalf("first stream tail = %q", ev.Text) + } + deltaClose(t, l, firstID, firstSeq) +} diff --git a/internal/remote/server.go b/internal/remote/server.go index cc30dfc182..5a341d8c64 100644 --- a/internal/remote/server.go +++ b/internal/remote/server.go @@ -135,6 +135,13 @@ type WrappedAgent interface { type Engine struct { // SyncService is the engine's address for chat outage notices. SyncService string + // DeltaWindow lets each of this conversation's streams fold streamed text + // for up to this long before it is framed (deltawindow.go): the first chunk + // still goes at once and nothing but text ever waits. Every delta otherwise + // costs a frame's encoding and a handful of goroutine handoffs on the way to + // the surface. Zero keeps one frame per event, which is what every surface + // that has not opted in, and its replay tests, are written against. + DeltaWindow time.Duration // Headless records the approval context the agent was built for. Reusing an // interactive agent must not silently give a headless caller its default gate. Headless bool @@ -1626,8 +1633,18 @@ func (sess *Session) mintLocked() (id, generation uint64) { func (sess *Session) pump(id, generation uint64, events <-chan session.Event) { defer sess.pumps.Done() defer guard.Recover("remote/engine stream") - for event := range events { - sess.emit(id, generation, event) + sess.mu.Lock() + var window time.Duration + if sess.engine != nil { + window = sess.engine.DeltaWindow + } + sess.mu.Unlock() + if window > 0 { + pumpDeltaWindow(window, events, func(event session.Event) { sess.emit(id, generation, event) }) + } else { + for event := range events { + sess.emit(id, generation, event) + } } sess.finish(id, generation) } From bb9fad4490f9b82e3595d7d7018257099f8b5eb4 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:53:29 -0500 Subject: [PATCH 10/28] factory: a write step that changed nothing is not done (#2158) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * Take the gated copy of the union merged docs --------- Co-authored-by: Claude Opus 5.5 --- .../appfactory_empty_write_retry_test.go | 191 ++++++++++++++++++ .../appfactory_read_without_git_test.go | 38 ++++ cmd/codeaf/appfactory_rerun_stopped_test.go | 4 +- cmd/codeaf/appfactory_slot_stall_test.go | 7 +- cmd/codeaf/factory_live_e2e_test.go | 85 +++++++- .../unreleased/2158-empty-write-not-done.md | 8 + .../2158-factory-write-retry-base.md | 16 ++ docs/design/app-v3/PROTOCOL.md | 16 +- internal/factory/run/change.go | 62 +++++- internal/factory/run/change_test.go | 56 ++++- internal/factory/run/loop.go | 9 +- .../chat/factory-stage-conversations.md | 10 + internal/manual/chat/factory.md | 32 +++ internal/manual/chat/reading-a-task-page.md | 4 + 14 files changed, 518 insertions(+), 20 deletions(-) create mode 100644 cmd/codeaf/appfactory_empty_write_retry_test.go create mode 100644 cmd/codeaf/appfactory_read_without_git_test.go create mode 100644 docs/changes/unreleased/2158-empty-write-not-done.md create mode 100644 docs/changes/unreleased/2158-factory-write-retry-base.md diff --git a/cmd/codeaf/appfactory_empty_write_retry_test.go b/cmd/codeaf/appfactory_empty_write_retry_test.go new file mode 100644 index 0000000000..551702ab5c --- /dev/null +++ b/cmd/codeaf/appfactory_empty_write_retry_test.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/client" + "github.com/Agent-Field/codeaf/internal/factory/lease" + "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/factory/worker" + "github.com/Agent-Field/codeaf/internal/session" +) + +// Earlier committed work belongs to the job even when its replacement attempt +// makes no new edit. An empty branch still needs the person's decision. +func TestFactoryWriteUsesJobBaseAcrossAttempts(t *testing.T) { + for _, action := range []string{"rerun", "try again", "resume after stop", "resume after reported stop", "empty rerun", "base unavailable"} { + t.Run(action, func(t *testing.T) { + s := newLocalSetup(t) + s.backend.emit = s.server + t.Setenv(factoryBenchesEnv, "1") + model := newStageModel(t) + t.Setenv("OPENROUTER_API_KEY", "test-key") + t.Setenv("CODEAF_BASE_URL", model.server.URL) + g := newRunRig(t) + freshEngineProcess(t) + freshEngineFactoryStore(t) + s.backend.open = appOpenInProcess + s.backend.local.parent = func() session.Config { + return session.Config{Workspace: g.workspace, Model: "test/model", OneModel: true, APIKey: "test-key", BaseURL: model.server.URL} + } + var st appwire.FactoryLocalStatus + if action == "resume after stop" { + if err := s.backend.startLocal(context.Background()); err != nil { + t.Fatal(err) + } + st = s.backend.localStatus() + } else { + st = s.start() + } + floor := s.backend.local.host.floor + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + entered, ready := filepath.Join(dir, "entered"), filepath.Join(dir, "ready") + command := "if ! test -f CHANGE.txt; then printf 'the requested change\n' > CHANGE.txt && git add CHANGE.txt && git -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit -qm change; fi; git diff --exit-code HEAD" + switch action { + case "try again": + command += fmt.Sprintf("; test -f %q", ready) + case "resume after stop", "resume after reported stop": + command += fmt.Sprintf("; touch %q; while ! test -f %q; do sleep 0.05; done", entered, ready) + case "empty rerun": + command = "true" + } + id, err := floor.Add(context.Background(), factory.Item{ + Repo: "acme/api", Num: 7, Kind: factory.KindIssue, Tier: factory.TierOwner, Cap: 10, + Title: "Keep the job's earlier change", Autonomy: factory.AutonomyManual, + Stages: []factory.Stage{ + {Name: "write", Kind: factory.StageCheck, On: true, Ask: command, Until: factory.UntilGreen, Max: 1}, + {Name: "approve-test", Kind: factory.StageGate, On: true}, + {Name: "test", Kind: factory.StageCheck, On: true, Ask: "true", Until: factory.UntilGreen, Max: 1}, + }, + }) + if err != nil { + t.Fatal(err) + } + if action == "base unavailable" { + if err := os.WriteFile(filepath.Join(g.workspace, ".git", "config.lock"), nil, 0600); err != nil { + t.Fatal(err) + } + } + stopUnreported := func() {} + if action == "resume after stop" { + mirror, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + runner := buildFactoryRunner(mirror, floor, g.workspace, t.TempDir(), nil, nil) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + worker.Run(ctx, worker.Options{ + Dial: func(int) *client.WorkerClient { return client.Open(st.URL, s.backend.local.host.token).Worker("") }, + Attach: api.WorkerAttach{Machine: "pause-fixture", Name: "pause fixture", Handle: "owner", TakesUnassigned: map[string]bool{lease.EveryRepo: true}}, + Runner: runner, Mirror: mirror, Slots: 1, ReportEvery: time.Hour, BeatEvery: 20 * time.Millisecond, + }) + }() + stopUnreported = func() { cancel(); <-done } + t.Cleanup(stopUnreported) + } + read := func() api.Job { + var job api.Job + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + return job + } + act := func(method string, extra map[string]any) { + job := read() + params := map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}} + for k, v := range extra { + params[k] = v + } + s.must(method, params, &appwire.FactoryJobActed{}) + } + wait := func(what string, ok func(api.Job) bool) api.Job { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + job := read() + if ok(job) { + return job + } + if time.Now().After(deadline) { + t.Fatalf("%s: state=%s needs=%+v steps=%+v", what, job.State, job.NeedsYou, job.Steps) + } + time.Sleep(20 * time.Millisecond) + } + } + waiting := func(job api.Job) bool { return job.State == api.StateNeedsYou } + gate := func(job api.Job) bool { + return waiting(job) && job.Steps[0].State == "done" && job.Steps[1].State == "waiting" + } + act("factory.job_start", nil) + if strings.HasPrefix(action, "resume after") { + wait("earlier attempt commits before Stop", func(api.Job) bool { _, err := os.Stat(entered); return err == nil }) + if action == "resume after reported stop" { + wait("first attempt is reported before Stop", func(job api.Job) bool { return job.Steps[0].State == "running" && job.Steps[0].Round > 0 }) + } + if action == "resume after stop" && read().Steps[0].Round != 0 { + t.Fatal("unreported worker sent its first round before Stop") + } + t.Logf("before Stop: write=%s round=%d", read().Steps[0].State, read().Steps[0].Round) + act("factory.job_step_stop", map[string]any{"step": "write"}) + wait("Stop ends the earlier attempt", func(job api.Job) bool { return job.Failure != nil && job.Failure.Code == factory.CodeStopped }) + if action == "resume after stop" { + stopUnreported() + mirror, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + startPauseWorker(t, s, st, buildFactoryRunner(mirror, floor, g.workspace, t.TempDir(), nil, nil), mirror) + } + } else { + first := wait("first attempt finishes", waiting) + if action == "base unavailable" { + if first.Steps[0].State != "failed" || first.Steps[0].Failure == nil || !strings.Contains(first.Steps[0].Failure.Detail, "could not preserve the job's change base") || first.Change != nil && len(first.Change.Files) > 0 { + t.Fatalf("write ran without preserving its base: %+v", first) + } + return + } + if action == "try again" && (first.NeedsYou == nil || first.NeedsYou.Kind != factory.NeedsOutOfAttempts || first.Steps[0].State != "waiting") { + t.Fatalf("first write did not reproduce a failed command: %+v", first) + } + if action == "rerun" && !gate(first) { + t.Fatalf("write did not finish against its committed change: %+v", first) + } + if action == "empty rerun" && (first.Steps[0].State != "waiting" || first.Steps[1].State != "pending" || first.NeedsYou == nil || first.NeedsYou.Question == nil || !strings.Contains(first.NeedsYou.Question.Text, "the checkout has no change from where the run began")) { + t.Fatalf("empty first attempt passed: %+v", first) + } + } + if err := os.WriteFile(ready, nil, 0600); err != nil { + t.Fatal(err) + } + before := read() + act("factory.job_step_rerun", map[string]any{"step": "write"}) + after := wait("replacement attempt finishes", func(job api.Job) bool { return waiting(job) && job.Steps[0].Round > before.Steps[0].Round }) + if action == "empty rerun" { + if after.Steps[0].State != "waiting" || after.Steps[1].State != "pending" || after.NeedsYou == nil || after.NeedsYou.Question == nil || !strings.Contains(after.NeedsYou.Question.Text, "the checkout has no change from where the run began") { + t.Fatalf("empty rerun passed: %+v", after) + } + } else { + if !gate(after) { + t.Fatalf("earlier committed change was called empty on %s: %+v", action, after) + } + if after.HeadCommit == "" || before.HeadCommit != "" && after.HeadCommit != before.HeadCommit || after.Change == nil || len(after.Change.Files) != 1 || after.Change.Files[0].Path != "CHANGE.txt" { + t.Fatalf("replacement did not retain the earlier change and commit: before=%s after=%s change=%+v", before.HeadCommit, after.HeadCommit, after.Change) + } + } + t.Logf("%s: write=%s round=%d next=%s", action, after.Steps[0].State, after.Steps[0].Round, after.Steps[1].State) + }) + } +} diff --git a/cmd/codeaf/appfactory_read_without_git_test.go b/cmd/codeaf/appfactory_read_without_git_test.go new file mode 100644 index 0000000000..72c806cf4b --- /dev/null +++ b/cmd/codeaf/appfactory_read_without_git_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/factory" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" +) + +// Read-only commands keep working when the runner cannot read Git metadata. +// A failed Git read is not a comparison base that needs preserving. +func TestFactoryReadOnlyCheckWithoutGitMetadata(t *testing.T) { + g := newRunRig(t) + dir := t.TempDir() + runner := factoryrun.New(factoryrun.Options{ + Store: g.st, + RepoDir: func(string) string { return dir }, + Exec: map[factory.StageKind]factoryrun.Executor{ + factory.StageCheck: factoryrun.NewCheckExecutor(factoryrun.CheckOptions{}), + }, + }) + stopAll(t, g.st, runner) + id, err := g.st.Add(context.Background(), factory.Item{ + Repo: "api", Title: "Read without Git metadata", Tier: factory.TierOwner, + Stages: []factory.Stage{{Name: "test", Kind: factory.StageCheck, On: true, Ask: "true", Until: factory.UntilGreen, Max: 1}}, + }) + if err != nil { + t.Fatal(err) + } + if err := runner.Launch(id); err != nil { + t.Fatal(err) + } + job := g.waitFor(t, id, factory.StateLanded) + if len(job.Stream.Phases) != 1 || job.Stream.Phases[0].State != factory.PhaseDone { + t.Fatalf("read-only check did not finish: %+v", job.Stream) + } +} diff --git a/cmd/codeaf/appfactory_rerun_stopped_test.go b/cmd/codeaf/appfactory_rerun_stopped_test.go index 26f1659cda..ab77c9214b 100644 --- a/cmd/codeaf/appfactory_rerun_stopped_test.go +++ b/cmd/codeaf/appfactory_rerun_stopped_test.go @@ -53,7 +53,7 @@ func TestFactoryRetryActionsDoNotReportStopped(t *testing.T) { id, err := floor.Add(context.Background(), factory.Item{ Repo: "acme/api", Num: 7, Kind: factory.KindIssue, Tier: factory.TierOwner, Cap: 10, Title: "Retry the failing check", Stages: []factory.Stage{ - {Name: "write", Kind: factory.StageCheck, On: true, Ask: "true", Until: factory.UntilGreen, Max: 1}, + {Name: "write", Kind: factory.StageCheck, On: true, Ask: "printf 'the requested change\\n' > CHANGE.txt", Until: factory.UntilGreen, Max: 1}, {Name: "test", Kind: factory.StageCheck, On: true, Ask: "false", Until: factory.UntilGreen, Max: 1}, }, }) @@ -101,7 +101,7 @@ func TestFactoryRetryActionsDoNotReportStopped(t *testing.T) { t.Fatal(err) } if before.Stream == nil || before.Stream.Cur != 1 || before.Stream.Phases[1].Result == nil || before.Stream.Phases[1].Result.Exit == 0 { - t.Fatalf("did not reproduce a failed command: %+v", before.Stream) + t.Fatalf("did not reproduce a failed command: question=%q stream=%+v", before.Question, before.Stream) } initialWaiting := false for _, e := range entries() { diff --git a/cmd/codeaf/appfactory_slot_stall_test.go b/cmd/codeaf/appfactory_slot_stall_test.go index 3971dc0498..d1f6be471c 100644 --- a/cmd/codeaf/appfactory_slot_stall_test.go +++ b/cmd/codeaf/appfactory_slot_stall_test.go @@ -40,7 +40,7 @@ func TestFactoryStepRetryWaitsForPreviousLeaseCleanup(t *testing.T) { id, err := floor.Add(context.Background(), factory.Item{ Repo: "acme/api", Num: 7, Kind: factory.KindIssue, Tier: factory.TierOwner, Cap: 10, Title: "Retry the failing check", Stages: []factory.Stage{ - {Name: "write", Kind: factory.StageCheck, On: true, Ask: "true", Until: factory.UntilGreen, Max: 1}, + {Name: "write", Kind: factory.StageCheck, On: true, Ask: "printf 'the requested change\\n' > CHANGE.txt", Until: factory.UntilGreen, Max: 1}, {Name: "test", Kind: factory.StageCheck, On: true, Ask: "false", Until: factory.UntilGreen, Max: 1}, }, }) @@ -211,7 +211,10 @@ func TestFactoryStepRerunFromApproveGate(t *testing.T) { s := newLocalSetup(t) s.backend.emit = s.server t.Setenv(factoryBenchesEnv, slots) - model := newStageModel(t) + model := newScriptedStageModel(t, + stageCall{"write", `{"path":"CHANGE.txt","content":"the requested change\n"}`}, + stageCall{"stage_result", stageModelReport}, + ) t.Setenv("OPENROUTER_API_KEY", "test-key") t.Setenv("CODEAF_BASE_URL", model.server.URL) g := newRunRig(t) diff --git a/cmd/codeaf/factory_live_e2e_test.go b/cmd/codeaf/factory_live_e2e_test.go index 3c5cf57bb8..ca36ba10c8 100644 --- a/cmd/codeaf/factory_live_e2e_test.go +++ b/cmd/codeaf/factory_live_e2e_test.go @@ -204,7 +204,11 @@ func (r *liveRecorder) read(chunk []byte) { r.cost += m.Usage.Cost } -func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { +// liveFactoryRig is the live lane's floor: the opt-in and key checks, a +// re-rooted profile, every model call through a [liveRecorder], and a local +// seam over the real runner and stage maker pinned to [liveFactoryModel]. +func liveFactoryRig(t *testing.T) (runRig, *liveRecorder, factory.Seam) { + t.Helper() if os.Getenv(liveFactoryOptIn) != "1" { t.Skipf("opt-in live lane: set %s=1 (spends a few cents on OpenRouter)", liveFactoryOptIn) } @@ -214,8 +218,6 @@ func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Fatal("BLOCKED: git is not installed") } - started := time.Now() - root := resolvedTempDir(t) t.Setenv("HOME", filepath.Join(root, "home")) t.Setenv(home.EnvVar, filepath.Join(root, "state")) @@ -245,7 +247,12 @@ func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { maker := factoryStageMaker(g.st, g.workspace, "", launch.Config) r := buildFactoryRunner(g.st, g.st, g.workspace, "", maker, nil) stopAll(t, g.st, r) - seam := factory.LocalSeam(g.st, time.Now(), factory.WithRunner(r)) + return g, recorder, factory.LocalSeam(g.st, time.Now(), factory.WithRunner(r)) +} + +func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { + started := time.Now() + g, recorder, seam := liveFactoryRig(t) const want = "hello factory" id, err := g.st.Add(context.Background(), factory.Item{ @@ -319,6 +326,13 @@ func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { if err != nil || strings.TrimSpace(string(got)) != want { t.Fatalf("HELLO.txt in the item's worktree = %q, %v", got, err) } + assertLiveRecorder(t, recorder) +} + +// assertLiveRecorder fails the lane on any call that was not the pinned +// model, any policy refusal, or spend over the lane's budget. +func assertLiveRecorder(t *testing.T, recorder *liveRecorder) { + t.Helper() recorder.mu.Lock() defer recorder.mu.Unlock() if recorder.calls == 0 { @@ -338,6 +352,69 @@ func TestLiveFactoryStageDrivesARealModelToTheApproveGate(t *testing.T) { } } +// A WRITE THAT CHANGES NOTHING IS HELD BEFORE ANY CHECK RUNS. The checkout +// already holds what the ask wants, so an honest model reports write done +// without changing a file; the check after it would pass on the unchanged +// tree. The runner must refuse that done and ask, and the check must not run. +func TestLiveFactoryWriteThatChangesNothingIsHeldBeforeTheCheck(t *testing.T) { + started := time.Now() + g, recorder, seam := liveFactoryRig(t) + const want = "hello factory" + if err := os.WriteFile(filepath.Join(g.workspace, "HELLO.txt"), []byte(want+"\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", "HELLO.txt"}, {"-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", "commit", "-q", "-m", "greeting"}} { + if out, err := exec.Command("git", append([]string{"-C", g.workspace}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + id, err := g.st.Add(context.Background(), factory.Item{ + Title: "make sure of the greeting", Repo: "api", Tier: factory.TierOwner, Cap: liveFactoryBudget, + Stages: []factory.Stage{ + {Name: "write", Kind: factory.StageChat, On: true, Until: "done", + Ask: "Make sure HELLO.txt in the current directory contains exactly `" + want + "` and a trailing newline. Read it first. Then call stage_result with done true, findings 0 and a one-line output."}, + {Name: "verify", Kind: factory.StageCheck, On: true, Ask: "grep -qx '" + want + "' HELLO.txt"}, + {Name: factory.ApproveName, Kind: factory.StageGate, On: true}, + }, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), liveFactoryWall) + defer cancel() + if err := seam.Launch(id); err != nil { + t.Fatal(err) + } + var it factory.Item + for { + it, _ = g.st.Get(id) + if it.State == factory.StateLanded || it.Failure != nil || it.Question != "" || ctx.Err() != nil { + break + } + time.Sleep(250 * time.Millisecond) + } + summary := map[string]any{"state": string(it.State), "question": it.Question, "seconds": int(time.Since(started).Seconds())} + recorder.mu.Lock() + summary["request_models"], summary["response_models"], summary["calls"], summary["cost_usd"] = recorder.requests, recorder.replies, recorder.calls, recorder.cost + recorder.mu.Unlock() + if it.Stream != nil { + summary["item_spent_usd"] = it.Stream.Spent + for _, ph := range it.Stream.Phases { + summary["phase_"+ph.Name] = string(ph.State) + } + } + writeLiveFactoryLog(t, summary) + + const q = "write did not finish: the checkout has no change from where the run began · try again, or stop the job?" + if it.Question != q || it.QKind != factory.QKindOutOfAttempts { + t.Fatalf("item is %s asking %q (%s); summary %v", it.State, it.Question, it.QKind, summary) + } + if verify := it.Stream.Phases[1]; verify.State != factory.PhasePending || verify.Round != 0 { + t.Fatalf("the check ran on an unchanged write: %+v", verify) + } + assertLiveRecorder(t, recorder) +} + // writeLiveFactoryLog keeps the summary outside the repository, for the person // reading it afterwards: identifiers, counts, state and cost, nothing else. func writeLiveFactoryLog(t *testing.T, summary map[string]any) { diff --git a/docs/changes/unreleased/2158-empty-write-not-done.md b/docs/changes/unreleased/2158-empty-write-not-done.md new file mode 100644 index 0000000000..49c7f2841e --- /dev/null +++ b/docs/changes/unreleased/2158-empty-write-not-done.md @@ -0,0 +1,8 @@ +--- +kind: fixed +title: a factory write step that changed nothing is not done +pr: 2158 +surface: [engine] +invalidates: + - "Was true: a factory `write` step passed on its own `stage_result` done, even when the job's checkout had no change, so test and review ran on an untouched tree and the empty change surfaced only at publish. Is true now: a `write` round that reports done on a checkout with no change from the run's base does not pass; it reads `the checkout has no change from where the run began`, goes round again, and out of rounds asks `write did not finish: the checkout has no change from where the run began · try again, or stop the job?`. Other steps may still finish with no change." +--- diff --git a/docs/changes/unreleased/2158-factory-write-retry-base.md b/docs/changes/unreleased/2158-factory-write-retry-base.md new file mode 100644 index 0000000000..1851a89cc6 --- /dev/null +++ b/docs/changes/unreleased/2158-factory-write-retry-base.md @@ -0,0 +1,16 @@ +--- +kind: fixed +title: stopped factory write steps keep their earlier change on retry +pr: 2158 +surface: [engine, factory] +invalidates: + - "Was true: stopping Write before its first progress report could make a retry call its earlier committed change empty. Is true now: the job branch keeps its original comparison base, so restarting the stopped step counts that change." +--- + +Rerun, Try again after a failure, and restarting a stopped Write count work already +on the job's branch, including committed changes. They do not require a new edit in +each attempt. A truly empty branch still waits for your decision on the first +attempt and on a rerun. Retry checks now make the change their Write step promises. + +If the original base cannot be saved, the round fails before editing so a later +retry cannot mistake its earlier work for an empty branch. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index 6c08f36c44..f0a1c1e2b7 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -471,9 +471,21 @@ A confirmed send-back reports acceptance, not worker startup; an unconfirmed result directs the manager to read status before retrying and explain the outcome to the person without internal identifiers. +Cap questions use one next budget: `factory.job`'s existing +`spending_limit.next_limit` and `questions[].next_limit` are the smallest of +$5, $10, $20, $30 strictly above recorded spend. Once spend reaches $30, +double the current cap, repeatedly if needed, until it is above spend. +The next limit is never zero. A new question says +`budget of $30 reached · $31 spent · raise it to $60, or stop?`. Yes or +continue, including on an older saved question, uses that same next cap. +The desk sends that next cap as the existing `factory.job_set_budget`'s +`budget`, never as an answer field. No new field, method, hello feature, or +error code is added; `factory.job_actions` remains the existing feature. + `factory.job_set_budget` keeps its existing params (`factory`, `job`, `budget`, -required `expect.version`) and `FactoryJobActed` reply. Budgets remain $5, $10, $20 or -$30. Raising the cap above recorded spend resolves a `cap` question and resumes +required `expect.version`) and `FactoryJobActed` reply. Budgets accept $5, $10, $20, $30 or exactly the job's next cap. +Other amounts are refused with HTTP 422 (`bad_input`), mapped to app-wire +`bad_params`. Raising the cap above recorded spend resolves a `cap` question and resumes the interrupted step in the same conversation and round, with spend retained, in Auto or Manual on local and leased runners. Manual still asks before its next step. Take over and publishing jobs do not resume through a budget change. diff --git a/internal/factory/run/change.go b/internal/factory/run/change.go index 21bdd1195f..edb0012097 100644 --- a/internal/factory/run/change.go +++ b/internal/factory/run/change.go @@ -13,20 +13,68 @@ import ( "github.com/Agent-Field/codeaf/internal/factory" ) +// sayWriteUnchanged is why a write round that reported done is not done. It +// is the first line of that round's output, so the next round's brief and the +// question a person reads after the last round both say it. +const sayWriteUnchanged = "the checkout has no change from where the run began" + +// unchangedWrite holds back a write round that reported done while the +// checkout has no change from the run's base. The write step is the one that +// changes the repository, so its done on a clean checkout is a shortfall +// found here, before test and review spend money on an unchanged tree. Every +// other step may finish without a change, and a checkout that cannot be read +// is never a reason to hold a round back. +func unchangedWrite(st factory.Stage, job Job, res factory.StageResult) factory.StageResult { + if !res.Done || job.dirErr != nil || !strings.EqualFold(stageLabel(st), writeStep) { + return res + } + change, err := readChange(job.Dir, job.Item.ChangeBase) + if err != nil || len(change.Files) > 0 { + return res + } + res.Done, res.Exit = false, 1 + res.Output = sayWriteUnchanged + "\nMake the change in the checkout, or report done: false with why none is needed.\n" + res.Output + return res +} + // changeBase fixes the comparison before any round can commit its edits. -// A PR includes its existing change; other jobs start at their first HEAD. -func changeBase(dir string, it factory.Item) string { +// The branch keeps the pin because Stop may revoke a worker's lease before +// its first report reaches the factory. A replacement must not pin the edited +// HEAD. A PR includes its existing change; other jobs start at their first HEAD. +func changeBase(dir string, it factory.Item) (string, error) { if it.ChangeBase != "" { - return it.ChangeBase + return it.ChangeBase, nil } git := ExecGit{} + branch, err := git.Run(dir, "symbolic-ref", "--quiet", "--short", "HEAD") + if err != nil { + branch = "" + } + key := "branch." + branch + ".codeaf-change-base" + if branch != "" { + if base, err := git.Run(dir, "config", "--local", "--get", key); err == nil && base != "" { + return base, nil + } + } + var base string if isPull(it) { - if base, err := git.Run(dir, "merge-base", "HEAD", PullBase(it)); err == nil { - return base + base, err = git.Run(dir, "merge-base", "HEAD", PullBase(it)) + if err != nil { + base = "" + } + } + if base == "" { + base, err = git.Run(dir, "rev-parse", "HEAD") + if err != nil { + return "", nil + } + } + if branch != "" && base != "" { + if _, err := git.Run(dir, "config", "--local", key, base); err != nil { + return "", fmt.Errorf("could not preserve the job's change base: %w", err) } } - base, _ := git.Run(dir, "rev-parse", "HEAD") - return base + return base, nil } // readChange includes committed, staged, unstaged and untracked work. Git's diff --git a/internal/factory/run/change_test.go b/internal/factory/run/change_test.go index d1e263941a..13ffc26c81 100644 --- a/internal/factory/run/change_test.go +++ b/internal/factory/run/change_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "reflect" "strings" + "sync/atomic" "testing" "github.com/Agent-Field/codeaf/internal/factory" @@ -13,7 +14,10 @@ import ( func TestChangeIncludesCommittedStagedUnstagedAndUntrackedFiles(t *testing.T) { dir := gitRig(t) - base := changeBase(dir, factory.Item{Kind: factory.KindIssue}) + base, err := changeBase(dir, factory.Item{Kind: factory.KindIssue}) + if err != nil { + t.Fatal(err) + } mustGit(t, dir, "checkout", "-q", "-b", "factory/change") write := func(path, content string) { t.Helper() @@ -75,7 +79,11 @@ func TestChangeForPRIncludesItsOriginalDiff(t *testing.T) { if err != nil { t.Fatal(err) } - change, err := readChange(dir, changeBase(dir, it)) + base, err := changeBase(dir, it) + if err != nil { + t.Fatal(err) + } + change, err := readChange(dir, base) if err != nil { t.Fatal(err) } @@ -173,3 +181,47 @@ func TestApprovedCommitMessageStaysInsidePublicBoundary(t *testing.T) { t.Fatalf("commit message: %q", msg) } } + +// A WRITE THAT CHANGED NOTHING IS NOT DONE. The 2026-11 factory-demo run on +// #94 reported write done on a clean checkout; test and review passed on the +// unchanged tree and only publishing found nothing to commit. A step that +// only reads (plan here) still finishes on a clean checkout. +func TestAWriteThatLeavesTheCheckoutUnchangedIsNotDone(t *testing.T) { + dir := gitRig(t) + mustGit(t, dir, "checkout", "-q", "-b", "factory/empty") + var writes, tests atomic.Int32 + var second string + g := newRig(t, map[factory.StageKind]Executor{ + factory.StageChat: ExecutorFunc(func(_ context.Context, job Job) (factory.StageResult, error) { + if job.Stage.Name == "write" { + if writes.Add(1) == 2 && job.Last != nil { + second = job.Last.Output + } + } + return factory.StageResult{Done: true, Output: job.Stage.Name + " is done"}, nil + }), + factory.StageCheck: ExecutorFunc(func(context.Context, Job) (factory.StageResult, error) { + tests.Add(1) + return factory.StageResult{Done: true}, nil + }), + }, func(o *Options) { o.RepoDir = func(string) string { return dir } }) + write := chat("write") + write.Max = 2 + id := g.add("empty write", chat("plan"), write, factory.Stage{Name: "test", Kind: factory.StageCheck, Ask: "true"}) + if err := g.r.Launch(id); err != nil { + t.Fatal(err) + } + it := g.wait(id, "asking about write", func(it factory.Item) bool { + return it.State == factory.StateNeedsYou && it.QKind == factory.QKindOutOfAttempts + }) + if writes.Load() != 2 || tests.Load() != 0 { + t.Fatalf("write ran %d rounds and test ran %d times", writes.Load(), tests.Load()) + } + q := "write is not done after 2 rounds: the checkout has no change from where the run began · try again, send it back, or stop the job?" + if it.Question != q || !strings.HasPrefix(second, "the checkout has no change from where the run began\n") { + t.Fatalf("question %q, round 2 read %q", it.Question, second) + } + if plan, w := it.Stream.Phases[0], it.Stream.Phases[1]; plan.State != factory.PhaseDone || w.State != factory.PhaseWaiting { + t.Fatalf("plan = %s, write = %s", plan.State, w.State) + } +} diff --git a/internal/factory/run/loop.go b/internal/factory/run/loop.go index c047ee9440..6775311525 100644 --- a/internal/factory/run/loop.go +++ b/internal/factory/run/loop.go @@ -1011,6 +1011,7 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { res.Notes = append(res.Notes, verification.FailurePrefix+" final review report: "+why) } } + res = unchangedWrite(st, job, res) lp.fold(c, it, st, i, res) if !lp.capCheck(c, i, name) { return false @@ -1314,6 +1315,9 @@ func shortfall(res factory.StageResult) string { if reason := verificationReason(res); reason != "" { return "verification blocked: " + reason } + if !res.Done && strings.HasPrefix(res.Output, sayWriteUnchanged+"\n") { + return sayWriteUnchanged + } return factory.Shortfall(res) } @@ -1502,7 +1506,10 @@ func (lp *floorLoop) job(c *loopCtl, st factory.Stage, index, i, round int) Job } } if dirErr == nil && dir != "" && it.ChangeBase == "" { - if base := changeBase(dir, it); base != "" { + base, err := changeBase(dir, it) + if err != nil { + dirErr = err + } else if base != "" { it.ChangeBase = base _ = lp.write(c, func(saved *factory.Item) error { saved.ChangeBase = base; return nil }) } diff --git a/internal/manual/chat/factory-stage-conversations.md b/internal/manual/chat/factory-stage-conversations.md index c6b2c05516..ca3d84b6af 100644 --- a/internal/manual/chat/factory-stage-conversations.md +++ b/internal/manual/chat/factory-stage-conversations.md @@ -74,6 +74,14 @@ the `stage_result` it reported, and only a report that says done can pass: - `until green`: done, and a check step's command exited 0. A conversation step has no exit code of its own, so for it green is the same as done: it reports done once the tests pass. +A `write` step's done also needs a change in the job's checkout. With none, the round reads +`the checkout has no change from where the run began` and does not pass, so later steps never +run on an untouched tree. The comparison uses the job's original base: a rerun, **Try again**, +or restarting a stopped step counts the change already on its branch, including committed +work. Each attempt does not need a new edit. A branch with no change still cannot pass Write, +even on a rerun. If the original base cannot be preserved, the round fails before editing; +try again after the repository becomes writable. Other steps may finish with no change. + A step that falls short of its until goes round again while it has rounds left (see a second round). @@ -135,6 +143,8 @@ round ran for two hours (or an idle Plan could not finish reporting within its t codeaf could not make the report; - `write ran out of time without reporting · …` - `plan did not finish: · …` when it reported that it could not; +- `write did not finish: the checkout has no change from where the run began · …` when write + reported done but changed nothing; - `test is not proven: 1 of 2 claims shown · …` for a step that runs once; - `review is not clean after 3 rounds: 1 finding · …` only for a step that loops. diff --git a/internal/manual/chat/factory.md b/internal/manual/chat/factory.md index cbf6a296e5..39257a2633 100644 --- a/internal/manual/chat/factory.md +++ b/internal/manual/chat/factory.md @@ -620,6 +620,21 @@ Terminal factory sentences still read those words as chips. Existing jobs are not rewritten; this applies when a new job is created. Factories without the separate-title creation capability retain their existing creation behavior. +## Job Brief inputs — later requests, answers and saved notes + +The Brief reads the issue plus your later requests from gate answers and saved +notes, alongside the plan and step results. A later request wins where it +disagrees with the issue or plan. Work that follows your request is treated as +asked, never as a fault, surprise or scope creep. A real miss against your +request is still reported, and the Brief names the request as asked. + +Plain continue, yes and no answers are omitted. The writer keeps the newest +12 distinct requests, up to 2 KiB per text and 16 KiB of request JSON, oldest +first. Dated answers carry their step, gate and answer time. Saved notes have +no recorded step or time; their saved order comes before dated answers. +Duplicate words appear once. Requests survive cuts to fit the writer's input. +A new request makes an earlier Brief out of date. + ## Job Brief after proof, publishing and an out of date line The job's Brief refreshes when proof waits for your publication approval and @@ -653,3 +668,20 @@ Automatic brief writes keep their existing limit: the last kept version must cost at most $0.03, and the kept versions plus one more at that cost must fit within 5% of the job's actual spend. Correcting a duplicate charge can therefore hold back an automatic write sooner. A requested retry still bypasses this guard. + +## budget reached — yes, raise the cap, next budget and stop + +When a job reaches its budget, the question names its current budget, what it +spent, and the next absolute budget: `budget of $0.01 reached · $0.01 spent · +raise it to $5, or stop?`. Yes or continue raises the cap to that $5 total; +the desktop's raise button at $5 does the same. It does not add $5 to the cap. + +The next budget is the smallest of $5, $10, $20 or $30 strictly above recorded +spending. If a round spent $5.25, the next budget is $10 even when the old cap +was $0.01. Once spending reaches $30, double the current cap, repeatedly if +needed, until it is above spend. A $30 cap with $31 spent offers $60. +Saved questions on older jobs can keep their old words, but yes now +uses this same rule. + +The raise button accepts $5, $10, $20, $30 or exactly the job's next budget. +Other amounts are refused. Stop keeps the recorded spend and work. diff --git a/internal/manual/chat/reading-a-task-page.md b/internal/manual/chat/reading-a-task-page.md index 735edb3a09..8f21a56586 100644 --- a/internal/manual/chat/reading-a-task-page.md +++ b/internal/manual/chat/reading-a-task-page.md @@ -546,6 +546,10 @@ this` is the third: there was nobody inside the task to read you, because its wo already in front of the checker. The words are kept on the task, the check is not allowed to land it as done over them, and the task takes another round with them instead. +A note delivered while the worker's final answer is streaming still gets a reader. +The worker takes another request with your words before its runner closes it; merely +saving the words in the transcript does not mark the note read. + **A correction can change what the task is judged by, and only yours can.** Say "CSV instead of JSON" into a running task and the worker can fold it into the task's own done-condition, citing the line you sent; from then on that is what the checker judges the From 54bef3dcbfeb492f53e174bb9ec12c1041c22272 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:53:41 -0500 Subject: [PATCH 11/28] factory: a job's budget ask offers one amount (#2258) * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_cap_truth_test.go | 89 +++++++++++++++++++ .../2258-factory-cap-continue-above-thirty.md | 13 +++ .../unreleased/2258-factory-cap-one-amount.md | 15 ++++ internal/appwire/factory_actions.go | 18 ++-- internal/factory/api/api_test.go | 2 +- internal/factory/api/codec.go | 6 +- internal/factory/api/needs_test.go | 2 +- internal/factory/api/wire.go | 3 +- internal/factory/needs.go | 33 +++++-- internal/factory/needs_test.go | 4 +- internal/factory/run/loop.go | 5 +- internal/factory/run/loop_test.go | 4 +- internal/factory/run/money.go | 5 +- internal/factory/run/money_test.go | 6 +- internal/factory/run/stagereport_test.go | 6 +- internal/factory/serve/jobs.go | 4 +- internal/factory/serve/verbs.go | 5 +- internal/factory/serve/workers_test.go | 3 + 18 files changed, 183 insertions(+), 40 deletions(-) create mode 100644 cmd/codeaf/appfactory_cap_truth_test.go create mode 100644 docs/changes/unreleased/2258-factory-cap-continue-above-thirty.md create mode 100644 docs/changes/unreleased/2258-factory-cap-one-amount.md diff --git a/cmd/codeaf/appfactory_cap_truth_test.go b/cmd/codeaf/appfactory_cap_truth_test.go new file mode 100644 index 0000000000..4535edf1ca --- /dev/null +++ b/cmd/codeaf/appfactory_cap_truth_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" +) + +// The runner's public spending answer and the desk's budget action promise +// the same next cap, including when a round spent past the old cap. +func TestAppFactoryCapQuestionHasOneAmount(t *testing.T) { + f := startActionFactory(t) + alice := signedIn(t, f, 1, "alice") + var sessions appwire.FactorySessionsTaken + alice.w.must("factory.sessions_take", struct{}{}, &sessions) + if len(sessions.Changes) != 1 || sessions.Changes[0].Token == "" { + t.Fatal("fixture has no signed-in session") + } + for _, tc := range []struct { + cap, spent, next float64 + question string + }{ + {.01, .01, 5, "budget of $0.01 reached · $0.01 spent · raise it to $5, or stop?"}, + {.01, 5.25, 10, "budget of $0.01 reached · $5.25 spent · raise it to $10, or stop?"}, + {5, 10, 20, "budget of $5 reached · $10 spent · raise it to $20, or stop?"}, + {10, 20, 30, "budget of $10 reached · $20 spent · raise it to $30, or stop?"}, + {20, 30, 40, "budget of $20 reached · $30 spent · raise it to $40, or stop?"}, + {30, 31, 60, "budget of $30 reached · $31 spent · raise it to $60, or stop?"}, + {30, 125, 240, "budget of $30 reached · $125 spent · raise it to $240, or stop?"}, + } { + t.Run(fmt.Sprintf("cap=%g/spent=%g", tc.cap, tc.spent), func(t *testing.T) { + it := factory.Item{State: factory.StateNeedsYou, Starter: "alice", Cap: tc.cap, QKind: "cap", Stream: &factory.Stream{Spent: tc.spent}} + question, _ := (*factoryrun.Money)(nil).CapQuestion(it) + if question != tc.question { + t.Errorf("question = %q, want %q", question, tc.question) + } + // Old saved words must use the current next-cap rule. + it.Question = "budget reached · more, or stop?" + saved := f.create(t, it) + job := alice.job(t, saved.ID) + if job.SpendingLimit == nil || job.SpendingLimit.NextLimit != tc.next { + t.Errorf("wire spending limit = %+v, want next %g", job.SpendingLimit, tc.next) + } + if len(job.Questions) != 1 || job.Questions[0].NextLimit != tc.next { + t.Fatalf("question metadata = %+v", job.Questions) + } + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/jobs/%d/answer", f.URL, saved.ID), strings.NewReader(`{"yes":true}`)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+sessions.Changes[0].Token) + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + answered := alice.job(t, saved.ID) + continued := it + changed := factoryrun.NoteContinue(&continued, time.Now()) + if response.StatusCode != 200 || answered.Budget != tc.next { + t.Errorf("yes status=%d body=%s budget=%g want %g", response.StatusCode, body, answered.Budget, tc.next) + } + saved = f.create(t, it) + if tc.cap == 30 && tc.spent == 31 { + alice.w.mustFail("factory.job_set_budget", alice.params(t, saved.ID, map[string]any{"budget": 50}), appwire.CodeBadParams) + } + alice.act(t, "factory.job_set_budget", saved.ID, map[string]any{"budget": tc.next}) + raised := alice.job(t, saved.ID) + if !changed || continued.Cap != tc.next || raised.Budget != continued.Cap { + t.Errorf("yes cap=%g, dock cap=%g, want %g", continued.Cap, raised.Budget, tc.next) + } + if factoryrun.NoteContinue(&continued, time.Now()) { + t.Error("second continue raised the cap again") + } + }) + } +} diff --git a/docs/changes/unreleased/2258-factory-cap-continue-above-thirty.md b/docs/changes/unreleased/2258-factory-cap-continue-above-thirty.md new file mode 100644 index 0000000000..e2ba7400b1 --- /dev/null +++ b/docs/changes/unreleased/2258-factory-cap-continue-above-thirty.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: Long jobs can continue beyond a thirty dollar budget +pr: 2258 +surface: [engine] +invalidates: + - "Spending thirty dollars left a job with only Stop and refused continuation." +--- + +Budget questions always offer a next cap. Above the standard $5, $10, $20 and +$30 choices, the cap doubles until it is above recorded spending. A job capped +at $30 with $31 spent offers $60; Yes and the raise button both set that cap. +Other nonstandard amounts remain refused. diff --git a/docs/changes/unreleased/2258-factory-cap-one-amount.md b/docs/changes/unreleased/2258-factory-cap-one-amount.md new file mode 100644 index 0000000000..8f26460ae1 --- /dev/null +++ b/docs/changes/unreleased/2258-factory-cap-one-amount.md @@ -0,0 +1,15 @@ +--- +kind: fixed +title: Budget questions and raise buttons agree on the next cap +pr: 2258 +surface: [engine] +invalidates: + - "Yes doubled a job's old cap while the desktop raised it to an allowed budget." +--- + +A budget question now names the amount spent and the next total budget. Yes, +continue and the desktop's raise button at that amount all use the smallest +of $5, $10, $20 or $30 above recorded spend. Older saved questions use the same +rule when answered. At $30 spent or more, the next budget doubles the current +cap repeatedly until it exceeds spending. The raise button accepts that exact +next budget as well as the standard choices. diff --git a/internal/appwire/factory_actions.go b/internal/appwire/factory_actions.go index c473b194bc..33f8be49cc 100644 --- a/internal/appwire/factory_actions.go +++ b/internal/appwire/factory_actions.go @@ -45,7 +45,8 @@ import ( // FeatureFactoryJobActions is announced in hello when the engine serves // factory.job_*. Job creation keeps separate title and details literal; -// neither field sets budget or edits the pipeline. +// neither field sets budget or edits the pipeline. Spending questions and +// budget edits share the allowed budgets from factory.JobBudgetSteps. const FeatureFactoryJobActions = "factory.job_actions" // FeatureFactoryStartPipeline is announced beside FeatureFactoryJobActions when @@ -64,9 +65,9 @@ type FactoryPipelineStarter interface { // old expect on an active job or its current running step. const FeatureFactoryStopAnyVersion = "factory.stop_any_version" -// FactoryBudgetSteps are the limits factory.job_set_budget accepts, in US -// dollars. The job owner can choose either direction, never below recorded spending. -var FactoryBudgetSteps = []float64{5, 10, 20, 30} +// FactoryBudgetSteps are the standard limits factory.job_set_budget accepts, +// in US dollars. Larger amounts must equal the job's next cap. The job owner can choose either direction, never below recorded spending. +var FactoryBudgetSteps = factory.JobBudgetSteps() // The priorities factory.job_set_priority takes, as the factory spells them. const ( @@ -160,7 +161,8 @@ type FactoryJobPriorityParams struct { Expect *api.Expect `json:"expect"` } -// FactoryJobBudgetParams changes a job's limit to one of [FactoryBudgetSteps]. +// FactoryJobBudgetParams changes a job's limit to one of [FactoryBudgetSteps] +// or its next cap above those steps, as validated by the factory. // An increase above spend resumes a cap wait; a smaller increase keeps the // question. A decrease below recorded spending is refused. type FactoryJobBudgetParams struct { @@ -556,13 +558,17 @@ func prepBudget(p *FactoryJobBudgetParams) error { if err := checkJob(&p.Factory, p.Job, p.Expect); err != nil { return err } + // The factory validates larger amounts against the job's current next cap. + if p.Budget > FactoryBudgetSteps[len(FactoryBudgetSteps)-1] { + return nil + } for _, step := range FactoryBudgetSteps { if math.Abs(p.Budget-step) < 1e-9 { p.Budget = step return nil } } - return Errorf(CodeBadParams, "budget must be one of 5, 10, 20 or 30 US dollars") + return Errorf(CodeBadParams, "budget must be 5, 10, 20, 30 US dollars or the job's next cap") } func prepApprove(p *FactoryJobApproveParams) error { diff --git a/internal/factory/api/api_test.go b/internal/factory/api/api_test.go index 14a4c54d71..601f0f057c 100644 --- a/internal/factory/api/api_test.go +++ b/internal/factory/api/api_test.go @@ -496,7 +496,7 @@ func TestQuestionKindsOptionsAndCapNumbers(t *testing.T) { Stream: &factory.Stream{Spent: 4.5, Cur: 0, Phases: []factory.Phase{{Name: "write"}}}, } qs := FromItem(capJob).Questions - if len(qs) != 1 || qs[0].Kind != "cap" || qs[0].Limit != 4 || qs[0].Spent != 4.5 || qs[0].NextLimit != 8 { + if len(qs) != 1 || qs[0].Kind != "cap" || qs[0].Limit != 4 || qs[0].Spent != 4.5 || qs[0].NextLimit != 5 { t.Fatalf("cap question %+v", qs) } if !reflect.DeepEqual(qs[0].Options, []string{"yes", "no"}) { diff --git a/internal/factory/api/codec.go b/internal/factory/api/codec.go index e4fb441bc7..072e17a0a3 100644 --- a/internal/factory/api/codec.go +++ b/internal/factory/api/codec.go @@ -274,7 +274,7 @@ func spendingLimit(it factory.Item) *SpendingLimit { if it.Stream != nil { spent = it.Stream.Spent } - return &SpendingLimit{Limit: it.Cap, Spent: spent, NextLimit: factory.NextJobCap(it.Cap), Currency: factory.CurrencyUSD} + return &SpendingLimit{Limit: it.Cap, Spent: spent, NextLimit: factory.NextJobCap(it), Currency: factory.CurrencyUSD} } func failureOf(it factory.Item) *Failure { @@ -1001,9 +1001,7 @@ func annotateQuestion(q *Question, it factory.Item) { if it.Stream != nil { q.Spent = it.Stream.Spent } - if it.Cap > 0 { - q.NextLimit = factory.NextJobCap(it.Cap) - } + q.NextLimit = factory.NextJobCap(it) } if it.QKind == factory.QKindOutOfAttempts && len(q.Options) == 0 { q.Options = append([]string(nil), factory.OutOfAttemptOptions...) diff --git a/internal/factory/api/needs_test.go b/internal/factory/api/needs_test.go index e1bc1e1591..a843a723b8 100644 --- a/internal/factory/api/needs_test.go +++ b/internal/factory/api/needs_test.go @@ -94,7 +94,7 @@ func itoa(n int) string { return string(b[i:]) } -func TestSpendingLimitIsTheDoubledCap(t *testing.T) { +func TestSpendingLimitIsTheNextAllowedBudget(t *testing.T) { j := FromItem(factory.Item{ ID: 4, State: factory.StateNeedsYou, Question: "budget of $5 reached", QKind: "cap", Cap: 5, Starter: "raj", Stream: &factory.Stream{Spent: 5, Cur: 0, Phases: []factory.Phase{{Name: "write"}}}, diff --git a/internal/factory/api/wire.go b/internal/factory/api/wire.go index 76aba5bbcc..511f8a5a40 100644 --- a/internal/factory/api/wire.go +++ b/internal/factory/api/wire.go @@ -606,7 +606,8 @@ type NeedsQuestion struct { } // SpendingLimit is the cap a continue answer would raise. next_limit comes -// from codeaf. The app never sends it. currency is USD. +// from codeaf: the smallest allowed budget above spend, or zero when none +// remains. The app never sends it as an answer. currency is USD. type SpendingLimit struct { Limit float64 `json:"limit"` Spent float64 `json:"spent"` diff --git a/internal/factory/needs.go b/internal/factory/needs.go index a68f05e07e..9917c06dd2 100644 --- a/internal/factory/needs.go +++ b/internal/factory/needs.go @@ -153,14 +153,29 @@ type Acts struct { Reply bool } -// NextJobCap is the ceiling a continue answer allows. THE NUMBER COMES FROM -// codeaf, NEVER FROM THE APP: it is this job's cap doubled. The TUI's cap -// ladder is a different control and is not this figure. -func NextJobCap(cap float64) float64 { - if cap <= 0 { - return 0 - } - return cap * 2 +// JobBudgetSteps are the standard absolute job budgets, in ascending US dollars. +func JobBudgetSteps() []float64 { + return []float64{5, 10, 20, 30} +} + +// NextJobCap chooses the smallest standard budget above recorded spending, +// then doubles the current cap until it covers that spending. +func NextJobCap(it Item) float64 { + spent := JobSpent(it) + steps := JobBudgetSteps() + for _, budget := range steps { + if budget > spent { + return budget + } + } + next := it.Cap * 2 + if next <= 0 { + next = steps[len(steps)-1] * 2 + } + for next <= spent { + next *= 2 + } + return next } // ContinueJob raises this job's cap to NextJobCap when the spend has already @@ -170,7 +185,7 @@ func ContinueJob(it *Item) bool { if it == nil || it.Cap <= 0 || it.Stream == nil || it.Stream.Spent < it.Cap { return false } - it.Cap = NextJobCap(it.Cap) + it.Cap = NextJobCap(*it) return true } diff --git a/internal/factory/needs_test.go b/internal/factory/needs_test.go index 944d3ced4d..9dc0968772 100644 --- a/internal/factory/needs_test.go +++ b/internal/factory/needs_test.go @@ -6,8 +6,8 @@ import ( ) func TestContinueRaisesThisJobOnce(t *testing.T) { - if NextJobCap(5) != 10 || NextJobCap(0) != 0 || NextJobCap(-1) != 0 { - t.Fatalf("next = %v %v", NextJobCap(5), NextJobCap(0)) + if NextJobCap(Item{Cap: 5, Stream: &Stream{Spent: 5}}) != 10 || NextJobCap(Item{}) != 5 || NextJobCap(Item{Cap: 30, Stream: &Stream{Spent: 30}}) != 60 { + t.Fatalf("next = %v %v", NextJobCap(Item{Cap: 5, Stream: &Stream{Spent: 5}}), NextJobCap(Item{})) } it := &Item{Cap: 5, Stream: &Stream{Spent: 5}} if !ContinueJob(it) || it.Cap != 10 { diff --git a/internal/factory/run/loop.go b/internal/factory/run/loop.go index 6775311525..3f29accbe6 100644 --- a/internal/factory/run/loop.go +++ b/internal/factory/run/loop.go @@ -1565,14 +1565,15 @@ func (lp *floorLoop) spender(c *loopCtl) func(usd float64) { } // capCheck parks the item when its budget is reached after a round: yes raises -// the budget by as much again, no stops it. It answers false when it stopped. +// the budget to the next allowed amount above spend, no stops it. +// It answers false when it stopped. func (lp *floorLoop) capCheck(c *loopCtl, i int, name string) bool { it, err := lp.r.opts.Store.Get(c.id) if err != nil { return false } reached := it.Cap > 0 && it.Stream != nil && it.Stream.Spent >= it.Cap - q, kind := "budget of "+loopUSD(it.Cap)+" reached · "+loopUSD(it.Cap)+" more, or stop?", "cap" + q, kind := (*Money)(nil).CapQuestion(it) if p := lp.r.opts.Pool; p != nil { reached = p.CapReached(it) if reached { diff --git a/internal/factory/run/loop_test.go b/internal/factory/run/loop_test.go index e55481ebf2..207f055bc2 100644 --- a/internal/factory/run/loop_test.go +++ b/internal/factory/run/loop_test.go @@ -955,7 +955,7 @@ func TestTheCapAsksAndYesRaisesIt(t *testing.T) { if err := g.r.Launch(id); err != nil { t.Fatal(err) } - it := g.asked(id, "budget of $1 reached · $1 more, or stop?") + it := g.asked(id, "budget of $1 reached · $1.50 spent · raise it to $5, or stop?") if it.QKind != "cap" || it.Stream.Spent != 1.5 { t.Fatalf("kind %q spent %v", it.QKind, it.Stream.Spent) } @@ -963,7 +963,7 @@ func TestTheCapAsksAndYesRaisesIt(t *testing.T) { t.Fatal(err) } it = g.waitState(id, factory.StateLanded) - if it.Cap != 2 || !logHas(it, "budget raised to $2 · carrying on") { + if it.Cap != 5 || !logHas(it, "budget raised to $5 · carrying on") { t.Fatalf("cap %v log %+v", it.Cap, it.Stream.Log) } } diff --git a/internal/factory/run/money.go b/internal/factory/run/money.go index 9765163d91..0f36ee8cff 100644 --- a/internal/factory/run/money.go +++ b/internal/factory/run/money.go @@ -188,8 +188,9 @@ func (m *Money) CapReached(it factory.Item) bool { // CapQuestion is the sentence the item waits on, and its QKind. func (m *Money) CapQuestion(it factory.Item) (q string, kind string) { - c := dollars(it.Cap) - return "budget of " + c + " reached · " + c + " more, or stop?", "cap" + spent := factory.JobSpent(it) + q = "budget of " + dollars(it.Cap) + " reached · " + dollars(spent) + " spent · " + return q + "raise it to " + dollars(factory.NextJobCap(it)) + ", or stop?", "cap" } func dollars(usd float64) string { diff --git a/internal/factory/run/money_test.go b/internal/factory/run/money_test.go index ffedcedee5..4ab50a44ce 100644 --- a/internal/factory/run/money_test.go +++ b/internal/factory/run/money_test.go @@ -86,10 +86,10 @@ func TestCapReachedAndQuestion(t *testing.T) { t.Fatal("at the cap") } q, kind := m.CapQuestion(it) - if q != "budget of $5 reached · $5 more, or stop?" || kind != "cap" { + if q != "budget of $5 reached · $5 spent · raise it to $10, or stop?" || kind != "cap" { t.Fatalf("%q %q", q, kind) } - if q, _ := m.CapQuestion(factory.Item{Cap: 2.5}); q != "budget of $2.50 reached · $2.50 more, or stop?" { + if q, _ := m.CapQuestion(factory.Item{Cap: 2.5}); q != "budget of $2.50 reached · $0 spent · raise it to $5, or stop?" { t.Fatal(q) } } @@ -131,7 +131,7 @@ func TestRailRead(t *testing.T) { func TestBudgetQuestionSpeaksTheFloorsWord(t *testing.T) { m := NewMoney(nil, nil, nil) q, kind := m.CapQuestion(factory.Item{Cap: 5}) - if q != "budget of $5 reached · $5 more, or stop?" { + if q != "budget of $5 reached · $0 spent · raise it to $5, or stop?" { t.Fatalf("question = %q", q) } if strings.Contains(q, "cap") { diff --git a/internal/factory/run/stagereport_test.go b/internal/factory/run/stagereport_test.go index 241100af85..cd6f9ad305 100644 --- a/internal/factory/run/stagereport_test.go +++ b/internal/factory/run/stagereport_test.go @@ -356,7 +356,7 @@ func TestJobBudgetResumeKeepsChatAndDoesNotDoubleChargeLiveSpend(t *testing.T) { if err := g.r.Launch(id); err != nil { t.Fatal(err) } - it := g.asked(id, "budget of $1 reached · $1 more, or stop?") + it := g.asked(id, "budget of $1 reached · $1 spent · raise it to $5, or stop?") if it.Stream.Spent != 1 { t.Fatalf("double charged %v", it.Stream.Spent) } @@ -364,7 +364,7 @@ func TestJobBudgetResumeKeepsChatAndDoesNotDoubleChargeLiveSpend(t *testing.T) { t.Fatal(err) } it = g.waitState(id, factory.StateLanded) - if calls != 2 || it.Stream.Spent != 1 || it.Cap != 2 { + if calls != 2 || it.Stream.Spent != 1 || it.Cap != 5 { t.Fatalf("calls %v spent %v cap %v", calls, it.Stream.Spent, it.Cap) } } @@ -399,7 +399,7 @@ func TestJobBudgetCancelsNonChatExecutorWhenManagerSpends(t *testing.T) { case <-time.After(2 * time.Second): t.Fatal("shared cap did not cancel executor") } - g.asked(id, "budget of $1 reached · $1 more, or stop?") + g.asked(id, "budget of $1 reached · $1 spent · raise it to $5, or stop?") if err := g.r.Answer(id, true, ""); err != nil { t.Fatal(err) } diff --git a/internal/factory/serve/jobs.go b/internal/factory/serve/jobs.go index f344947898..3f1c56db46 100644 --- a/internal/factory/serve/jobs.go +++ b/internal/factory/serve/jobs.go @@ -749,8 +749,8 @@ func (h *Handler) raiseBudget(w http.ResponseWriter, r *http.Request, it factory if !h.decode(w, r, &body, false) || !h.fresh(w, r, it, "raise_budget", body.Expect) { return } - if !slices.Contains([]float64{5, 10, 20, 30}, body.Budget) { - h.fail(w, 422, "choose a budget: 5, 10, 20 or 30 USD") + if !slices.Contains(factory.JobBudgetSteps(), body.Budget) && body.Budget != factory.NextJobCap(it) { + h.fail(w, 422, "choose a budget: 5, 10, 20, 30 USD or the job's next budget") return } var budgetRefused error diff --git a/internal/factory/serve/verbs.go b/internal/factory/serve/verbs.go index 4e00e7f0f3..118fbd1e40 100644 --- a/internal/factory/serve/verbs.go +++ b/internal/factory/serve/verbs.go @@ -275,10 +275,11 @@ func (h *Handler) answer(w http.ResponseWriter, r *http.Request, it factory.Item if h.workers != nil { now = h.workers.now() } - if err := h.config.Store.Update(it.ID, func(item *factory.Item) error { + err := h.config.Store.Update(it.ID, func(item *factory.Item) error { run.NoteContinue(item, now) return nil - }); h.failed(w, err) { + }) + if h.failed(w, err) { return } } diff --git a/internal/factory/serve/workers_test.go b/internal/factory/serve/workers_test.go index ad01ee3cdc..255a15b36d 100644 --- a/internal/factory/serve/workers_test.go +++ b/internal/factory/serve/workers_test.go @@ -369,6 +369,9 @@ func TestWorkerDiffReportsOneQuestionOnTheWire(t *testing.T) { stamp := time.Unix(0, f.clock.Load()).UTC() want := []api.Question{{Text: cur.Question, State: "open", Kind: wantKind, FromStep: stage, With: factory.AskedYou, Options: cur.Asking.Options, Pick: "yes", Why: cur.Asking.Why, At: &stamp}} + if kind == "cap" { + want[0].NextLimit = 5 + } if !reflect.DeepEqual(got.Questions, want) { t.Fatalf("GET job questions = %+v; want %+v", got.Questions, want) } From 158a8ed93c3537dc729ba662a0decb2c8caf7146 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:53:56 -0500 Subject: [PATCH 12/28] session: a note sent during a final answer gets its read receipt (#2259) * test: force task notes to arrive before the final answer ends Co-Authored-By: Claude Opus 5.5 * test: hold the worker request rather than its helper Co-Authored-By: Claude Opus 5.5 * fix: keep task directions queued until a worker request reads them Turn cleanup recorded late directions and emptied the queue before the runner checked it. The task could finish with saved notes that no worker request had read. Keep those directions queued for the runner to take another turn and stamp the keyed reads. Co-Authored-By: Claude Opus 5.5 * test: clarify the final-answer note regression fixture Co-Authored-By: Claude Opus 5.5 * docs: distinguish a missing note receipt from lost words Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appnote_engine_test.go | 73 +++++++++++++++++-- .../unreleased/2259-task-note-final-answer.md | 13 ++++ internal/session/agent.go | 12 ++- 3 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 docs/changes/unreleased/2259-task-note-final-answer.md diff --git a/cmd/codeaf/appnote_engine_test.go b/cmd/codeaf/appnote_engine_test.go index 239745b3b4..dcdfe2b228 100644 --- a/cmd/codeaf/appnote_engine_test.go +++ b/cmd/codeaf/appnote_engine_test.go @@ -5,7 +5,7 @@ package main // The registered app server answers `tasks.note`; the backend sends it through // the remote socket (remote.Loopback) to a real session engine built the public // way (session.New) and running a task admitted the public way (StartTask) -// against a local model that holds its first request; the task's worker reads +// against a local model that holds its first worker answer; the task's worker reads // the notes at its next request, which is the producer stamping the step; the // watcher says `note.changed read` for the key and only for the key; and an // engine opened again on the same session file holds the same keys on the same @@ -16,6 +16,8 @@ import ( "context" "encoding/json" "io" + "net/http" + "net/http/httptest" "path/filepath" "strconv" "testing" @@ -28,7 +30,7 @@ import ( ) func TestATaskNoteCrossesTheAppServerAndTheSocketToTheRealNodeAndIsReadByItsKey(t *testing.T) { - model := newHeldModel(t) + model := newFinishingNoteModel(t) sources := modelsource.NewSet(modelsource.Connected{ Source: modelsource.DefaultSource(model.server.URL), Key: "test-key", Address: model.server.URL, }) @@ -49,10 +51,24 @@ func TestATaskNoteCrossesTheAppServerAndTheSocketToTheRealNodeAndIsReadByItsKey( if err != nil { t.Fatal(err) } - select { - case <-model.arrived: - case <-time.After(30 * time.Second): - t.Fatal("the task's worker never reached the model") + events, stop, err := engine.WatchTaskRoom(id) + if err != nil { + t.Fatal(err) + } + defer stop() + // A productive request is not cut by a note. Hold its final answer so all + // three notes arrive after its last request and before the turn ends. + ready := time.After(30 * time.Second) +waiting: + for { + select { + case event := <-events: + if event.Kind == session.EventTextDelta && event.Text == "working" { + break waiting + } + case <-ready: + t.Fatal("the task's worker never streamed its answer") + } } local := strconv.FormatUint(id, 10) @@ -145,9 +161,13 @@ func TestATaskNoteCrossesTheAppServerAndTheSocketToTheRealNodeAndIsReadByItsKey( for key, entry := range map[string]appwire.Entry{"k-1": first, "k-2": second} { got, ok := read[key] if !ok || got.ReadAtStep == nil || got.ID != entry.ID { - t.Fatalf("no read event for %q from the real producer: %+v", key, read) + view, _, pageErr := near.AppTask(local) + t.Fatalf("no read event for %q from the real producer: %+v; state=%s notes=%+v page_error=%v model_saw_words=%v", key, read, view.State, view.Notes, pageErr, model.sentWords("same words")) } } + if !model.sentWords("same words") { + t.Fatal("the notes were marked read without reaching the model") + } // With no key nothing may join the note: not its words, not its place. time.Sleep(2*appNoteBeat + 200*time.Millisecond) for _, changed := range noteEvents(rec) { @@ -190,3 +210,42 @@ func TestATaskNoteCrossesTheAppServerAndTheSocketToTheRealNodeAndIsReadByItsKey( t.Fatalf("keys after the restart = %q, before %q", got, keys(view.Notes)) } } + +// newFinishingNoteModel streams words before holding the final answer. The +// task room's text event proves the worker has received them before notes land. +func newFinishingNoteModel(t *testing.T) *heldModel { + t.Helper() + model := &heldModel{release: make(chan struct{})} + model.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request recordedRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + model.mu.Lock() + model.requests = append(model.requests, request) + first := false + if len(request.Tools) > 0 { + model.turns++ + first = model.turns == 1 + } + model.mu.Unlock() + if !request.Stream { + _, _ = io.WriteString(w, `{"choices":[{"message":{"role":"assistant","content":"done"},"finish_reason":"stop"}]}`) + return + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"working\"}}]}\n\n") + w.(http.Flusher).Flush() + if first { + select { + case <-model.release: + case <-r.Context().Done(): + return + } + } + _, _ = io.WriteString(w, "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n") + })) + t.Cleanup(model.server.Close) + return model +} diff --git a/docs/changes/unreleased/2259-task-note-final-answer.md b/docs/changes/unreleased/2259-task-note-final-answer.md new file mode 100644 index 0000000000..2d51172bbd --- /dev/null +++ b/docs/changes/unreleased/2259-task-note-final-answer.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: codeaf workers read notes delivered while their final answer is streaming +pr: 2259 +surface: [engine, app, chat] +invalidates: + - "A task could finish with an accepted note saved under its key but without a read receipt, because turn cleanup emptied the worker's queue before its runner checked for more words. Pending directions now stay queued until the worker's next request carries them and records the read step." +--- + +A note accepted while a worker finishes its answer now reaches the worker's next +request before that worker closes. Its read event keeps the same client key. +The note's words and key remain saved across reopening as before. No wire +methods, fields or limits change. diff --git a/internal/session/agent.go b/internal/session/agent.go index 131113cba1..c65f00ccab 100644 --- a/internal/session/agent.go +++ b/internal/session/agent.go @@ -2186,9 +2186,17 @@ func (a *Agent) startTurnLocked(ctx context.Context, user userMessage, watcher * // lifted off the queue first and becomes a message waiting for a turn // of its own (steer.go's [Agent.liftSteersLocked]) — which the // follow-up drain below then starts, on the follow-up queue's own - // terms. Everything else on the queue drains exactly as it always has. + // terms. a.liftSteersLocked(hub) - drained := a.drainQueueLocked(hub, true) + // A worker's directions stay queued until a request carries them. + // Its runner checks that queue before closing the worker and takes + // another turn when words remain. Draining here would hide them + // from that check without ever putting them in front of the model. + // A room thread owns its wake instead of being driven by a runner. + var drained queueDrain + if !a.config.InTask || a.config.roomThread || len(queuedDirections(a.steering)) == 0 { + drained = a.drainQueueLocked(hub, true) + } unanswered := drained.owed // A PROGRAM'S ENDING THIS TURN COULD NOT ANSWER IS SAID ANYWAY, in // the program's own words (program_outcome.go), and a person's stop From b39a5602cec7b42040a7d17d043077b6ba1d0bf6 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:54:09 -0500 Subject: [PATCH 13/28] factory: a job's Brief counts a person's later requests as asked (#2262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: expose later person requests to the job brief Co-Authored-By: Claude Opus 5.5 * fix: judge job briefs against later person requests Carry gate answers and saved notes into the writer input so requested work is judged against the person’s current request. Preserve bounded requests through input cuts and include them in freshness checks. Co-Authored-By: Claude Opus 5.5 * test: prepare paired live brief request probe Compare the same scripted work with and without person requests using the pinned live model. Print checked verdicts and blocks, and reject the false-fault vocabulary seen in the report. The probe is opt-in and left for the caller to run. Co-Authored-By: Claude Opus 5.5 * test: log why a live brief reply ended Co-Authored-By: Claude Opus 5.5 * test: a live brief names the request in its text The verdict is at most fourteen words and may refer to the request by name. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * manual: say Brief, not the writer's word for its headline Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ...ory_brief_person_requests_live_e2e_test.go | 135 ++++++++++++++++++ .../unreleased/2262-brief-person-requests.md | 18 +++ .../factory/brief_person_requests_test.go | 107 ++++++++++++++ internal/factory/brief_prompt.go | 7 +- internal/factory/brief_writer.go | 59 +++++++- internal/manual/chat_test.go | 16 +++ 6 files changed, 337 insertions(+), 5 deletions(-) create mode 100644 cmd/codeaf/appfactory_brief_person_requests_live_e2e_test.go create mode 100644 docs/changes/unreleased/2262-brief-person-requests.md create mode 100644 internal/factory/brief_person_requests_test.go diff --git a/cmd/codeaf/appfactory_brief_person_requests_live_e2e_test.go b/cmd/codeaf/appfactory_brief_person_requests_live_e2e_test.go new file mode 100644 index 0000000000..14e7d44f59 --- /dev/null +++ b/cmd/codeaf/appfactory_brief_person_requests_live_e2e_test.go @@ -0,0 +1,135 @@ +//go:build e2e + +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" +) + +// The paired live probe changes only person_requests, using the public writer +// input and draft checker. It makes exactly two calls, without model fallback. +func TestLiveBriefPersonRequests(t *testing.T) { + if os.Getenv(liveFactoryOptIn) != "1" { + t.Skip("set CODEAF_LIVE_FACTORY=1 to run the paired live probe; spend cap $1") + } + if liveFactoryKey == "" { + t.Fatal("BLOCKED: no OpenRouter key") + } + t.Setenv("CODEAF_MODEL_POOL", "off") + recorder := newLiveRecorder(t, config.DefaultBaseURL) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + const words = "also add a second line: see you" + it := factory.Item{Title: "Farewell", Body: "Add exactly one line saying farewell", State: factory.StateLanded, + AnsweredQuestions: []factory.Question{{FromStep: "test", Kind: "approve", Answer: &factory.Answer{Text: words, At: time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)}}}, + Change: &factory.Change{Stat: &factory.ChangeStat{Files: 1, Added: 2}, Files: []factory.ChangedFile{{Path: "farewell.txt", Added: 2}}, Patch: "diff --git a/farewell.txt b/farewell.txt\n--- /dev/null\n+++ b/farewell.txt\n@@ -0,0 +1,2 @@\n+farewell\n+see you\n"}, + Stream: &factory.Stream{Phases: []factory.Phase{ + {Name: "plan", State: factory.PhaseDone, Round: 1, Result: &factory.StageResult{Done: true, Output: "Add exactly one farewell line to farewell.txt."}}, + {Name: "write", State: factory.PhaseDone, Round: 2, Result: &factory.StageResult{Done: true, Output: "farewell.txt now contains two lines:\nfarewell\nsee you"}}, + {Name: "test", State: factory.PhaseDone, Round: 2, Result: &factory.StageResult{Done: true, Output: "Read farewell.txt: farewell followed by see you. Two lines present.", Tests: &factory.TestCounts{Command: "check farewell.txt", Passed: 1}}}, + }}, + } + in := factory.BriefInputOf(it) + verdicts := map[string]string{} + texts := map[string]string{} + for _, label := range []string{"with_request", "without_request"} { + if label == "without_request" { + in.PersonRequests = nil + } + input := factory.BriefInputText(in) + t.Logf("LIVE BRIEF %s input=%s", label, input) + raw, err := json.Marshal(map[string]any{ + "model": liveFactoryModel, "temperature": 0, "max_tokens": briefMaxTokens, "stream": false, + "response_format": map[string]string{"type": "json_object"}, + "messages": []map[string]string{{"role": "system", "content": factory.BriefSystem()}, {"role": "user", "content": input}}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, recorder.server.URL+"/chat/completions", bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+liveFactoryKey) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s: provider status %d", label, resp.StatusCode) + } + var reply struct { + Model string `json:"model"` + Choices []struct { + Finish string `json:"finish_reason"` + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage map[string]any `json:"usage"` + } + if err := json.Unmarshal(body, &reply); err != nil { + t.Fatal(err) + } + if reply.Model != liveFactoryModel || len(reply.Choices) != 1 { + t.Fatalf("unexpected model or choice count: %q, %d", reply.Model, len(reply.Choices)) + } + draft := reply.Choices[0].Message.Content + t.Logf("LIVE BRIEF %s finish=%s usage=%v draft=%s", label, reply.Choices[0].Finish, reply.Usage, draft) + v, drops, fault := factory.CheckBriefDraft(it, []byte(draft)) + t.Logf("LIVE BRIEF %s verdict=%q blocks=%+v drops=%+v fault=%q", label, v.Verdict, v.Blocks, drops, fault) + if fault != "" || len(v.Blocks) == 0 { + t.Fatalf("%s: no checked brief", label) + } + verdicts[label] = v.Verdict + checked, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + texts[label] = strings.ToLower(string(checked)) + } + recorder.mu.Lock() + defer recorder.mu.Unlock() + t.Logf("LIVE BRIEF calls=%d cost_usd=%.6f models=%v problems=%v", recorder.calls, recorder.cost, recorder.replies, recorder.problems) + if recorder.cost > 1 || recorder.calls != 2 || len(recorder.problems) != 0 || len(recorder.replies) != 1 || recorder.replies[liveFactoryModel] != 2 { + t.Fatal("paired probe exceeded its budget or model/call limits") + } + if !strings.Contains(texts["with_request"], "see you") { + t.Error("brief did not name the later request") + } + // These checks catch the reported false-fault vocabulary; the paired logs + // retain the full checked text for the caller to assess the judgment. + accepted := strings.ToLower(verdicts["with_request"]) + if !strings.Contains(accepted, "asked") && !strings.Contains(accepted, "request") { + t.Error("verdict did not treat the later request as asked") + } + fault := regexp.MustCompile(`(?i)fault|surpris|scope creep|unrequested|unwanted|exceed|mismatch|violat`) + if fault.MatchString(texts["with_request"]) { + t.Error("requested work still described as a fault; inspect paired blocks") + } + discrepancy := regexp.MustCompile(`(?i)extra|exceed|mismatch|scope|unrequested|unwanted|not asked|only one|exactly one|single.line|one.line|not met|instead of|violat`) + if !discrepancy.MatchString(texts["without_request"]) { + t.Error("control did not identify the discrepancy against the issue and plan") + } + if verdicts["with_request"] == verdicts["without_request"] { + t.Error("removing person_requests did not change the verdict; inspect paired blocks") + } +} diff --git a/docs/changes/unreleased/2262-brief-person-requests.md b/docs/changes/unreleased/2262-brief-person-requests.md new file mode 100644 index 0000000000..893e924fbc --- /dev/null +++ b/docs/changes/unreleased/2262-brief-person-requests.md @@ -0,0 +1,18 @@ +--- +kind: fixed +title: Judge a job against the person's later requests +pr: 2262 +surface: [engine] +invalidates: + - "Was true: the Brief read only the issue and plan as the request. Is true now: it also reads later answers and saved notes, and later requests take precedence." +--- + +The job Brief now reads what you asked at later gates and in saved notes. +Work that follows your later request is treated as asked, rather than a fault, +surprise or scope creep. A real miss against your request is still reported. +Adding a request makes an earlier Brief out of date. + +The writer keeps the newest 12 distinct requests, up to 2 KiB per text and +16 KiB of request JSON. Requests survive the ordinary input cuts. Answers +retain their step, gate and time; notes have no recorded gate or time and +keep their saved order before dated answers. diff --git a/internal/factory/brief_person_requests_test.go b/internal/factory/brief_person_requests_test.go new file mode 100644 index 0000000000..48ca13824e --- /dev/null +++ b/internal/factory/brief_person_requests_test.go @@ -0,0 +1,107 @@ +package factory_test + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/Agent-Field/codeaf/internal/factory" +) + +func TestBriefPersonRequests(t *testing.T) { + at := time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC) + const words = "also add a second line: see you" + it := factory.Item{Title: "Farewell", Body: "Add a line saying farewell"} + before := factory.BriefInputHash(factory.BriefInputOf(it)) + it.AnsweredQuestions = []factory.Question{{FromStep: "test", Kind: "approve", Answer: &factory.Answer{Text: words, At: at}}} + in := factory.BriefInputOf(it) + var got struct { + Requests []struct { + Text string `json:"text"` + Step string `json:"step"` + Gate string `json:"gate"` + At time.Time `json:"at"` + } `json:"person_requests"` + } + if err := json.Unmarshal([]byte(factory.BriefInputText(in)), &got); err != nil { + t.Fatal(err) + } + if len(got.Requests) != 1 { + t.Fatalf("writer input person_requests = %d; want later request %q", len(got.Requests), words) + } + r := got.Requests[0] + if r.Text != words || r.Step != "test" || r.Gate != "approve" || !r.At.Equal(at) { + t.Fatalf("later request lost words or context: %+v", r) + } + if factory.BriefInputHash(in) == before { + t.Error("a person's later request did not change the brief input hash") + } + for _, rule := range []string{"issue plus person_requests", "later request wins", "never reported as a fault", "real miss"} { + if !strings.Contains(factory.BriefSystem(), rule) { + t.Errorf("BriefSystem missing request rule %q", rule) + } + } + t.Run("sources and oldest first", func(t *testing.T) { + job := factory.Item{Notes: []string{"first note", words, "last note"}, AnsweredQuestions: []factory.Question{ + {}, {Answer: &factory.Answer{Text: " continue "}}, {Answer: &factory.Answer{Text: "YES"}}, {Answer: &factory.Answer{Text: " No "}}, {Answer: &factory.Answer{Text: " "}}, + {FromStep: "proof", Answer: &factory.Answer{Text: "no, keep the extra line", At: at.Add(time.Hour)}}, + {FromStep: "test", Answer: &factory.Answer{Text: words, At: at}}, + }} + requests := factory.BriefInputOf(job).PersonRequests + var texts []string + for _, r := range requests { + texts = append(texts, r.Text) + } + if want := []string{"first note", "last note", words, "no, keep the extra line"}; !reflect.DeepEqual(texts, want) { + t.Fatalf("requests = %q; want %q", texts, want) + } + if !requests[0].At.IsZero() || requests[0].Step != "" || requests[2].Step != "test" { + t.Fatalf("invented or lost context: %+v", requests) + } + }) + t.Run("newest survive count and input cuts", func(t *testing.T) { + job := factory.Item{} + for i := range 20 { + job.Notes = append(job.Notes, fmt.Sprintf("request %02d", i)) + } + input := factory.BriefInputOf(job) + if len(input.PersonRequests) != 12 || input.PersonRequests[0].Text != "request 08" || input.PersonRequests[11].Text != "request 19" { + t.Fatalf("count cuts did not keep newest: %+v", input.PersonRequests) + } + want := append([]factory.BriefPersonRequest(nil), input.PersonRequests...) + input.Plan = strings.Repeat("plan", 20000) + input.Steps = []factory.BriefStepInput{{Output: strings.Repeat("output", 20000)}} + input.Previous = &factory.BriefVersion{Verdict: strings.Repeat("previous", 20000)} + raw := factory.BriefInputText(input) + var decoded factory.BriefInput + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + t.Fatal(err) + } + if len(raw) > factory.BriefInputMost || !reflect.DeepEqual(decoded.PersonRequests, want) { + t.Fatalf("input cuts lost requests or exceeded cap: %d bytes, %+v", len(raw), decoded.PersonRequests) + } + }) + t.Run("text and encoded byte bounds", func(t *testing.T) { + job := factory.Item{Notes: []string{strings.Repeat("界", 1000)}} + requests := factory.BriefInputOf(job).PersonRequests + if len(requests) != 1 || len(requests[0].Text) > 2<<10 || !utf8.ValidString(requests[0].Text) { + t.Fatalf("invalid bounded text: %+v", requests) + } + job.Notes = nil + for i := range 12 { + job.Notes = append(job.Notes, fmt.Sprintf("%02d", i)+strings.Repeat("\x00", 2000)) + } + requests = factory.BriefInputOf(job).PersonRequests + raw, err := json.Marshal(requests) + if err != nil { + t.Fatal(err) + } + if len(raw) > 16<<10 || len(requests) == 0 || !strings.HasPrefix(requests[len(requests)-1].Text, "11") { + t.Fatalf("JSON byte bound lost newest: %d bytes, %d requests", len(raw), len(requests)) + } + }) +} diff --git a/internal/factory/brief_prompt.go b/internal/factory/brief_prompt.go index e864b03b5f..ee8c6090fd 100644 --- a/internal/factory/brief_prompt.go +++ b/internal/factory/brief_prompt.go @@ -15,7 +15,9 @@ const BriefInputMost = 40 << 10 // constants, so the prompt and the check that reads the answer cannot drift. func BriefSystem() string { return fmt.Sprintf(`You write the brief of one factory job: its state in one screen, for the person who owns it. -You are given the job as JSON: the issue, the plan, each step's result per try, the change's stats and hunk index, the rule checks, the facts row and the brief before. +You are given the job as JSON: the issue, person_requests, the plan, each step's result per try, the change's stats and hunk index, the rule checks, the facts row and the brief before. +What was asked is the issue plus person_requests, oldest first. A later request wins over the issue and the plan where they disagree. +Work that follows a person's request is never reported as a fault, a surprise or scope creep. The verdict names the request as asked. A real miss against a request is still reported. Judge the work against those requests using the recorded evidence. Answer ONE JSON object and nothing else: {"verdict": string, "verdict_evidence": [evidence], "blocks": [block]} Every "evidence" field is a JSON ARRAY of {"step": string, "try": number, "kind": one of %s, "ref": string}. tests, exit, findings, claims and output name a step's latest try; summary names any try; file names a changed file in ref; change is the whole change's stat (files, added, removed); hunk names "file:start" in ref; rule_checks names no step; pr names the published PR and shows its number, with no step. @@ -38,7 +40,8 @@ EVERY NUMBER YOU WRITE MUST BE ONE ITS EVIDENCE SHOWS; a claim or block that bre // BriefInputText is the input as the writer reads it, cut to // [BriefInputMost]: the hunk index first, then the steps' outputs, then the -// issue's body, so what is kept is what the brief rests on most. +// issue's body and plan, then the previous brief. Person requests are bounded +// when assembled and never removed by these cuts: they define what was asked. func BriefInputText(in BriefInput) string { cuts := []func(*BriefInput){ func(in *BriefInput) { in.Hunks = in.Hunks[:len(in.Hunks)/2] }, diff --git a/internal/factory/brief_writer.go b/internal/factory/brief_writer.go index 97383cf0d5..5e9823c4a7 100644 --- a/internal/factory/brief_writer.go +++ b/internal/factory/brief_writer.go @@ -17,7 +17,7 @@ import ( // // The writer is one small model call, not the manager's chat turn. The // factory assembles everything it may read ([BriefInputOf]) from the job's -// own records: the issue, the plan, each step's structured result per try, +// own records: the issue, later person requests, the plan, each step's result per try, // the change's stats and a hunk index, the rules that applied and the brief // before. It is never the full diff. // @@ -39,8 +39,10 @@ const ( BriefDebounce = 10 * time.Second // briefOutputMost bounds each step output and the issue body the writer // reads, in bytes. - briefOutputMost = 2 << 10 - briefHunksMost = 40 + briefOutputMost = 2 << 10 + briefHunksMost = 40 + briefRequestsMost = 12 + briefRequestsBytesMost = 16 << 10 ) // The triggers of a write. A person's retry and an ask that opened are @@ -82,6 +84,15 @@ type BriefStepInput struct { Output string `json:"output,omitempty"` } +// BriefPersonRequest is a person's later words, with the context the job kept. +// Notes have no recorded step, gate or time; their saved order precedes dated answers. +type BriefPersonRequest struct { + Text string `json:"text"` + Step string `json:"step,omitempty"` + Gate string `json:"gate,omitempty"` + At time.Time `json:"at,omitzero"` +} + // BriefInput is everything the writer reads. Previous is not part of the // input's hash: a new version changes it, and that alone is no reason to // write another. @@ -102,6 +113,7 @@ type BriefInput struct { Question string `json:"question,omitempty"` Model string `json:"model,omitempty"` Plan string `json:"plan,omitempty"` + PersonRequests []BriefPersonRequest `json:"person_requests,omitempty"` Steps []BriefStepInput `json:"steps"` Change *ChangeStat `json:"change,omitempty"` Hunks []PatchHunk `json:"hunks,omitempty"` @@ -190,6 +202,7 @@ func BriefVerdictBehind(it Item) bool { // BriefInputOf assembles the writer's input from the job. func BriefInputOf(it Item) BriefInput { var in BriefInput + in.PersonRequests = briefPersonRequests(it) in.Issue.Title, in.Issue.Body, in.Issue.Labels = it.Title, clip(it.Body, briefOutputMost), it.Labels in.NeedsYou, in.Question, in.Model = JobState(it) == "needs_you", it.Question, it.Model in.State = strings.ReplaceAll(string(it.State), "_", " ") @@ -232,6 +245,46 @@ func BriefInputOf(it Item) BriefInput { return in } +// briefPersonRequests keeps the newest distinct words without inventing note clocks. +func briefPersonRequests(it Item) []BriefPersonRequest { + var requests []BriefPersonRequest + for _, text := range it.Notes { + if text = strings.TrimSpace(text); text != "" { + requests = append(requests, BriefPersonRequest{Text: text}) + } + } + for _, q := range it.AnsweredQuestions { + if q.Answer == nil { + continue + } + text := strings.TrimSpace(q.Answer.Text) + if text == "" || IsContinue(false, text) || strings.EqualFold(text, "no") { + continue + } + requests = append(requests, BriefPersonRequest{Text: text, Step: clip(q.FromStep, 128), Gate: clip(q.Kind, 128), At: q.Answer.At}) + } + slices.SortStableFunc(requests, func(a, b BriefPersonRequest) int { return a.At.Compare(b.At) }) + seen := map[string]bool{} + var kept []BriefPersonRequest + size := 2 // The JSON array's brackets. + for i := len(requests) - 1; i >= 0; i-- { + r := requests[i] + if seen[r.Text] { + continue + } + seen[r.Text] = true + r.Text = strings.ToValidUTF8(clip(r.Text, briefOutputMost), "") + raw, _ := json.Marshal(r) + size += len(raw) + 1 + if len(kept) == briefRequestsMost || size > briefRequestsBytesMost { + break + } + kept = append(kept, r) + } + slices.Reverse(kept) + return kept +} + // stepInputs is one line per finished try of a phase, then its latest // result, which carries the structured fields. func stepInputs(st Stage, p Phase) []BriefStepInput { diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 3f6efe3cc1..0cf2d48cdb 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -4796,3 +4796,19 @@ func TestSyncOffManualQuotesExactSentences(t *testing.T) { }) } } + +func TestFactoryBriefLaterRequestsReachInputs(t *testing.T) { + const heading = "Job Brief inputs" + for _, asked := range []string{"job brief later requests answers saved notes", "job brief inputs later requests"} { + sections := Chat().Search(asked, DefaultResults) + found := false + for _, section := range sections { + if section.Page == "factory" && strings.HasPrefix(section.Title, heading) { + found = true + } + } + if !found { + t.Errorf("%q does not reach %q", asked, heading) + } + } +} From 17ccbdd58d7d632ea4daf48f3938d9b8dc63aa70 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:54:22 -0500 Subject: [PATCH 14/28] test: two process and pipeline intermittents wait on the right signal (#2264) * test: force action reads while a light crew owner opens Co-Authored-By: Claude Opus 5.5 * test: wait for crew readiness before requiring live process actions Reads deliberately answer saved history while a light owner is opening. Check that nonblocking answer during a forced open, then require live actions after agent.changed instead of depending on scheduler timing. Co-Authored-By: Claude Opus 5.5 * test: reproduce preview refusal digits in a factory identity Pin the fixture identity to include 820 so the whole-error wording check fails consistently despite a clean displayed message. Co-Authored-By: Claude Opus 5.5 * test: prove the preview wording guard catches a displayed numeric leak Check only the displayed message and deliberately inject 820 there for the negative control. The following commit removes this test-only mutation after the remote guard verdict. Co-Authored-By: Claude Opus 5.5 * test: keep preview refusal wording checks on the displayed message Remove the negative-control message injection after confirming that the guard rejects 820 in displayed words. Factory identity digits remain deterministic and typed refusal details are still checked separately. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_pipeline_edit_test.go | 9 +++- cmd/codeaf/appprocessactions_test.go | 43 ++++++++++++++++++- .../2264-process-action-readiness-checks.md | 13 ++++++ 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 docs/changes/unreleased/2264-process-action-readiness-checks.md diff --git a/cmd/codeaf/appfactory_pipeline_edit_test.go b/cmd/codeaf/appfactory_pipeline_edit_test.go index bbdce396c6..4587405fe9 100644 --- a/cmd/codeaf/appfactory_pipeline_edit_test.go +++ b/cmd/codeaf/appfactory_pipeline_edit_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -528,6 +529,12 @@ func TestPipelinePreviewRefusalsAreTypedAndPlainlyWorded(t *testing.T) { t.Run(name, func(t *testing.T) { stub := newEditStub(t, api.Me{Mode: api.FactoryTeam}) r := newEditWire(t, stub, false, api.FactoryTeam, prToken) + // A factory id may contain digits forbidden in the displayed words. + r.id = "factory_0000000000000820" + r.hub.mu.Lock() + r.hub.rows[0].ID = r.id + r.hub.mu.Unlock() + r.hub.hold(r.id, prToken, 0) _, err := r.preview(c.base, c.edit) we := prWantCode(t, err, appwire.CodeBadParams) d, ok := we.Detail.(appwire.FactoryPipelineEditRefusal) @@ -537,7 +544,7 @@ func TestPipelinePreviewRefusalsAreTypedAndPlainlyWorded(t *testing.T) { if !strings.HasSuffix(we.Message, "Nothing was written.") { t.Fatal(we.Message) } - prNoNames(t, err, "deploy", "approve") + prNoNames(t, errors.New(we.Message), "deploy", "approve") }) } } diff --git a/cmd/codeaf/appprocessactions_test.go b/cmd/codeaf/appprocessactions_test.go index 13ebaadc2b..e79c36a13c 100644 --- a/cmd/codeaf/appprocessactions_test.go +++ b/cmd/codeaf/appprocessactions_test.go @@ -398,15 +398,21 @@ func TestStopAllOnANotHeldConversationNeverOpensItOrInventsAnEmptyJobSnapshot(t } func TestProcessActionsOpensALightCrewOwnerOnce(t *testing.T) { - b, _, paths, opens := lazyBackend(t, 1) + b, rec, paths, opens := lazyBackend(t, 1) b.crews() job := session.JobNotice{ID: 41, Kind: session.JobKindCommand, State: session.JobDone, Started: time.UnixMilli(1700000000000)} seedRecords(t, filepath.Dir(paths[0]), job) engine := &actorEngine{fakeConversation: &fakeConversation{}, record: job, facts: session.ProcessActions{ Record: job, Rerun: session.ProcessAction{Available: true}, }} + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + releaseOpen := func() { once.Do(func() { close(release) }) } + defer releaseOpen() opener := b.open b.open = func(ask appOpenAsk) (*appConn, error) { + close(entered) + <-release conn, err := opener(ask) if err == nil { conn.Conv = engine @@ -414,12 +420,45 @@ func TestProcessActionsOpensALightCrewOwnerOnce(t *testing.T) { return conn, err } id := appProcessID(appAgentID(paths[0]), job) + first := make(chan struct{}) + go func() { + defer close(first) + got, err := b.ProcessActions(context.Background(), id) + if err != nil || !got.Rerun.Available { + t.Errorf("opening caller's actions: %+v %v", got, err) + } + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("the owner was never asked to open") + } + t.Log("interleaving: first actions caller entered the opener; owner publication is blocked") var wg sync.WaitGroup + for range 12 { + wg.Go(func() { + got, err := b.ProcessActions(context.Background(), id) + if err != nil || got.State != appwire.ProcessEnded || got.Rerun.Available || got.Rerun.Reason != appwire.ProcessActionNotHeld { + t.Errorf("opening owner's saved actions: %+v %v", got, err) + } + }) + } + answersWithin(t, 2*time.Second, "actions while the owner opens", wg.Wait) + t.Log("interleaving: concurrent reads returned ended and not_held without waiting for the opener") + releaseOpen() + rec.waitFor(t, 5*time.Second, func(event string, data any) bool { + changed, ok := data.(appwire.AgentChanged) + return event == appwire.EventAgentChanged && ok && changed.Agent.ID == appAgentID(paths[0]) + }) + t.Log("interleaving: opener released; agent.changed received") + <-first + // The readiness event, rather than another caller's timing, makes an + // owner-backed answer available to every subsequent read. for range 12 { wg.Go(func() { got, err := b.ProcessActions(context.Background(), id) if err != nil || !got.Rerun.Available { - t.Errorf("light owner's actions: %+v %v", got, err) + t.Errorf("ready owner's actions: %+v %v", got, err) } }) } diff --git a/docs/changes/unreleased/2264-process-action-readiness-checks.md b/docs/changes/unreleased/2264-process-action-readiness-checks.md new file mode 100644 index 0000000000..ac30634922 --- /dev/null +++ b/docs/changes/unreleased/2264-process-action-readiness-checks.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: Process action checks respect a crew conversation still opening +pr: 2264 +surface: [engine] +invalidates: + - "Was assumed by the concurrent action check: every read of an idle crew member already has a live conversation. Is true: reads during opening can answer from saved history; the conversation announces readiness with agent.changed." +--- +An ended process can temporarily have Run again unavailable while its crew +conversation opens. Once the conversation announces readiness, fresh action +reads use its live answer. The regression check now waits for that signal and +also checks that reads during opening return promptly. Product behavior is +unchanged. From 3d0ce97924d9bd8192c83a59066bcb1eba07815f Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:51:57 -0500 Subject: [PATCH 15/28] factory: job memory calls count in its header and budget (#2261) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * Take the gated copy of the union merged docs --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf-replay/replay_test.go | 5 + cmd/codeaf/app_factory.go | 8 +- .../appfactory_budget_regression_test.go | 53 ++ cmd/codeaf/appfactory_memory_handoff_test.go | 184 ++++++ cmd/codeaf/appfactory_memory_law_test.go | 128 ++++ cmd/codeaf/appfactory_memory_live_e2e_test.go | 591 ++++++++++++++++++ cmd/codeaf/appfactory_memory_owner_test.go | 179 ++++++ cmd/codeaf/appfactory_memory_spend_test.go | 580 +++++++++++++++++ cmd/codeaf/appfactory_memory_tail_test.go | 215 +++++++ cmd/codeaf/appfactory_memory_worker_test.go | 429 +++++++++++++ cmd/codeaf/chatv3_knowledge.go | 2 +- cmd/codeaf/factory_chatspend.go | 17 + cmd/codeaf/factory_live_e2e_test.go | 11 +- cmd/codeaf/factory_stage.go | 23 +- cmd/codeaf/factory_stage_budget.go | 74 ++- .../unreleased/2261-memory-budget-cause.md | 18 + .../2261-memory-durable-stage-spend.md | 12 + .../2261-memory-paused-admission.md | 14 + .../unreleased/2261-memory-quit-settlement.md | 13 + .../unreleased/2261-memory-spend-counted.md | 40 ++ docs/design/app-v3/PROTOCOL.md | 12 + docs/knowledge.md | 61 ++ internal/factory/api/codec.go | 1 + internal/factory/report_receipt.go | 1 + internal/factory/run/chatstage.go | 9 +- internal/factory/run/contract.go | 5 + internal/factory/run/loop.go | 45 +- internal/factory/run/stagereport.go | 8 +- internal/factory/serve/workers.go | 29 + internal/factory/spend.go | 9 + internal/factory/types.go | 2 + internal/factory/worker/worker.go | 83 ++- internal/knowledge/accounting.go | 57 ++ internal/knowledge/build.go | 46 +- internal/knowledge/compiler.go | 105 +++- .../compiler_batch_regression_test.go | 42 ++ internal/knowledge/compiler_provider.go | 21 +- internal/knowledge/knowledge.go | 6 +- .../knowledge/refinement_concurrency_test.go | 4 + internal/knowledge/store.go | 23 +- internal/manual/chat/commands.md | 4 +- internal/manual/chat/desktop-sync.md | 6 + .../chat/factory-stage-conversations.md | 8 + internal/manual/chat/places.md | 69 +- .../manual/chat/running-on-another-machine.md | 7 +- internal/manual/chat/settings.md | 37 +- .../manual/chat/skills-from-other-tools.md | 4 +- .../manual/chat/staying-on-that-machine.md | 5 +- internal/manual/chat/use-skill.md | 4 +- internal/manual/chat/what-i-remember.md | 31 +- internal/session/artifact_append_law_test.go | 29 +- internal/session/factory_contract.go | 6 + internal/session/knowledge.go | 26 +- internal/session/knowledge_history.go | 78 ++- internal/session/knowledge_usage.go | 255 ++++++++ internal/session/loop.go | 22 +- internal/session/session.go | 8 + internal/session/sessionfile.go | 20 +- internal/session/stopwork.go | 1 + 59 files changed, 3615 insertions(+), 170 deletions(-) create mode 100644 cmd/codeaf/appfactory_budget_regression_test.go create mode 100644 cmd/codeaf/appfactory_memory_handoff_test.go create mode 100644 cmd/codeaf/appfactory_memory_law_test.go create mode 100644 cmd/codeaf/appfactory_memory_live_e2e_test.go create mode 100644 cmd/codeaf/appfactory_memory_owner_test.go create mode 100644 cmd/codeaf/appfactory_memory_spend_test.go create mode 100644 cmd/codeaf/appfactory_memory_tail_test.go create mode 100644 cmd/codeaf/appfactory_memory_worker_test.go create mode 100644 docs/changes/unreleased/2261-memory-budget-cause.md create mode 100644 docs/changes/unreleased/2261-memory-durable-stage-spend.md create mode 100644 docs/changes/unreleased/2261-memory-paused-admission.md create mode 100644 docs/changes/unreleased/2261-memory-quit-settlement.md create mode 100644 docs/changes/unreleased/2261-memory-spend-counted.md create mode 100644 internal/knowledge/accounting.go create mode 100644 internal/knowledge/compiler_batch_regression_test.go create mode 100644 internal/session/knowledge_usage.go diff --git a/cmd/codeaf-replay/replay_test.go b/cmd/codeaf-replay/replay_test.go index 0d43fe90c2..7ddd827919 100644 --- a/cmd/codeaf-replay/replay_test.go +++ b/cmd/codeaf-replay/replay_test.go @@ -243,6 +243,11 @@ func TestEveryTagTheBuildWritesResolvesToARoleSomebodyDeclared(t *testing.T) { // that passes forever the moment somebody builds a tag out of a variable, which // is exactly how the log came to carry 2,309 untagged rows in the first place. var derivedTags = map[string]func(yield func(tag, where string)){ + // internal/knowledge/compiler_provider.go tags each compile call with the + // lane role memory runs under; that is the whole of what it can be. + "string(lanes.RoleMemory)": func(yield func(tag, where string)) { + yield("memory", "internal/knowledge/compiler_provider.go, a memory compile call") + }, // internal/session/toolask.go: a tool's own call carries the tool's name // after the role word, and no walk can enumerate the belt. "toolCallTag(question.tool)": func(yield func(tag, where string)) { diff --git a/cmd/codeaf/app_factory.go b/cmd/codeaf/app_factory.go index 2f2627b1f9..a965e9cf8a 100644 --- a/cmd/codeaf/app_factory.go +++ b/cmd/codeaf/app_factory.go @@ -562,7 +562,13 @@ func (h *appFactoryHost) runWorker(ctx context.Context, target factoryTarget) { turns := factoryShapeTurnsFrom(h.floor, h.workspace, h.profile, current) turns.refusal = source.Refusal turns.liveOK = factoryLiveOK(raw, h.profile) - runner := buildFactoryRunner(mirror, h.floor, h.workspace, h.profile, h.stepMaker(mirror, source), turns) + maker := h.stepMaker(mirror, source) + if target.url == h.address { + // Memory admission reads Pause and Stop before their worker signal arrives. + // Receipts still book on the mirror and follow its ordinary settlement path. + maker.memoryOwner = h.floor + } + runner := buildFactoryRunner(mirror, h.floor, h.workspace, h.profile, maker, turns) dial := h.runnerDial(target) machine, _ := os.Hostname() kind := api.RunnerCI diff --git a/cmd/codeaf/appfactory_budget_regression_test.go b/cmd/codeaf/appfactory_budget_regression_test.go new file mode 100644 index 0000000000..fb5ff818d8 --- /dev/null +++ b/cmd/codeaf/appfactory_budget_regression_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" +) + +func TestStageOwnBudgetRejectsEarlyReport(t *testing.T) { + maker := &factoryrun.FakeMaker{Script: func(ctx context.Context, c *factoryrun.FakeConversation) { + _ = c.Spec.Stage.Report(ctx, factory.StageResult{Done: true, Output: "early report"}) + c.Fail(factoryrun.SpendStop{Job: true, Said: "job budget reached"}) + }} + result, err := factoryrun.NewChatExecutor(maker).Run(context.Background(), factoryrun.Job{Stage: factory.Stage{Name: "write", Kind: factory.StageChat, On: true, Until: "done"}, Round: 1}) + if err != nil { + t.Fatal(err) + } + if result.Done || !result.JobBudgetStopped { + t.Fatalf("own budget accepted early completion: %+v", result) + } +} + +func TestStageOwnSpendAsksAtCompletion(t *testing.T) { + floor, _ := chatSpendFloor(t) + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, State: factory.StateNew, Stages: []factory.Stage{{Name: "write", Kind: factory.StageChat, On: true, Until: "done"}}}) + if err != nil { + t.Fatal(err) + } + runner := factoryrun.New(factoryrun.Options{Store: floor, Exec: map[factory.StageKind]factoryrun.Executor{factory.StageChat: factoryrun.ExecutorFunc(func(context.Context, factoryrun.Job) (factory.StageResult, error) { + return factory.StageResult{Done: true, Output: "completed work", Spent: 1.5}, nil + })}}) + t.Cleanup(func() { _ = runner.Stop(item.ID) }) + if err := runner.Launch(item.ID); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(3 * time.Second) + for { + got, err := floor.Get(item.ID) + if err != nil { + t.Fatal(err) + } + if got.QKind == "cap" { + return + } + if got.State == factory.StateLanded || time.Now().After(deadline) { + t.Fatalf("own spend did not ask at cap: state=%s question=%q spend=%v", got.State, got.QKind, factory.JobSpent(got)) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/cmd/codeaf/appfactory_memory_handoff_test.go b/cmd/codeaf/appfactory_memory_handoff_test.go new file mode 100644 index 0000000000..2a7c25766d --- /dev/null +++ b/cmd/codeaf/appfactory_memory_handoff_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestMemoryLaterStageDrainsSkippedEvidence(t *testing.T) { + s := newLocalSetup(t) + s.start() + floor := s.backend.local.host.floor + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + entered, release, cancelled := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var extraction atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + fmt.Fprint(w, `{"data":[]}`) + return + } + var body struct { + Messages []struct{ Content string } `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return + } + result := "Cedar queues jobs." + var payload struct { + Messages []knowledge.Source `json:"messages"` + } + if len(body.Messages) > 1 { + _ = json.Unmarshal([]byte(body.Messages[1].Content), &payload) + } + if len(payload.Messages) > 0 { + if extraction.Add(1) == 1 { + close(entered) + select { + case <-release: + case <-r.Context().Done(): + close(cancelled) + return + } + } + claims := []any{} + for _, source := range payload.Messages { + claims = append(claims, map[string]any{"local_id": source.ID, "text": source.Text, "topic": "Queues", "scope": "Cedar", "kind": "fact", "evidence": []any{map[string]any{"source": source.ID, "quote": source.Text}}}) + } + raw, _ := json.Marshal(map[string]any{"claims": claims}) + result = string(raw) + } else if len(body.Messages) > 1 && strings.HasPrefix(body.Messages[1].Content, "{") { + result = `{"subjects":[]}` + } + raw, _ := json.Marshal(map[string]any{"model": knowledge.Model, "choices": []any{map[string]any{"delta": map[string]any{"content": result}, "finish_reason": "stop"}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + })) + t.Cleanup(server.Close) + original := http.DefaultClient + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + clone := r.Clone(r.Context()) + u := *r.URL + clone.URL = &u + if strings.Contains(u.Host, "openrouter.ai") { + clone.URL.Scheme = "http" + clone.URL.Host = server.Listener.Addr().String() + } + return http.DefaultTransport.RoundTrip(clone) + })} + t.Cleanup(func() { http.DefaultClient = original }) + jobCtx, endJob := context.WithCancel(context.Background()) + t.Cleanup(endJob) + root, profile := t.TempDir(), filepath.Join(s.home, "handoff-profile") + meter := &stageBudget{floor: floor, item: item} + agents := []*session.Agent{} + for n := 0; n < 3; n++ { + place := session.Place{Dir: filepath.Join(root, "bucket", session.NewSessionID())} + if err := os.MkdirAll(place.Dir, 0700); err != nil { + t.Fatal(err) + } + if err := markFactoryChat(floor, item, place.Transcript()); err != nil { + t.Fatal(err) + } + agent, err := session.New(session.Config{Workspace: root, Place: place, SessionFile: place.Transcript(), ProfileDir: profile, Model: knowledge.Model, APIKey: "fixture", BaseURL: server.URL, JobBudget: meter, JobContext: jobCtx, Unattended: true, DeterministicNames: true}) + if err != nil { + t.Fatal(err) + } + agents = append(agents, agent) + t.Cleanup(func() { _ = agent.Close() }) + } + vault := agents[0].Knowledge() + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + submit := func(agent *session.Agent, text string) { + t.Helper() + events, err := agent.Submit(context.Background(), text) + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + for range events { + } + close(done) + }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("stage did not end within memory cap") + } + } + submit(agents[0], "Cedar queues jobs.") + select { + case <-entered: + default: + t.Fatal("first stage did not start memory") + } + if err := agents[0].Close(); err != nil { + t.Fatal(err) + } + select { + case <-cancelled: + t.Fatal("closing a finished stage cancelled job-owned compilation") + default: + } + submit(agents[1], "Birch retains skipped evidence.") + if err := agents[1].Close(); err != nil { + t.Fatal(err) + } + if extraction.Load() != 1 { + t.Fatal("later stage stacked a compiler") + } + close(release) + deadline := time.Now().Add(3 * time.Second) + for { + snapshot, err := vault.Snapshot() + if err != nil { + t.Fatal(err) + } + if snapshot.Pending == 0 && !snapshot.NeedsUpdate { + break + } + if time.Now().After(deadline) { + t.Fatalf("first compile did not finish: %+v", snapshot) + } + time.Sleep(10 * time.Millisecond) + } + // Publication finishes only after its provider charges have been banked. + want := agents[0].Usage().CostUSD + reopened, err := session.New(session.Config{Workspace: root, SessionFile: agents[0].SessionPath(), ProfileDir: profile, Model: knowledge.Model, APIKey: "fixture", BaseURL: server.URL, JobBudget: meter, JobContext: jobCtx, Unattended: true, DeterministicNames: true}) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if got := reopened.Usage().CostUSD; want <= .01 || math.Abs(got-want) > .000001 { + t.Fatalf("reopened stage lost paid memory: USD=%.4f want %.4f", got, want) + } + submit(agents[2], "Cedar queues jobs.") + hits, err := vault.Search("Birch", 5) + if err != nil || len(hits) == 0 { + t.Fatalf("later finished turn lost skipped stage's searchable evidence: hits=%v error=%v", hits, err) + } + page, err := vault.Read(hits[0].Page.ID) + if err != nil || len(page.Evidence) == 0 { + t.Fatalf("skipped stage citation missing: %+v %v", page, err) + } + endJob() +} diff --git a/cmd/codeaf/appfactory_memory_law_test.go b/cmd/codeaf/appfactory_memory_law_test.go new file mode 100644 index 0000000000..34e9aa1df8 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_law_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +// Every completion in the knowledge package must be the compiler's one billed +// provider call. Adding another client or removing its billing arm fails here. +func memoryFunnelViolations(root string) ([]string, error) { + files, err := filepath.Glob(filepath.Join(root, "internal", "knowledge", "*.go")) + if err != nil { + return nil, err + } + var violations []string + calls := 0 + for _, path := range files { + if strings.HasSuffix(path, "_test.go") { + continue + } + source, err := os.ReadFile(path) + if err != nil { + return nil, err + } + f, err := parser.ParseFile(token.NewFileSet(), path, source, 0) + if err != nil { + return nil, err + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + var armed token.Pos + ast.Inspect(fn.Body, func(n ast.Node) bool { + if assignment, ok := n.(*ast.AssignStmt); ok && len(assignment.Lhs) == 1 && len(assignment.Rhs) == 1 { + lhs, lhsOK := assignment.Lhs[0].(*ast.Ident) + call, callOK := assignment.Rhs[0].(*ast.CallExpr) + if lhsOK && lhs.Name == "ctx" && callOK { + if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "memoryBilling" && len(call.Args) == 1 { + if arg, ok := call.Args[0].(*ast.Ident); ok && arg.Name == "ctx" { + armed = assignment.Pos() + } + } + } + } + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + if strings.HasPrefix(selector.Sel.Name, "Complete") { + calls++ + contextOK := false + if len(call.Args) > 0 { + if arg, ok := call.Args[0].(*ast.Ident); ok { + contextOK = arg.Name == "ctx" + } + } + if !contextOK { + violations = append(violations, path+": completion discards the billed context") + } + if filepath.Base(path) != "compiler_provider.go" || fn.Name.Name != "call" || armed == 0 || armed > call.Pos() { + violations = append(violations, path+": completion bypasses the memory billing funnel") + } + } + return true + }) + } + } + if calls != 1 { + violations = append(violations, "knowledge must have exactly one billed completion door") + } + return violations, nil +} + +func checkMemoryFunnel(t *testing.T) { + t.Helper() + root := filepath.Join("..", "..") + violations, err := memoryFunnelViolations(root) + if err != nil { + t.Fatal(err) + } + if len(violations) > 0 { + t.Fatal(strings.Join(violations, "\n")) + } + // The application's ledger binding is part of the law, not just a hook that + // a future caller can leave unconnected. + source, err := os.ReadFile(filepath.Join(root, "internal", "session", "knowledge_usage.go")) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"knowledge.SetUsageRecorder(", "RecordUsage(", "a.bank(", "ledger: true"} { + if !bytes.Contains(source, []byte(required)) { + t.Fatalf("memory ledger binding lost %s", required) + } + } + // Plant a real bypass in a copy: the law must reject its own failure mode. + copyRoot := t.TempDir() + dir := filepath.Join(copyRoot, "internal", "knowledge") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(filepath.Join(root, "internal", "knowledge", "compiler_provider.go")) + if err != nil { + t.Fatal(err) + } + planted := bytes.Replace(original, []byte("ctx = memoryBilling(ctx)"), []byte("memoryBilling(ctx)"), 1) + if bytes.Equal(original, planted) { + t.Fatal("billing mutation did not land") + } + if err := os.WriteFile(filepath.Join(dir, "compiler_provider.go"), planted, 0600); err != nil { + t.Fatal(err) + } + violations, err = memoryFunnelViolations(copyRoot) + if err != nil || len(violations) == 0 { + t.Fatalf("law accepted an unbilled compiler: %v %v", violations, err) + } +} diff --git a/cmd/codeaf/appfactory_memory_live_e2e_test.go b/cmd/codeaf/appfactory_memory_live_e2e_test.go new file mode 100644 index 0000000000..5d442a84d8 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_live_e2e_test.go @@ -0,0 +1,591 @@ +//go:build e2e + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/calllog" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/provider" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestLiveMemoryStageMeasure(t *testing.T) { + liveMemoryStageMeasure(t, false) +} + +func TestLiveMemoryPauseMeasure(t *testing.T) { + liveMemoryStageMeasure(t, true) +} + +func liveMemoryStageMeasure(t *testing.T, pause bool) { + if os.Getenv(liveFactoryOptIn) != "1" { + t.Skipf("opt-in live lane: set %s=1 (spends a few cents on OpenRouter)", liveFactoryOptIn) + } + if liveFactoryKey == "" { + t.Fatal("BLOCKED: no OpenRouter key by any road the product reads; not faking a pass") + } + + root := resolvedTempDir(t) + appHome := filepath.Join(root, "state") + userHome := filepath.Join(root, "home") + callPath := filepath.Join(t.TempDir(), "logs", "calls.jsonl") + t.Setenv(calllog.EnvVar, callPath) + t.Setenv(calllog.BodiesEnvVar, "") + t.Setenv("HOME", userHome) + t.Setenv(home.EnvVar, appHome) + profile := v3Profile(t, map[string]any{ + config.KeyTaskModelsMode: "one", config.KeyTaskModelsOne: liveFactoryModel, + "models.roles": "", "models.tiers.low": liveFactoryModel, "models.tiers.high": liveFactoryModel, + "models.tiers.reflex": liveFactoryModel, "models.tiers.mastermind": liveFactoryModel, + "models.tiers.worker": liveFactoryModel, "task.model": liveFactoryModel, "models.fallbacks": "", + config.KeyKnowledgeModel: liveFactoryModel, + }) + t.Setenv(config.ProfileDirEnv, profile) + t.Setenv("CODEAF_PROFILE_DIR", profile) + // NO KEY IN THE ENVIRONMENT: the only road to one is the app's memory. + t.Setenv("OPENROUTER_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "") + t.Setenv("CODEAF_MODEL_POOL", "off") + t.Setenv("GH_TOKEN", "") + t.Setenv("GITHUB_TOKEN", "") + t.Setenv(factoryPostureEnv, "") + keys := config.NewAppCredentials() + t.Cleanup(config.UseAppCredentials(keys)) + keys.Set(modelsource.DefaultID, liveFactoryKey) + + recorder := newLiveRecorder(t, config.DefaultBaseURL) + recorder.budget, recorder.maxCalls = 1, 250 + memoryReady := make(chan struct{}) + memoryResponse := sync.OnceFunc(func() { close(memoryReady) }) + var receiptMu sync.Mutex + receipts := map[string]float64{} + originalClient := http.DefaultClient + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + copy := r.Clone(r.Context()) + u := *r.URL + copy.URL = &u + if r.Method == http.MethodGet && u.Host == recorder.server.Listener.Addr().String() && strings.HasSuffix(u.Path, "/generation") { + target, err := url.Parse(config.DefaultBaseURL) + if err != nil { + return nil, err + } + target.Path = strings.TrimRight(target.Path, "/") + "/generation" + target.RawQuery = u.RawQuery + copy.URL = target + copy.Host = target.Host + } + if r.Method == http.MethodPost && strings.Contains(u.Host, "openrouter.ai") { + copy.URL.Scheme = "http" + copy.URL.Host = recorder.server.Listener.Addr().String() + } + response, err := http.DefaultTransport.RoundTrip(copy) + if err == nil && r.Method == http.MethodGet && strings.HasSuffix(u.Path, "/generation") { + raw, readErr := io.ReadAll(response.Body) + response.Body.Close() + response.Body = io.NopCloser(bytes.NewReader(raw)) + var receipt struct { + Data struct { + Cost *float64 `json:"total_cost"` + } `json:"data"` + } + if readErr == nil && json.Unmarshal(raw, &receipt) == nil && receipt.Data.Cost != nil { + receiptMu.Lock() + receipts[u.Query().Get("id")] = *receipt.Data.Cost + receiptMu.Unlock() + } + } + if err == nil && r.Method == http.MethodPost && r.GetBody != nil { + body, bodyErr := r.GetBody() + if bodyErr == nil { + var request struct { + Messages []struct{ Content string } `json:"messages"` + } + _ = json.NewDecoder(body).Decode(&request) + body.Close() + if len(request.Messages) > 1 { + var payload map[string]json.RawMessage + if json.Unmarshal([]byte(request.Messages[1].Content), &payload) == nil && payload["messages"] != nil { + response.Body = &memoryGenerationBody{ReadCloser: response.Body, ready: memoryResponse} + } + } + } + } + return response, err + })} + t.Cleanup(func() { http.DefaultClient = originalClient }) + t.Setenv("CODEAF_BASE_URL", recorder.server.URL) + + g := newRunRig(t) + proc, err := openV3Process("chat") + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.closeAll) + launch, err := openV3Launch(proc, v3Options{Model: liveFactoryModel, OneModel: true, Workspace: g.workspace}) + if err != nil { + t.Fatal(err) + } + assertFactoryPin(t, launch.Config, liveFactoryModel) + if err := knowledge.Open(profile).StartBuild(); err != nil { + t.Fatal(err) + } + + stageSettle = 300 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + + w := newFactoryWire(t, testFactoryHub(t)) + w.backend.open = appOpenInProcess + gh := &refusingGitHub{} + local := &w.backend.local + local.githubHTTP = gh.client() + local.rules = func(context.Context, string, string) (pipeline.File, error) { return pipeline.File{}, os.ErrNotExist } + // The conversation config the runner opens its stages from: the one-model + // launch above, as a terminal's launch hands one to its worker. + local.parent = func() session.Config { return launch.Config } + // WHOSE JOBS THEY ARE comes from the factory's GitHub link; none is made. + local.seamHook = func(seam factory.Seam) factory.Seam { + seam.GitHub = func(context.Context) (factory.GitHubLink, error) { return factory.GitHubLink{Login: "owner"}, nil } + return seam + } + + var st appwire.FactoryLocalStatus + // factory.local_start turns the runner on itself (ensureLocalRunner); a + // job still runs only once a person presses Start. + w.must("factory.local_start", map[string]any{"caller": "live"}, &st) + if !st.Runner { + t.Fatalf("start left the runner off: %+v", st) + } + floor := local.host.floor + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + + const want = "hello factory" + id, err := floor.Add(context.Background(), factory.Item{ + Title: "write the greeting", Repo: "acme/api", Tier: factory.TierOwner, Cap: 1, Model: liveFactoryModel, + Stages: []factory.Stage{ + {Name: "plan", Kind: factory.StageChat, On: true, Until: "done", Ask: "Plan one change: write greet/HELLO.txt containing hello factory and a newline. Do not delegate. Call stage_result with done true and output describing this single change. Do nothing else."}, + {Name: "write", Kind: factory.StageChat, On: true, Until: "done", + Ask: "Create the file greet/HELLO.txt (under the current directory) with the write tool, its content exactly `" + want + "` and a trailing newline. Then call stage_result with done true, findings 0 and a one-line output. Do nothing else."}, + {Name: "test", Kind: factory.StageCheck, On: true, Ask: "grep -qx '" + want + "' greet/HELLO.txt"}, + {Name: "review", Kind: factory.StageChat, On: true, Until: "clean", Ask: "Read greet/HELLO.txt. Check it contains exactly hello factory and a newline. Do not delegate or change files. Call stage_result with done true, findings 0 and output describing what you checked."}, + {Name: "proof", Kind: factory.StageChat, On: true, Until: "proven", Ask: "Read greet/HELLO.txt. Call stage_result with done true and a claims array containing one claim: text greeting is correct, ok true, evidence greet/HELLO.txt, medium file. Do not delegate or change files."}, + {Name: factory.ApproveName, Kind: factory.StageGate, On: true}, + }, + }) + if err != nil { + t.Fatal(err) + } + + stopJob := func() { + var current api.Job + w.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, ¤t) + var stopped appwire.FactoryJobActed + w.must("factory.job_stop", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": current.Version}}, &stopped) + } + + // BEFORE ANY START: the job waits in Backlog, and no model was called. + var job api.Job + w.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + if job.State != api.StateBacklog { + t.Fatalf("the job is %q before any start", job.State) + } + recorder.mu.Lock() + early := recorder.calls + recorder.mu.Unlock() + if early != 0 { + t.Fatalf("something ran before the explicit start: calls %d", early) + } + + // THE EXPLICIT RUN, THEN THE EXPLICIT START: the two doors a person uses. + var run appwire.FactoryLocalStatus + w.must("factory.local_run", map[string]any{"caller": "live"}, &run) + if !run.Runner { + t.Fatalf("run: %+v", run) + } + var acted appwire.FactoryJobActed + w.must("factory.job_start", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + var it factory.Item + var pausedAt time.Time + for { + it, _ = floor.Get(id) + ready := false + select { + case <-memoryReady: + ready = true + default: + } + if pause && ready && pausedAt.IsZero() && memoryCallsAfter(t, callPath, time.Time{}).active > 0 { + var current api.Job + w.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, ¤t) + w.must("factory.job_hold", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": current.Version}}, &appwire.FactoryJobActed{}) + pausedAt = time.Now() + memoryAwaitReceipts(t, callPath) + before := memoryCallsAfter(t, callPath, time.Time{}).started + time.Sleep(5 * time.Second) + after := memoryCallsAfter(t, callPath, time.Time{}).started + t.Logf("PAUSE memory_calls_before=%d after=%d new_calls_after_pause=%d hold_seconds=5", before, after, memoryCallsAfter(t, callPath, pausedAt).started) + if after != before || memoryCallsAfter(t, callPath, pausedAt).started != 0 { + t.Error("paused job admitted memory") + } + break + } + if it.State == factory.StateLanded || it.Failure != nil || it.Question != "" || it.State == factory.StateDismissed || ctx.Err() != nil { + break + } + recorder.mu.Lock() + over := recorder.cost >= 1 + recorder.mu.Unlock() + if over { + stopJob() + cancel() + t.Fatal("live run reached $1; no further calls authorized") + } + time.Sleep(10 * time.Millisecond) + } + + if ctx.Err() != nil { + stopJob() + } + if pause && pausedAt.IsZero() { + t.Error("job ended before a running memory call could be paused") + } + if !pause && it.State != factory.StateLanded { + t.Errorf("job state=%s question=%q failure=%+v", it.State, it.Question, it.Failure) + } + if gh.count() != 0 { + t.Errorf("GitHub requests: %d", gh.count()) + } + // Let the real app's final debounced brief enter the measured call log. + t.Logf("allowing %s for the final automatic brief", factory.BriefDebounce+time.Second) + time.Sleep(factory.BriefDebounce + time.Second) + memoryAwaitReceipts(t, callPath) + it, err = floor.Get(id) + if err != nil { + t.Fatal(err) + } + var header api.Job + w.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &header) + receiptMu.Lock() + lateUSD := 0.0 + for _, cost := range receipts { + lateUSD += cost + } + receiptMu.Unlock() + memoryMeasureTable(t, callPath, it, header.Cost, lateUSD) + recorder.mu.Lock() + defer recorder.mu.Unlock() + t.Logf("models requested=%v answered=%v provider_total=%.8f problems=%v", recorder.requests, recorder.replies, recorder.cost, recorder.problems) + if len(recorder.problems) > 0 || len(recorder.replies) != 1 || recorder.replies[liveFactoryModel] == 0 { + t.Error("live model policy failed") + } +} + +// Tables come from the shipped call log, ledger and stage records. They are +// printed only; no measurement artifact becomes part of the repository. +func memoryMeasureTable(t *testing.T, path string, it factory.Item, headerUSD, receiptUSD float64) { + t.Helper() + if !session.FlushUsage() { + t.Error("usage ledger did not flush") + } + ledger, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + var calls []calllog.Record + starts := map[string]time.Time{} + var ledgerUSD, memoryUSD float64 + providerUSD := receiptUSD + total, memory, afterEnd := 0, 0, 0 + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), 4<<20) + for scanner.Scan() { + var row calllog.Record + if err := json.Unmarshal(scanner.Bytes(), &row); err != nil { + t.Fatal(err) + } + if row.Model != "" && row.Model != liveFactoryModel { + t.Errorf("call log model=%s", row.Model) + } + at, err := time.Parse(time.RFC3339Nano, row.Time) + if err != nil { + t.Fatal(err) + } + if row.Phase == "start" { + if row.Tag == "memory" && it.Stream != nil && !it.Stream.Ended.IsZero() && at.After(it.Stream.Ended) { + afterEnd++ + } + starts[row.ID] = at + continue + } + calls = append(calls, row) + total++ + providerUSD += row.Cost + if row.Tag == "memory" { + memory++ + } + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + for _, row := range ledger { + if row.Model != "" && row.Model != liveFactoryModel { + t.Errorf("ledger model=%s", row.Model) + } + ledgerUSD += row.USD + if row.Role == "memory" { + memoryUSD += row.USD + } + } + t.Logf("memory calls started after job end=%d", afterEnd) + if afterEnd != 0 { + t.Error("memory started after the job ended") + } + t.Log("stage turn | memory calls | memory USD | step-end ms | memory-only ms") + var delays, memoryDelays []int64 + if it.Stream == nil { + t.Fatal("no stage stream recorded") + } + for _, phase := range it.Stream.Phases { + if phase.Kind == factory.StageGate || phase.Started.IsZero() { + continue + } + var header struct { + ID string `json:"id"` + } + if f, err := os.Open(phase.Chat); err == nil { + _ = json.NewDecoder(bufio.NewReader(f)).Decode(&header) + f.Close() + } + if phase.Kind == factory.StageChat && header.ID == "" { + t.Errorf("stage %s has no readable conversation identity", phase.Name) + } + count, usd := 0, 0.0 + for _, row := range ledger { + if row.Session == header.ID && row.Role == "memory" { + usd += row.USD + } + } + for _, row := range calls { + if row.Tag != "memory" { + continue + } + at, _ := time.Parse(time.RFC3339Nano, row.Time) + owned := row.Node == header.ID && header.ID != "" + if row.Node == "" { + owned = !at.Before(phase.Started) && !at.After(phase.At) + } + if owned { + count++ + } + } + stepEnd := phase.At + for _, next := range it.Stream.Phases { + if next.Started.After(phase.Started) && next.Started.Before(stepEnd) { + stepEnd = next.Started + } + } + workEnd := phase.Started + if phase.Kind == factory.StageCheck { + workEnd = stepEnd + } + for _, row := range calls { + at, _ := time.Parse(time.RFC3339Nano, row.Time) + if row.Tag == "turn" && at.After(workEnd) && !at.After(stepEnd) { + workEnd = at + } + } + delay := max(int64(0), stepEnd.Sub(workEnd).Milliseconds()) + // Parallel compiler requests overlap. Count their wall-time union in + // this stage's tail rather than adding their request durations. + intervals := [][2]time.Time{} + for _, row := range calls { + if row.Tag != "memory" || (row.Node != "" && row.Node != header.ID) { + continue + } + end, _ := time.Parse(time.RFC3339Nano, row.Time) + begin := starts[row.ID] + if begin.Before(workEnd) { + begin = workEnd + } + if end.After(stepEnd) { + end = stepEnd + } + if end.After(begin) { + intervals = append(intervals, [2]time.Time{begin, end}) + } + } + sort.Slice(intervals, func(i, j int) bool { return intervals[i][0].Before(intervals[j][0]) }) + var memoryMS int64 + var covered time.Time + for _, interval := range intervals { + begin, end := interval[0], interval[1] + if begin.Before(covered) { + begin = covered + } + if end.After(begin) { + memoryMS += end.Sub(begin).Milliseconds() + covered = end + } + } + delays = append(delays, delay) + memoryDelays = append(memoryDelays, memoryMS) + t.Logf("%s/%d | %d | %.8f | %d | %d", phase.Name, phase.Round, count, usd, delay, memoryMS) + } + sort.Slice(delays, func(i, j int) bool { return delays[i] < delays[j] }) + sort.Slice(memoryDelays, func(i, j int) bool { return memoryDelays[i] < memoryDelays[j] }) + if len(delays) > 0 { + t.Logf("memory-only median_ms=%d max_ms=%d", memoryDelays[len(memoryDelays)/2], memoryDelays[len(memoryDelays)-1]) + t.Logf("step-end median_ms=%d max_ms=%d", delays[len(delays)/2], delays[len(delays)-1]) + } + t.Logf("TOTAL calls=%d memory_calls=%d memory_USD=%.8f provider_USD=%.8f ledger_USD=%.8f job_header_USD=%.8f", total, memory, memoryUSD, providerUSD, ledgerUSD, headerUSD) + t.Logf("late_provider_receipts_USD=%.8f", receiptUSD) + if math.Abs(ledgerUSD-providerUSD) > 0.000001 { + t.Errorf("ledger lost provider-reported cost: %.8f vs %.8f", ledgerUSD, providerUSD) + } + if math.Abs(headerUSD-ledgerUSD) > 0.000001 { + t.Errorf("job header differs from ledger: %.8f vs %.8f", headerUSD, ledgerUSD) + } + if providerUSD >= 1 || ledgerUSD >= 1 || headerUSD >= 1 { + t.Error("live run reached $1") + } + if memory == 0 { + t.Error("no memory call measured") + } +} + +// Receipt recovery is measured after the job, outside every stage-tail interval. +// The live fixture does not turn the two-second user wait into a receipt wait. +func memoryAwaitReceipts(t *testing.T, path string) { + t.Helper() + deadline := time.Now().Add(provider.ReceiptWait) + for { + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + pending := map[string]bool{} + unpriced := false + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), 4<<20) + for scanner.Scan() { + var row calllog.Record + if err := json.Unmarshal(scanner.Bytes(), &row); err != nil { + file.Close() + t.Fatal(err) + } + if row.Phase == "start" { + pending[row.ID] = true + continue + } + delete(pending, row.ID) + if row.Cost == 0 && (row.Error != "" || row.Ended != "") { + unpriced = true + } + } + scanErr := scanner.Err() + file.Close() + if scanErr != nil { + t.Fatal(scanErr) + } + if len(pending) == 0 { + if unpriced { + t.Logf("waiting %s for interrupted-call receipts after the job ended", provider.ReceiptWait+time.Second) + time.Sleep(provider.ReceiptWait + time.Second) + } + return + } + if time.Now().After(deadline) { + t.Errorf("%d model calls still lack terminal records after job end", len(pending)) + return + } + time.Sleep(50 * time.Millisecond) + } +} + +type memoryCallWindow struct{ started, active int } + +func memoryCallsAfter(t *testing.T, path string, since time.Time) memoryCallWindow { + t.Helper() + file, err := os.Open(path) + if os.IsNotExist(err) { + return memoryCallWindow{} + } + if err != nil { + t.Fatal(err) + } + defer file.Close() + count := memoryCallWindow{} + pending := map[string]bool{} + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), 4<<20) + for scanner.Scan() { + var row calllog.Record + if json.Unmarshal(scanner.Bytes(), &row) != nil { + continue + } + at, _ := time.Parse(time.RFC3339Nano, row.Time) + if row.Phase == "start" && row.Tag == "memory" && at.After(since) { + count.started++ + pending[row.ID] = true + } + if row.Phase != "start" { + delete(pending, row.ID) + } + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + count.active = len(pending) + return count +} + +// Wait for a generation identity so cancellation exercises paid receipt recovery. +type memoryGenerationBody struct { + io.ReadCloser + seen []byte + ready func() +} + +func (b *memoryGenerationBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if b.ready != nil { + b.seen = append(b.seen, p[:n]...) + if bytes.Contains(b.seen, []byte("gen-")) { + b.ready() + b.ready, b.seen = nil, nil + } + } + return n, err +} diff --git a/cmd/codeaf/appfactory_memory_owner_test.go b/cmd/codeaf/appfactory_memory_owner_test.go new file mode 100644 index 0000000000..25fcd110a6 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_owner_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "errors" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "math" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestMemoryRecognizesOldJobOwnership(t *testing.T) { + places, profile, conversation := bringBackFixture(t) + floor, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + old := conversation("old-stage") + conversation("ordinary") + if err := markFactoryChat(floor, factory.Item{ID: 41, Repo: "acme/api"}, old); err != nil { + t.Fatal(err) + } + journals, err := session.KnowledgeHistory(places, profile).Inventory(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, journal := range journals { + if journal.ID == old { + if !journal.Job { + t.Fatal("old job transcript with factory-owner.json was admitted as ordinary chat") + } + return + } + } + t.Fatal("old job transcript missing from inventory") +} + +func TestMemoryChargeCannotUndoSuccessfulStage(t *testing.T) { + floor, _ := chatSpendFloor(t) + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, ChatSpent: .99}) + if err != nil { + t.Fatal(err) + } + meter := &stageBudget{floor: floor, item: item} + maker := &factoryrun.FakeMaker{Script: func(ctx context.Context, c *factoryrun.FakeConversation) { + if err := c.Spec.Stage.Report(ctx, factory.StageResult{Done: true, Output: "completed work"}); err != nil { + t.Error(err) + } + // The successful work is already reported. Its deferred memory receipt + // arrives before Wait returns, taking this job from $0.99 to $1.01. + meter.SpendMemory(.02) + c.Fail(factoryrun.SpendStop{Job: true, Memory: true, Said: meter.Refusal()}) + }} + stage := factory.Stage{Name: "work", Kind: factory.StageChat, On: true, Until: "done"} + result, err := factoryrun.NewChatExecutor(maker).Run(context.Background(), factoryrun.Job{Item: item, Stage: stage, Round: 1}) + if err != nil { + t.Fatal(err) + } + if !result.Done || result.JobBudgetStopped || result.Output != "completed work" { + t.Fatalf("memory charge undid successful stage: %+v", result) + } + charged, err := floor.Get(item.ID) + if err != nil { + t.Fatal(err) + } + if got := factory.JobSpent(charged); math.Abs(got-1.01) > .000001 { + t.Fatalf("memory charge missing: %.4f", got) + } + next := &stageBudget{floor: floor, item: charged} + if next.Refusal() == "" { + t.Fatal("next stage ignored memory spending") + } +} + +func TestMemoryCapKeepsFinishedPhase(t *testing.T) { + for _, last := range []bool{false, true} { + t.Run(map[bool]string{false: "next admission", true: "last step"}[last], func(t *testing.T) { + floor, _ := chatSpendFloor(t) + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, ChatSpent: .99, State: factory.StateNew, + Stages: []factory.Stage{{Name: "work", Kind: factory.StageChat, On: true, Until: "done"}, {Name: "review", Kind: factory.StageChat, On: true, Until: "done"}}}) + if err != nil { + t.Fatal(err) + } + if last { + if err := floor.Annotate(item.ID, func(it *factory.Item) error { it.Stages = it.Stages[:1]; return nil }); err != nil { + t.Fatal(err) + } + } + var rounds atomic.Int64 + runner := factoryrun.New(factoryrun.Options{Store: floor, Exec: map[factory.StageKind]factoryrun.Executor{ + factory.StageChat: factoryrun.ExecutorFunc(func(ctx context.Context, job factoryrun.Job) (factory.StageResult, error) { + rounds.Add(1) + meter := &stageBudget{floor: floor, item: job.Item} + meter.SpendMemory(.02) + return factory.StageResult{Done: true, Output: "completed work", Spent: .02, SpendRecorded: true}, nil + }), + }}) + t.Cleanup(func() { _ = runner.Stop(item.ID) }) + if err := runner.Launch(item.ID); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(3 * time.Second) + for { + got, err := floor.Get(item.ID) + if err != nil { + t.Fatal(err) + } + if last && got.State == factory.StateLanded { + if got.QKind != "" || got.Stream.Phases[0].State != factory.PhaseDone || math.Abs(factory.JobSpent(got)-1.01) > .000001 { + t.Fatalf("last step lost completion or true spend: %+v", got) + } + return + } + if got.QKind == "cap" { + if last { + t.Fatal("last step asked for unused budget") + } + if got.Stream == nil || got.Stream.Phases[0].State != factory.PhaseDone || got.Stream.Phases[0].Result == nil || !got.Stream.Phases[0].Result.Done { + t.Fatalf("memory cap parked completed work instead of next-stage admission: %+v", got.Stream) + } + if got.Stream.Phases[1].State != factory.PhaseWaiting || rounds.Load() != 1 { + t.Fatalf("cap retried successful work: cur=%d rounds=%d", got.Stream.Cur, rounds.Load()) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("next stage did not observe memory charge: %+v", got) + } + time.Sleep(10 * time.Millisecond) + } + }) + } +} + +// A daily refusal is not a memory charge: memory crossing the job cap at the +// same time must not let an early report hide that the day's limit stopped it. +func TestMemoryCapDoesNotHideADailyRefusal(t *testing.T) { + floor, _ := chatSpendFloor(t) + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, ChatSpent: .99}) + if err != nil { + t.Fatal(err) + } + meter := &stageBudget{floor: floor, item: item} + meter.SpendMemory(.02) + done := make(chan struct{}) + close(done) + daily := factoryrun.SpendStop{Daily: true, Said: "daily limit reached"} + conv := &stageConversation{budget: meter, done: done, failure: daily} + var stop factoryrun.SpendStop + if err := conv.Wait(context.Background()); !errors.As(err, &stop) { + t.Fatalf("Wait = %v, want a spend stop", err) + } + if stop.Memory { + t.Fatalf("a daily refusal reads as memory: %+v", stop) + } +} + +// With no other refusal, memory crossing the cap still reads as memory. +func TestMemoryCapReadsAsMemoryWithoutADailyRefusal(t *testing.T) { + floor, _ := chatSpendFloor(t) + item, err := floor.Create(factory.Item{Repo: "acme/api", Cap: 1, ChatSpent: .99}) + if err != nil { + t.Fatal(err) + } + meter := &stageBudget{floor: floor, item: item} + meter.SpendMemory(.02) + done := make(chan struct{}) + close(done) + conv := &stageConversation{budget: meter, done: done} + var stop factoryrun.SpendStop + if err := conv.Wait(context.Background()); !errors.As(err, &stop) || !stop.Memory { + t.Fatalf("Wait = %v (%+v), want a memory stop", err, stop) + } +} diff --git a/cmd/codeaf/appfactory_memory_spend_test.go b/cmd/codeaf/appfactory_memory_spend_test.go new file mode 100644 index 0000000000..3cb3437151 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_spend_test.go @@ -0,0 +1,580 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/session" +) + +// The provider's own counter is independent of the knowledge receipts and the +// Spending ledger. Both compilation and reading use their exported interfaces. +func TestMemorySpendAndLifecycle(t *testing.T) { + s := newLocalSetup(t) + t.Run("funnel law", checkMemoryFunnel) + var calls atomic.Int64 + var blockNext atomic.Bool + var failUpdates atomic.Bool + var richModel atomic.Bool + entered := make(chan struct{}) + model := knowledge.Model + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Stream bool `json:"stream"` + ResponseFormat json.RawMessage `json:"response_format"` + Messages []struct { + Content string `json:"content"` + } `json:"messages"` + } + if r.Method != http.MethodPost { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[]}`) + return + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + calls.Add(1) + result := `{"subjects":[]}` + var payload map[string]json.RawMessage + if len(body.Messages) > 1 { + _ = json.Unmarshal([]byte(body.Messages[1].Content), &payload) + } + if payload["messages"] != nil && blockNext.CompareAndSwap(true, false) { + close(entered) + <-r.Context().Done() + return + } + if payload["messages"] != nil { + var sources []knowledge.Source + if err := json.Unmarshal(payload["messages"], &sources); err != nil || len(sources) == 0 { + t.Errorf("sources: %s", payload["messages"]) + return + } + content, _ := json.Marshal(map[string]any{"claims": []any{map[string]any{"local_id": "c", "text": "Cedar queues jobs.", "topic": "Queues", "scope": "Cedar", "kind": "fact", "evidence": []any{map[string]any{"source": sources[0].ID, "quote": "Cedar queues jobs."}}}}}) + result = string(content) + if failUpdates.Load() { + result = "{broken" + } + } + if richModel.Load() && len(body.Messages) > 1 { + var payload map[string]any + if err := json.Unmarshal([]byte(body.Messages[1].Content), &payload); err != nil { + t.Error(err) + return + } + content, err := json.Marshal(memoryRichReply(payload)) + if err != nil { + t.Error(err) + return + } + result = string(content) + } + if len(body.ResponseFormat) == 0 { + result = "Cedar queues jobs." + } + if body.Stream { + w.Header().Set("Content-Type", "text/event-stream") + chunk, _ := json.Marshal(map[string]any{"model": model, "choices": []any{map[string]any{"delta": map[string]any{"content": result}, "finish_reason": nil}}}) + fmt.Fprintf(w, "data: %s\n\n", chunk) + fmt.Fprintf(w, "data: {\"model\":%q,\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4400,\"completion_tokens\":100,\"cost\":0.01}}\n\ndata: [DONE]\n\n", model) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"model": model, "choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"content": result}}}, "usage": map[string]any{"prompt_tokens": 4400, "completion_tokens": 100, "cost": 0.01}}) + })) + defer server.Close() + original := http.DefaultTransport + originalClient := http.DefaultClient + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + if !strings.Contains(r.URL.Host, "openrouter.ai") { + return original.RoundTrip(r) + } + copy := r.Clone(r.Context()) + u := *r.URL + copy.URL = &u + copy.URL.Scheme = "http" + copy.URL.Host = server.Listener.Addr().String() + return original.RoundTrip(copy) + })} + t.Cleanup(func() { http.DefaultClient = originalClient }) + vault := knowledge.Open(filepath.Join(s.home, "profile")) + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + if err := vault.Capture(knowledge.Source{ID: "source", Session: "job-chat", Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: "Cedar queues jobs."}); err != nil { + t.Fatal(err) + } + if err := vault.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + hits, err := vault.Search("queues", 5) + if err != nil || len(hits) != 1 { + t.Fatalf("memory output lost: %v %v", hits, err) + } + before := calls.Load() + if err := vault.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + if calls.Load() != before { + t.Errorf("unchanged input: calls %d -> %d", before, calls.Load()) + } + if !session.FlushUsage() { + t.Fatal("ledger did not flush") + } + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + var ledgerCalls int + var usd float64 + for _, line := range lines { + if line.Role == "memory" { + ledgerCalls += line.Calls + usd += line.USD + } + } + t.Logf("provider: calls=%d USD=%.4f; memory ledger: calls=%d USD=%.4f", calls.Load(), float64(calls.Load())*.01, ledgerCalls, usd) + if int64(ledgerCalls) != calls.Load() || fmt.Sprintf("%.4f", usd) != fmt.Sprintf("%.4f", float64(calls.Load())*.01) { + t.Errorf("memory spend missing: provider calls=%d USD=%.4f; ledger calls=%d USD=%.4f", calls.Load(), float64(calls.Load())*.01, ledgerCalls, usd) + } + + t.Run("owned jobs", func(t *testing.T) { + st := s.start() + floor := s.backend.local.host.floor + for n := 0; n < 3; n++ { + item, err := floor.Create(factory.Item{Repo: "acme/api", Title: "README memory", State: factory.StateNew, Cap: 5}) + if err != nil { + t.Fatal(err) + } + meter := &stageBudget{floor: floor, item: item} + workspace := t.TempDir() + if err := os.MkdirAll(filepath.Join(workspace, "bucket"), 0700); err != nil { + t.Fatal(err) + } + place := session.Place{Dir: filepath.Join(workspace, "bucket", session.NewSessionID())} + agent, err := session.New(session.Config{Workspace: workspace, Place: place, SessionFile: place.Transcript(), ProfileDir: filepath.Join(s.home, "job-profile"), Model: model, APIKey: "fixture", BaseURL: server.URL, JobBudget: meter, Unattended: true, DeterministicNames: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { agent.Close() }) + vault := agent.Knowledge() + if vault == nil { + t.Fatal("job has no knowledge vault") + } + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + start := calls.Load() + events, err := agent.Submit(context.Background(), "Cedar queues jobs.") + if err != nil { + t.Fatal(err) + } + for event := range events { + if event.Kind == session.EventError { + t.Fatalf("turn: %v", event.Err) + } + } + if !session.FlushUsage() { + t.Fatal("ledger flush") + } + made := calls.Load() - start + var job api.Job + s.must("factory.job", appwire.FactoryJobParams{Factory: st.Factory, Job: item.ID}, &job) + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + var ownedCalls, memoryCalls int + var ownedUSD float64 + for _, line := range lines { + if line.Workspace == workspace { + ownedCalls += line.Calls + ownedUSD += line.USD + if line.Role == "memory" { + memoryCalls += line.Calls + } + } + } + t.Logf("job %d: provider calls=%d USD=%.4f; ledger calls=%d USD=%.4f; job header USD=%.4f; memory calls=%d", n+1, made, float64(made)*.01, ownedCalls, ownedUSD, job.Cost, memoryCalls) + if made == 0 || int64(ownedCalls) != made || fmt.Sprintf("%.4f", ownedUSD) != fmt.Sprintf("%.4f", float64(made)*.01) || fmt.Sprintf("%.4f", job.Cost) != fmt.Sprintf("%.4f", ownedUSD) || memoryCalls == 0 { + t.Fatalf("job's provider, ledger and header disagree") + } + hits, err := vault.Search("queues", 5) + if err != nil || len(hits) == 0 { + t.Fatalf("job memory output lost: %v %v", hits, err) + } + if n == 0 { + t.Run("delegated reports", func(t *testing.T) { + place := session.Place{Dir: filepath.Dir(agent.SessionPath())} + if err := os.MkdirAll(place.NodeJournals(), 0700); err != nil { + t.Fatal(err) + } + report := `{"type":"message","role":"assistant","content":"Cedar queues jobs.","timestamp":` + fmt.Sprintf("%q", time.Now().UTC().Format(time.RFC3339Nano)) + `,"presentation":{"audience":"operational"}}` + "\n" + if err := os.WriteFile(filepath.Join(place.NodeJournals(), "1.jsonl"), []byte(report), 0600); err != nil { + t.Fatal(err) + } + before := calls.Load() + events, err := agent.Submit(context.Background(), "Cedar queues jobs.") + if err != nil { + t.Fatal(err) + } + for range events { + } + inventory, err := session.KnowledgeHistory(workspace).Inventory(context.Background()) + if err != nil { + t.Fatal(err) + } + var task knowledge.Journal + for _, journal := range inventory { + if strings.HasPrefix(journal.Session, "task:") { + task = journal + } + } + if task.ID == "" || !task.Job { + t.Fatalf("delegated job report missing ownership: %+v", inventory) + } + batch, err := session.KnowledgeHistory(workspace).Read(context.Background(), task, knowledge.JournalCursor{Scope: task.Project}) + if err != nil || len(batch.Sources) != 1 { + t.Fatalf("delegated report: %+v %v", batch, err) + } + // Re-capturing the same immutable source proves it was consumed by + // the owner's finished turn rather than left for idle polling. + batch.Sources[0].Job = true + if err := vault.Capture(batch.Sources[0]); err != nil { + t.Fatal(err) + } + snapshot, err := vault.Snapshot() + if err != nil || snapshot.Pending != 0 { + t.Fatalf("delegated report stayed pending after owner's turn: pending=%d error=%v", snapshot.Pending, err) + } + if calls.Load() <= before { + t.Fatal("owner did not compile delegated report") + } + }) + } + if n == 0 { + before := calls.Load() + for _, state := range []string{"paused", "published"} { + if err := floor.Annotate(item.ID, func(it *factory.Item) error { + if state == "paused" { + it.Stream = &factory.Stream{Paused: true} + } else { + it.State = factory.StateShipped + it.Stream = nil + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := vault.Flush(context.Background(), "fixture", true); err == nil { + t.Errorf("%s job admitted memory compilation", state) + } + } + if calls.Load() != before { + t.Error("memory spent for a paused or published job") + } + if err := floor.Annotate(item.ID, func(it *factory.Item) error { it.State = factory.StateNew; it.Stream = nil; return nil }); err != nil { + t.Fatal(err) + } + } + if n == 0 { + inventory, err := session.KnowledgeHistory(workspace).Inventory(context.Background()) + if err != nil || len(inventory) < 1 { + t.Fatalf("job inventory: %v %v", inventory, err) + } + var parent string + for _, journal := range inventory { + if !strings.HasPrefix(journal.Session, "task:") { + parent = journal.Session + } + } + for extra := 0; extra < 65; extra++ { + if err := vault.Capture(knowledge.Source{ID: fmt.Sprintf("large-%d", extra), Session: parent, Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: "Cedar queues jobs.", Job: true}); err != nil { + t.Fatal(err) + } + } + if err := vault.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + snapshot, err := vault.Snapshot() + if err != nil || snapshot.Pending != 0 { + t.Errorf("finished-turn compilation left captured sources unpublished: pending=%d error=%v", snapshot.Pending, err) + } + } + before := calls.Load() + if err := vault.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + if calls.Load() != before { + t.Fatal("unchanged job input spent again") + } + if n == 2 { + blockNext.Store(true) + interrupted, err := agent.Submit(context.Background(), "Cedar queues jobs.") + if err != nil { + t.Fatal(err) + } + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("memory request did not reach pause barrier") + } + if err := agent.StopWork(); err != nil { + t.Fatal(err) + } + drained := make(chan struct{}) + go func() { + for range interrupted { + } + close(drained) + }() + select { + case <-drained: + case <-time.After(5 * time.Second): + t.Fatal("Pause did not cancel the memory request") + } + before = calls.Load() + } + if err := agent.StopWork(); err != nil { + t.Fatal(err) + } + if err := vault.Capture(knowledge.Source{ID: fmt.Sprintf("paused-%d", n), Session: "paused", Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: "Cedar queues jobs.", Job: true}); err != nil { + t.Fatal(err) + } + if err := vault.Flush(context.Background(), "fixture", true); err == nil { + t.Fatal("paused job admitted compilation") + } + if err := agent.Close(); err != nil { + t.Fatal(err) + } + if err := vault.Flush(context.Background(), "fixture", true); err == nil { + t.Fatal("ended job admitted compilation") + } + if calls.Load() != before { + t.Fatal("memory spent during Pause or after the end") + } + if n == 2 { + history := &memoryObservedHistory{History: session.KnowledgeHistory(workspace, filepath.Join(s.home, "job-profile")), passes: make(chan struct{}, 4)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- knowledge.Open(filepath.Join(s.home, "job-profile")).Run(ctx, history, func() string { return "fixture" }) + }() + for pass := 0; pass < 2; pass++ { + select { + case <-history.passes: + case <-time.After(5 * time.Second): + cancel() + t.Fatal("background inventory did not run") + } + } + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("background worker did not stop") + } + if calls.Load() != before { + t.Fatal("process worker spent on a finished job") + } + } + + inventory, err := session.KnowledgeHistory(workspace).Inventory(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, journal := range inventory { + if strings.Contains(journal.ID, workspace) && !journal.Job { + t.Fatal("process worker admitted an owned journal") + } + } + if _, err := os.Stat(agent.SessionPath() + ".knowledge-job"); err != nil { + t.Fatal("job journal has no durable ownership marker") + } + } + }) + t.Run("captured job migration", func(t *testing.T) { + v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } + source := knowledge.Source{ID: "old", Session: "old-job", Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: "Cedar queues jobs."} + history := memoryFixtureHistory{source: source, journal: knowledge.Journal{ID: "old-journal", Session: source.Session, Size: 1}} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- v.Run(ctx, history, func() string { return "" }) }() + deadline := time.Now().Add(3 * time.Second) + for { + snapshot, err := v.Snapshot() + if err != nil { + cancel() + t.Fatal(err) + } + if snapshot.Build.Phase == knowledge.BuildFailed { + break + } + if time.Now().After(deadline) { + cancel() + t.Fatal("old journal cursor never imported") + } + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + history.journal.Job = true + observed := &memoryObservedHistory{History: history, passes: make(chan struct{}, 4)} + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + before := calls.Load() + go func() { done <- v.Run(ctx, observed, func() string { return "fixture" }) }() + for pass := 0; pass < 2; pass++ { + select { + case <-observed.passes: + case <-time.After(5 * time.Second): + cancel() + t.Fatal("migrated inventory did not run") + } + } + cancel() + <-done + if calls.Load() != before { + t.Errorf("already captured ended job still spent: %d provider calls", calls.Load()-before) + } + }) + t.Run("unchanged failed input", func(t *testing.T) { + v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } + source := knowledge.Source{ID: "failed", Session: "ordinary-chat", Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: "Cedar queues jobs."} + history := memoryFixtureHistory{source: source, journal: knowledge.Journal{ID: "failed-journal", Session: source.Session, Size: 1}} + failUpdates.Store(true) + defer failUpdates.Store(false) + before := calls.Load() + ctx, cancel := context.WithTimeout(context.Background(), 6500*time.Millisecond) + defer cancel() + _ = v.Run(ctx, history, func() string { return "fixture" }) + if got := calls.Load() - before; got != 1 { + t.Errorf("unchanged failed input spent %d model calls, want 1", got) + } + }) + t.Run("unchanged refinement", func(t *testing.T) { + richModel.Store(true) + defer richModel.Store(false) + v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } + for _, round := range []string{"baseline", "added"} { + for _, scope := range []string{"Cedar", "Maple", "Birch"} { + if err := v.Capture(knowledge.Source{ID: scope + round, Session: "chat", Role: "user", TS: time.Now().UTC().Format(time.RFC3339Nano), Text: scope + " " + round + " guidance."}); err != nil { + t.Fatal(err) + } + } + if err := v.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + } + before := calls.Load() + if err := v.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + if calls.Load() != before { + t.Errorf("unchanged sources launched %d further refinement calls", calls.Load()-before) + } + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } + if err := v.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + if calls.Load() == before { + t.Error("explicit Build did not advance retained refinement") + } + + }) +} + +// This model supplies three independent concepts so unfinished refinement cannot +// hide behind an empty subject list. The production compiler owns every stage. +func memoryRichReply(p map[string]any) map[string]any { + if messages, ok := p["messages"].([]any); ok { + claims := []any{} + for _, raw := range messages { + s := raw.(map[string]any) + scope := strings.Fields(s["text"].(string))[0] + claims = append(claims, map[string]any{"local_id": s["id"], "text": s["text"], "scope": scope, "topic": "Operations", "kind": "fact", "evidence": []any{map[string]any{"source": s["id"], "quote": s["text"]}}}) + } + return map[string]any{"claims": claims} + } + if catalog, ok := p["catalog"].([]any); ok { + subjects := []any{} + for _, raw := range catalog { + n := raw.(map[string]any) + subjects = append(subjects, map[string]any{"title": n["scope"].(string) + " guide", "scope": n["scope"], "notes": []any{n["id"]}}) + } + return map[string]any{"subjects": subjects} + } + if catalog, ok := p["consolidation_catalog"].([]any); ok { + groups := []any{} + for _, raw := range catalog { + d := raw.(map[string]any) + groups = append(groups, map[string]any{"members": []any{d["id"]}, "title": d["title"], "scope": d["scope"], "purpose": d["purpose"], "folder": d["folder"]}) + } + return map[string]any{"groups": groups} + } + if p["remaining_steps"] != nil { + refs := []any{} + for _, raw := range p["claims"].([]any) { + refs = append(refs, raw.(map[string]any)["id"]) + } + scope := p["subject"].(map[string]any)["scope"].(string) + return map[string]any{"action": "write", "sections": []any{map[string]any{"heading": "Current operations", "paragraphs": []any{map[string]any{"text": scope + " baseline and added guidance.", "claims": refs}}}}} + } + if p["paragraphs"] != nil { + return map[string]any{"checks": []any{map[string]any{"id": "P0", "supported": true, "useful": true}}} + } + return map[string]any{"sections": []any{}, "questions": []any{}, "conflicts": []any{}} +} + +type memoryFixtureHistory struct { + source knowledge.Source + journal knowledge.Journal +} + +func (h memoryFixtureHistory) Inventory(context.Context) ([]knowledge.Journal, error) { + return []knowledge.Journal{h.journal}, nil +} +func (h memoryFixtureHistory) Read(context.Context, knowledge.Journal, knowledge.JournalCursor) (knowledge.JournalBatch, error) { + return knowledge.JournalBatch{Sources: []knowledge.Source{h.source}, Cursor: knowledge.JournalCursor{Offset: 1}}, nil +} + +// Observe the real worker through its exported history interface, without a +// timer assumption about when its first or second inventory happens. +type memoryObservedHistory struct { + knowledge.History + passes chan struct{} +} + +func (h *memoryObservedHistory) Inventory(ctx context.Context) ([]knowledge.Journal, error) { + journals, err := h.History.Inventory(ctx) + select { + case h.passes <- struct{}{}: + default: + } + return journals, err +} diff --git a/cmd/codeaf/appfactory_memory_tail_test.go b/cmd/codeaf/appfactory_memory_tail_test.go new file mode 100644 index 0000000000..63c98fc4e5 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_tail_test.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestMemoryCompileTailLifecycle(t *testing.T) { + for _, end := range []string{"pause", "job end", "session close"} { + t.Run(end, func(t *testing.T) { + s := newLocalSetup(t) + s.start() + floor := s.backend.local.host.floor + item, err := floor.Create(factory.Item{Repo: "acme/api", Title: "Cedar memory", Cap: 1, State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + entered, cancelled := make(chan struct{}), make(chan struct{}) + var memoryCalls atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + fmt.Fprint(w, `{"data":{"total_cost":0.02,"tokens_prompt":20,"tokens_completion":5}}`) + return + } + var body struct { + Messages []struct{ Content string } `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return + } + isMemory := false + for _, m := range body.Messages { + var p map[string]json.RawMessage + if json.Unmarshal([]byte(m.Content), &p) == nil && p["messages"] != nil { + isMemory = true + } + } + w.Header().Set("Content-Type", "text/event-stream") + if isMemory { + n := memoryCalls.Add(1) + fmt.Fprintf(w, "data: {\"id\":\"gen-memory-%d\",\"model\":%q,\"choices\":[{\"delta\":{\"content\":\"{\\\"claims\\\":[]}\"}}]}\n\n", n, knowledge.Model) + w.(http.Flusher).Flush() + if n == 1 { + close(entered) + } + <-r.Context().Done() + if n == 1 { + close(cancelled) + } + return + } + fmt.Fprintf(w, "data: {\"model\":%q,\"choices\":[{\"delta\":{\"content\":\"Cedar queues jobs.\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":5,\"cost\":0.01}}\n\ndata: [DONE]\n\n", knowledge.Model) + })) + t.Cleanup(server.Close) + original := http.DefaultClient + transport := http.DefaultTransport + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + clone := r.Clone(r.Context()) + u := *r.URL + clone.URL = &u + if strings.Contains(u.Host, "openrouter.ai") { + clone.URL.Scheme = "http" + clone.URL.Host = server.Listener.Addr().String() + } + return transport.RoundTrip(clone) + })} + t.Cleanup(func() { http.DefaultClient = original }) + meter := &memoryDelayedJobBudget{stageBudget: &stageBudget{floor: floor, item: item}} + workspace := t.TempDir() + agent, err := session.New(session.Config{Workspace: workspace, SessionFile: filepath.Join(workspace, "chat.jsonl"), ProfileDir: filepath.Join(s.home, "memory-profile"), Model: knowledge.Model, APIKey: "fixture", BaseURL: server.URL, JobBudget: meter, Unattended: true, DeterministicNames: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = agent.Close() }) + vault := agent.Knowledge() + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + events, err := agent.Submit(context.Background(), "Cedar queues jobs.") + if err != nil { + t.Fatal(err) + } + drained := make(chan struct{}) + go func() { + for range events { + } + close(drained) + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("memory call did not start") + } + select { + case <-drained: + case <-time.After(3 * time.Second): + t.Fatal("step stream still blocked on memory after 3 seconds; want a 2 second cap") + } + select { + case <-cancelled: + t.Fatal("step-end cap cancelled compilation instead of letting it continue") + default: + } + // A later finished turn must not stack a second compiler behind this one. + next, err := agent.Submit(context.Background(), "Cedar queues jobs.") + if err != nil { + t.Fatal(err) + } + later := make(chan struct{}) + go func() { + for range next { + } + close(later) + }() + select { + case <-later: + case <-time.After(3 * time.Second): + t.Fatal("later turn stacked behind unfinished memory") + } + if got := memoryCalls.Load(); got != 1 { + t.Fatalf("stacked memory calls=%d", got) + } + switch end { + case "pause": + if err := agent.StopWork(); err != nil { + t.Fatal(err) + } + case "job end": + if err := floor.Annotate(item.ID, func(it *factory.Item) error { it.State = factory.StateShipped; return nil }); err != nil { + t.Fatal(err) + } + case "session close": + if err := agent.Close(); err != nil { + t.Fatal(err) + } + } + select { + case <-cancelled: + case <-time.After(3 * time.Second): + t.Fatalf("%s did not cancel in-flight memory", end) + } + deadline := time.Now().Add(5 * time.Second) + for { + session.FlushUsage() + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + usd, calls := 0.0, 0 + for _, line := range lines { + if line.Workspace == workspace && line.Role == "memory" { + usd += line.USD + calls += line.Calls + } + } + job, err := floor.Get(item.ID) + if err != nil { + t.Fatal(err) + } + spent := factory.JobSpent(job) + if calls == 1 && math.Abs(usd-.02) < .000001 && math.Abs(spent-.04) < .000001 && math.Abs(job.MemorySpent-.02) < .000001 { + break + } + if time.Now().After(deadline) { + t.Fatalf("late memory accounting incomplete: calls=%d ledger=%.4f job=%.4f memory=%.4f want .02/.04/.02", calls, usd, spent, job.MemorySpent) + } + time.Sleep(10 * time.Millisecond) + } + if err := vault.Flush(context.Background(), "fixture", true); err == nil { + t.Fatal("ended owner admitted memory") + } + if got := memoryCalls.Load(); got != 1 { + t.Fatalf("memory calls after %s=%d", end, got) + } + snapshot, err := vault.Snapshot() + if err != nil || snapshot.Pending == 0 { + t.Fatalf("cancelled compile lost its pending source: %+v %v", snapshot, err) + } + }) + } +} + +// The ledger can be observable before the job meter finishes its separate write. +// Delay that public budget callback so the fixture exercises this order reliably. +type memoryDelayedJobBudget struct { + *stageBudget +} + +func (b *memoryDelayedJobBudget) Spend(usd float64) { + if usd == .02 { + time.Sleep(100 * time.Millisecond) + } + b.stageBudget.Spend(usd) +} + +func (b *memoryDelayedJobBudget) SpendMemoryModel(usd float64, model string, calls, input, output int) { + if usd == .02 { + time.Sleep(100 * time.Millisecond) + } + b.stageBudget.SpendMemoryModel(usd, model, calls, input, output) +} diff --git a/cmd/codeaf/appfactory_memory_worker_test.go b/cmd/codeaf/appfactory_memory_worker_test.go new file mode 100644 index 0000000000..899d05a7a9 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_worker_test.go @@ -0,0 +1,429 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestMemoryPausedJobLifecycle(t *testing.T) { + for _, end := range []string{"pause tail", "pause boundary", "pause resume", "stop tail", "quit tail"} { + t.Run(end, func(t *testing.T) { memoryLateReceiptFactoryHeader(t, end, 0) }) + } +} + +func TestMemoryPauseStopsFinishedStageTail(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "pause tail", 0) +} + +func TestMemoryPauseRetainsPendingForResume(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "pause resume", 0) +} + +func TestMemoryPauseClosesTailAdmission(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "pause boundary", 0) +} + +func TestMemoryLateReceiptReachesFactoryHeaderAfterWorkerFinishes(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "finish", 0) +} + +func TestMemoryLateReceiptReachesFactoryHeaderAfterStop(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "stop", 0) +} + +func TestMemoryLateReceiptPreservesExistingJobSpend(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "stop", .30) +} + +func TestMemoryLateReceiptReachesFactoryHeaderAfterQuit(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "quit", 0) +} + +func TestMemoryLateReceiptReachesFactoryHeaderWhenQuitInterruptsSettlement(t *testing.T) { + memoryLateReceiptFactoryHeader(t, "settling quit", 0) +} + +func memoryLateReceiptFactoryHeader(t *testing.T, end string, existing float64) { + pauseResume := end == "pause resume" + pauseBoundary := end == "pause boundary" + pauseTail := end == "pause tail" || pauseBoundary || pauseResume + settlingQuit := end == "settling quit" + stop, quit := end == "stop" || end == "stop tail" || settlingQuit, end == "quit" || end == "quit tail" || settlingQuit + detachedTail := pauseTail || end == "stop tail" || end == "quit tail" + quitDone := make(chan struct{}) + s := newLocalSetup(t) + release := make(chan struct{}) + releaseReceipt := sync.OnceFunc(func() { close(release) }) + cancelled := make(chan struct{}, 1) + started := make(chan struct{}, 1) + var memoryCalls, providerUnits, stageTurns atomic.Int64 + chargeReceipt := sync.OnceFunc(func() { providerUnits.Add(2) }) + learning, finishTurn := make(chan struct{}), make(chan struct{}) + finishLearning := sync.OnceFunc(func() { close(finishTurn) }) + t.Cleanup(finishLearning) + finishMemory := make(chan struct{}) + finishPaidCall := sync.OnceFunc(func() { close(finishMemory) }) + t.Cleanup(finishPaidCall) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + select { + case <-release: + case <-r.Context().Done(): + return + } + w.Header().Set("Content-Type", "application/json") + chargeReceipt() + fmt.Fprint(w, `{"data":{"total_cost":0.02,"tokens_prompt":20,"tokens_completion":5}}`) + return + } + var body struct { + Messages []struct{ Content string } `json:"messages"` + Tools []struct { + Function struct{ Name string } `json:"function"` + } `json:"tools"` + } + if json.NewDecoder(r.Body).Decode(&body) != nil { + return + } + var payload map[string]json.RawMessage + if len(body.Messages) > 1 { + _ = json.Unmarshal([]byte(body.Messages[1].Content), &payload) + } + w.Header().Set("Content-Type", "text/event-stream") + if payload["messages"] != nil || ((pauseBoundary || pauseResume) && len(payload) > 0) { + n := memoryCalls.Add(1) + if pauseResume && n > 1 { + providerUnits.Add(1) + var p map[string]any + _ = json.Unmarshal([]byte(body.Messages[1].Content), &p) + result, _ := json.Marshal(memoryRichReply(p)) + raw, _ := json.Marshal(map[string]any{"model": knowledge.Model, "choices": []any{map[string]any{"delta": map[string]any{"content": string(result)}, "finish_reason": "stop"}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + return + } + select { + case started <- struct{}{}: + default: + } + fmt.Fprintf(w, "data: {\"id\":\"gen-late-memory\",\"model\":%q,\"choices\":[{\"delta\":{\"content\":\"\"}}]}\n\n", knowledge.Model) + w.(http.Flusher).Flush() + if pauseBoundary && n == 1 { + select { + case <-finishMemory: + chargeReceipt() + var p map[string]any + _ = json.Unmarshal([]byte(body.Messages[1].Content), &p) + result, _ := json.Marshal(memoryRichReply(p)) + raw, _ := json.Marshal(map[string]any{"model": knowledge.Model, "choices": []any{map[string]any{"delta": map[string]any{"content": string(result)}, "finish_reason": "stop"}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .02}}) + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + return + case <-r.Context().Done(): + } + } else { + <-r.Context().Done() + } + select { + case cancelled <- struct{}{}: + default: + } + return + } + delta := map[string]any{"content": `{"title":"Cedar","name":"Cedar"}`} + finish := "stop" + for _, tool := range body.Tools { + if tool.Function.Name == "stage_result" { + if pauseResume && stageTurns.Add(1) == 2 { + close(learning) + select { + case <-finishTurn: + case <-r.Context().Done(): + return + } + } + delta = map[string]any{"tool_calls": []any{map[string]any{"index": 0, "id": "report", "type": "function", "function": map[string]any{"name": "stage_result", "arguments": `{"done":true,"output":"Cedar queues jobs."}`}}}} + finish = "tool_calls" + } + } + providerUnits.Add(1) + raw, _ := json.Marshal(map[string]any{"model": knowledge.Model, "choices": []any{map[string]any{"delta": delta, "finish_reason": finish}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + })) + t.Cleanup(server.Close) + t.Cleanup(releaseReceipt) + original := http.DefaultClient + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + clone := r.Clone(r.Context()) + u := *r.URL + clone.URL = &u + if strings.Contains(u.Host, "openrouter.ai") { + clone.URL.Scheme = "http" + clone.URL.Host = server.Listener.Addr().String() + } + return http.DefaultTransport.RoundTrip(clone) + })} + t.Cleanup(func() { http.DefaultClient = original }) + profile := v3Profile(t, map[string]any{config.KeyTaskModelsMode: "one", config.KeyTaskModelsOne: knowledge.Model, "models.tiers.low": knowledge.Model, "models.tiers.reflex": knowledge.Model, "models.roles": ""}) + t.Setenv(config.ProfileDirEnv, profile) + t.Setenv("OPENROUTER_API_KEY", "fixture") + t.Setenv("CODEAF_BASE_URL", server.URL) + freshEngineProcess(t) + g := newRunRig(t) + proc, err := openV3Process("chat") + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.closeAll) + launch, err := openV3Launch(proc, v3Options{Model: knowledge.Model, OneModel: true, Workspace: g.workspace}) + if err != nil { + t.Fatal(err) + } + if err := knowledge.Open(profile).StartBuild(); err != nil { + t.Fatal(err) + } + stageSettle = 20 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + local := &s.backend.local + local.parent = func() session.Config { return launch.Config } + local.rules = func(context.Context, string, string) (pipeline.File, error) { return pipeline.File{}, os.ErrNotExist } + local.seamHook = func(seam factory.Seam) factory.Seam { + seam.GitHub = func(context.Context) (factory.GitHubLink, error) { return factory.GitHubLink{Login: "owner"}, nil } + return seam + } + s.backend.open = appOpenInProcess + st := s.start() + floor := local.host.floor + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + stages := []factory.Stage{ + {Name: "work", Kind: factory.StageChat, On: true, Until: "done", Ask: "Call stage_result with done true and output Cedar queues jobs."}, + {Name: factory.ApproveName, Kind: factory.StageGate, On: true}, + } + if detachedTail { + stages = append(stages[:1], factory.Stage{Name: "test", Kind: factory.StageCheck, On: true, Ask: "sleep 60"}, stages[1]) + } + if pauseResume { + stages[1].Ask = "if test ! -e memory-check-ran; then touch memory-check-ran; sleep 60; fi" + stages = append(stages[:2], factory.Stage{Name: "learn", Kind: factory.StageChat, On: true, Until: "done", Ask: "Call stage_result with done true and output Cedar queues jobs."}, stages[2]) + } + id, err := floor.Add(context.Background(), factory.Item{Repo: "acme/api", Title: "Cedar queues jobs", ChatSpent: existing, Cap: 1, Tier: factory.TierOwner, Model: knowledge.Model, Stages: stages}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.backend.closeLocalFactory() }) + var job api.Job + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + var acted appwire.FactoryJobActed + s.must("factory.job_start", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + if (stop || quit) && !detachedTail { + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("memory did not start") + } + if quit && !settlingQuit { + go func() { s.backend.closeLocalFactory(); close(quitDone) }() + } else { + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + s.must("factory.job_stop", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + } + } + deadline := time.Now().Add(10 * time.Second) + for detachedTail || (!stop && !quit) { + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + finished := false + for _, step := range job.Steps { + finished = finished || (step.Name == "work" && step.State == "done") + } + if finished { + break + } + if time.Now().After(deadline) { + t.Fatalf("job did not finish: state=%s", job.State) + } + time.Sleep(10 * time.Millisecond) + } + if detachedTail { + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + switch { + case quit: + go func() { s.backend.closeLocalFactory(); close(quitDone) }() + case stop: + s.must("factory.job_stop", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + default: + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + s.must("factory.job_hold", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + } + } + if pauseBoundary { + finishPaidCall() + time.Sleep(250 * time.Millisecond) + if got := memoryCalls.Load(); got != 1 { + t.Fatalf("Pause returned but admitted %d new memory calls", got-1) + } + } + if !pauseBoundary { + select { + case <-cancelled: + case <-time.After(3 * time.Second): + t.Fatal("job end did not cancel memory") + } + } + // Give the existing final-report path a chance to return its lease while + // the receipt is deliberately still unavailable. A settlement-aware worker + // may retain that lease; either way the public job must receive the charge. + delay := 750 * time.Millisecond + if stop || quit { + delay = 6 * time.Second + } + until := time.Now().Add(delay) + for time.Now().Before(until) { + if quit { + time.Sleep(10 * time.Millisecond) + continue + } + var runners api.Runners + s.must("factory.runners", map[string]any{"factory": st.Factory}, &runners) + busy := false + for _, runner := range runners.Runners { + for _, active := range runner.Jobs { + busy = busy || active == id + } + } + if !busy { + t.Log("worker lease ended before receipt was released") + if !stop { + break + } + } + time.Sleep(10 * time.Millisecond) + } + if settlingQuit { + // Stop has revoked the lease and the worker's five-second drop wait + // has elapsed. Quit now interrupts its already-running settlement. + local.host.cancel() + go func() { s.backend.closeLocalFactory(); close(quitDone) }() + } + releaseReceipt() + if quit { + select { + case <-quitDone: + case <-time.After(5 * time.Second): + t.Fatal("quit did not finish within its bounded wait") + } + } + item, err := floor.Get(id) + if err != nil { + t.Fatal(err) + } + deadline = time.Now().Add(5 * time.Second) + for { + session.FlushUsage() + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + usd, memoryUSD := 0.0, 0.0 + for _, line := range lines { + if line.FactoryJob == factory.LedgerName(item) { + usd += line.USD + if line.Role == "memory" { + memoryUSD += line.USD + } + } + } + if quit { + stored, err := floor.Get(id) + if err != nil { + t.Fatal(err) + } + job.Cost = factory.JobSpent(stored) + } else { + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + } + if memoryUSD >= .02 && math.Abs(job.Cost-usd-existing) < .000001 && math.Abs(usd-float64(providerUnits.Load())/100) < .000001 { + if pauseTail { + before := memoryCalls.Load() + time.Sleep(3 * time.Second) + if got := memoryCalls.Load(); got != before { + t.Fatalf("paused memory count grew: %d -> %d", before, got) + } + snapshot, err := knowledge.Open(profile).Snapshot() + // A completed extraction can consume raw sources while leaving + // its durable claims waiting for the rest of compilation. + if err != nil || (snapshot.Pending == 0 && (!pauseBoundary || !snapshot.NeedsUpdate)) { + t.Fatalf("pause lost pending memory: %+v %v", snapshot, err) + } + t.Logf("paused memory calls=%d; ledger=%.4f header=%.4f pending=%d", before, usd, job.Cost, snapshot.Pending) + if pauseResume { + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + s.must("factory.job_resume", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &acted) + select { + case <-learning: + case <-time.After(10 * time.Second): + t.Fatal("resumed job did not reach its later turn") + } + time.Sleep(250 * time.Millisecond) + if got := memoryCalls.Load(); got != before { + t.Fatalf("Resume started memory before a finished turn: %d -> %d", before, got) + } + finishLearning() + until := time.Now().Add(15 * time.Second) + for { + snapshot, err = knowledge.Open(profile).Snapshot() + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + learned := false + for _, step := range job.Steps { + learned = learned || (step.Name == "learn" && step.State == "done") + } + session.FlushUsage() + ledger, _ := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + resumedUSD := 0.0 + for _, line := range ledger { + if line.FactoryJob == factory.LedgerName(item) { + resumedUSD += line.USD + } + } + if err == nil && learned && snapshot.Pending == 0 && memoryCalls.Load() > before && math.Abs(job.Cost-resumedUSD) < .000001 && math.Abs(resumedUSD-float64(providerUnits.Load())/100) < .000001 { + break + } + if time.Now().After(until) { + t.Fatalf("resume did not compile pending evidence: calls=%d snapshot=%+v error=%v", memoryCalls.Load(), snapshot, err) + } + time.Sleep(10 * time.Millisecond) + } + // One pass through this scripted evidence takes at most ten calls. + // The interrupted pre-pause call is the eleventh provider request. + time.Sleep(250 * time.Millisecond) + if got := memoryCalls.Load(); got > 11 { + t.Fatalf("resume exceeded one normal memory pass: calls=%d, limit=11", got) + } + t.Logf("resume compiled retained evidence: memory_calls=%d pending=%d", memoryCalls.Load(), snapshot.Pending) + } + } + return + } + if time.Now().After(deadline) { + t.Fatalf("memory accounting differs: provider=%.4f memory=%.4f ledger=%.4f header=%.4f", float64(providerUnits.Load())/100, memoryUSD, usd, job.Cost) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/cmd/codeaf/chatv3_knowledge.go b/cmd/codeaf/chatv3_knowledge.go index c6c603c6dd..461e0d043c 100644 --- a/cmd/codeaf/chatv3_knowledge.go +++ b/cmd/codeaf/chatv3_knowledge.go @@ -25,7 +25,7 @@ func (p *v3Process) startKnowledge() { } return service.Key } - if err := vault.Run(p.lifetime(), session.KnowledgeHistory(session.PlacesRoot()), key); err != nil && !errors.Is(err, context.Canceled) { + if err := vault.Run(p.lifetime(), session.KnowledgeHistory(session.PlacesRoot(), config.ProfilePath(p.ProfileDir, "")), key); err != nil && !errors.Is(err, context.Canceled) { log.Printf("knowledge: background worker stopped: %v", err) } }) diff --git a/cmd/codeaf/factory_chatspend.go b/cmd/codeaf/factory_chatspend.go index ad9eacc136..155ace8e3a 100644 --- a/cmd/codeaf/factory_chatspend.go +++ b/cmd/codeaf/factory_chatspend.go @@ -373,3 +373,20 @@ func ownsStageChat(it factory.Item, talk talkKey) bool { } return false } + +func (c *jobChat) MemoryRefusal() string { + // Admission follows the owning factory before a worker's delayed mirror. + for _, st := range append([]*store.Store{c.floor}, liveMirrors(c.floor, strconv.Itoa(c.id)+".json")...) { + if it, err := st.Get(c.id); err == nil && (c.talk.is(it.Talk) || ownsStageChat(it, c.talk)) { + return jobMemoryRefusal(it) + } + } + return "job could not be read; memory stopped" +} + +func (c ownedStageBudget) MemoryRefusal() string { + if owner, ok := c.JobChat.(interface{ MemoryRefusal() string }); ok { + return owner.MemoryRefusal() + } + return "job could not be read; memory stopped" +} diff --git a/cmd/codeaf/factory_live_e2e_test.go b/cmd/codeaf/factory_live_e2e_test.go index ca36ba10c8..3cdecd915f 100644 --- a/cmd/codeaf/factory_live_e2e_test.go +++ b/cmd/codeaf/factory_live_e2e_test.go @@ -65,6 +65,8 @@ func liveFactoryProfileDir() string { // liveRecorder is an HTTP forwarding recorder for /chat/completions. type liveRecorder struct { + budget float64 + maxCalls int client *http.Client server *httptest.Server mu sync.Mutex @@ -78,7 +80,7 @@ type liveRecorder struct { func newLiveRecorder(t *testing.T, upstream string) *liveRecorder { t.Helper() - r := &liveRecorder{client: &http.Client{}, requests: map[string]int{}, replies: map[string]int{}, statuses: map[int]int{}} + r := &liveRecorder{budget: liveFactoryBudget, maxCalls: liveFactoryMaxHit, client: &http.Client{}, requests: map[string]int{}, replies: map[string]int{}, statuses: map[int]int{}} r.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Method != http.MethodPost || !strings.HasSuffix(req.URL.Path, "/chat/completions") { http.Error(w, "the live lane forwards chat completions only", http.StatusServiceUnavailable) @@ -103,9 +105,9 @@ func newLiveRecorder(t *testing.T, upstream string) *liveRecorder { over = "request model " + asked + " is not " + liveFactoryModel case body["models"] != nil: over = "request carries a models fallback list" - case r.cost >= liveFactoryBudget: + case r.cost >= r.budget: over = "the budget is spent" - case r.calls > liveFactoryMaxHit: + case r.calls > r.maxCalls: over = "the call cap is reached" } if over != "" { @@ -129,6 +131,9 @@ func newLiveRecorder(t *testing.T, upstream string) *liveRecorder { out.Header.Set("Accept-Encoding", "identity") resp, err := r.client.Do(out) if err != nil { + if req.Context().Err() != nil { + return + } r.note("upstream unreachable") http.Error(w, "upstream unreachable", http.StatusBadGateway) return diff --git a/cmd/codeaf/factory_stage.go b/cmd/codeaf/factory_stage.go index 2f475d3eb3..3c258dd9f0 100644 --- a/cmd/codeaf/factory_stage.go +++ b/cmd/codeaf/factory_stage.go @@ -58,12 +58,13 @@ type stageMaker struct { floor *store.Store // jobs is where the job a step runs for is kept: the worker's mirror of // the job the factory leased it. Nil is floor. - jobs *store.Store - parent session.Config - current func() session.Config - refusal func() error - workspace string - profileDir string + jobs *store.Store + memoryOwner *store.Store + parent session.Config + current func() session.Config + refusal func() error + workspace string + profileDir string } // factoryStageMaker is this process's maker. @@ -152,8 +153,10 @@ func (m stageMaker) Open(ctx context.Context, spec factoryrun.ConversationSpec) if jobs == nil { jobs = m.floor } - budget := &stageBudget{floor: jobs, item: spec.Item} + budget := &stageBudget{floor: jobs, memoryOwner: m.memoryOwner, item: spec.Item} cfg.JobBudget = budget + cfg.JobContext = spec.JobContext + cfg.JobPendingSpend = spec.PendingSpend gate := stageApprovalGate{rows: &v3ApprovalGate{workspace: where, profileDir: m.profileDir, headless: true}, posture: factoryStagePosture()} cfg.ApprovalGate = gate cfg.ApprovalPosture = gate.posture @@ -621,7 +624,11 @@ func (c *stageConversation) Wait(ctx context.Context) error { defer c.mu.Unlock() if c.budget != nil { if why := c.budget.Refusal(); why != "" { - return factoryrun.SpendStop{Job: c.budget.reachedCap(), Said: why} + // A daily refusal already recorded was not caused by memory, so + // the stop keeps its ordinary meaning: an early report is rejected. + var refused factoryrun.SpendStop + daily := errors.As(c.failure, &refused) && refused.Daily + return factoryrun.SpendStop{Job: c.budget.reachedCap(), Memory: !daily && c.budget.memoryCap(), Said: why} } } return c.failure diff --git a/cmd/codeaf/factory_stage_budget.go b/cmd/codeaf/factory_stage_budget.go index 98471acc4a..a14b40ad7c 100644 --- a/cmd/codeaf/factory_stage_budget.go +++ b/cmd/codeaf/factory_stage_budget.go @@ -11,13 +11,14 @@ import ( // One live meter per round; descendants share it. The store is the worker's // authoritative mirror, also read by the manager's meter, not a copied cap. type stageBudget struct { - floor *store.Store - item factory.Item - mu sync.Mutex - spent float64 - failed bool - stopped string - reached bool + floor *store.Store + memoryOwner *store.Store + item factory.Item + mu sync.Mutex + spent float64 + failed bool + stopped string + reached bool } func (b *stageBudget) FactoryAuto() bool { @@ -31,14 +32,22 @@ func (b *stageBudget) FactoryAuto() bool { // FactoryJob names the job on the ledger rows of every call this step makes. func (b *stageBudget) FactoryJob() string { return factory.LedgerName(b.item) } -func (b *stageBudget) Spend(usd float64) { b.book(usd, "", 0, 0, 0) } +func (b *stageBudget) Spend(usd float64) { b.book(usd, false, "", 0, 0, 0) } // SpendModel books one priced call and the model that answered it. func (b *stageBudget) SpendModel(usd float64, model string, calls, input, output int) { - b.book(usd, model, calls, input, output) + b.book(usd, false, model, calls, input, output) } -func (b *stageBudget) book(usd float64, model string, calls, input, output int) { +// SpendMemory records optional learning separately inside the same job total. +func (b *stageBudget) SpendMemory(usd float64) { b.book(usd, true, "", 0, 0, 0) } + +// SpendMemoryModel is SpendMemory with the model that answered the call. +func (b *stageBudget) SpendMemoryModel(usd float64, model string, calls, input, output int) { + b.book(usd, true, model, calls, input, output) +} + +func (b *stageBudget) book(usd float64, memory bool, model string, calls, input, output int) { if usd <= 0 { return } @@ -53,7 +62,11 @@ func (b *stageBudget) book(usd float64, model string, calls, input, output int) if it.Repo != b.item.Repo { return errNotThisChat } - factory.AddChatSpend(it, usd) + if memory { + factory.AddMemorySpend(it, usd) + } else { + factory.AddChatSpend(it, usd) + } factory.AddModelUse(it, model, usd, calls, int64(input), int64(output)) return nil }); err != nil { @@ -86,3 +99,42 @@ func (b *stageBudget) total() float64 { } func (b *stageBudget) reachedCap() bool { b.mu.Lock(); defer b.mu.Unlock(); return b.reached } + +// MemoryRefusal keeps optional learning inside the job's active work, even if +// a foreground conversation is still open after its job published or paused. +func (b *stageBudget) MemoryRefusal() string { + owner := b.memoryOwner + if owner == nil { + owner = b.floor + } + if owner == nil { + return "job could not be read; memory stopped" + } + it, err := owner.Get(b.item.ID) + if err != nil { + return "job could not be read; memory stopped" + } + return jobMemoryRefusal(it) +} + +func jobMemoryRefusal(it factory.Item) string { + if it.Stream != nil && it.Stream.Paused { + return "job is paused; memory stopped" + } + if it.State == factory.StateShipped || factory.JobState(it) == "done" || (it.Stream != nil && !it.Stream.Ended.IsZero()) { + return "job has ended; memory stopped" + } + return "" +} + +// memoryCap reports a crossing caused only by memory since this round began. +func (b *stageBudget) memoryCap() bool { + if b.floor == nil { + return false + } + it, err := b.floor.Get(b.item.ID) + if err != nil || it.Repo != b.item.Repo || it.Cap <= 0 { + return false + } + return factory.JobSpent(it) >= it.Cap && factory.JobSpent(it)-max(0, it.MemorySpent-b.item.MemorySpent) < it.Cap +} diff --git a/docs/changes/unreleased/2261-memory-budget-cause.md b/docs/changes/unreleased/2261-memory-budget-cause.md new file mode 100644 index 0000000000..dc06347416 --- /dev/null +++ b/docs/changes/unreleased/2261-memory-budget-cause.md @@ -0,0 +1,18 @@ +--- +kind: fixed +title: codeaf preserves stage budget questions and bounded knowledge batches +pr: 2261 +surface: [engine] +invalidates: + - "A completed stage was exempt from every budget stop instead of only a crossing caused by optional memory." + - "An ordinary forced knowledge Flush drained additional batches instead of retaining its single-batch limit." +--- + +A step's own model and tool spending still asks at the cap and rejects an early +completion report when that spending stops it. Memory charges remain included +in the job total and cannot revoke completed work when only memory crosses the +cap. The next step asks before starting; the last step finishes at the true +spend without asking for unused budget. + +Ordinary knowledge compilation processes one bounded batch per Flush. Owned job +turns continue draining their finite captured snapshot. diff --git a/docs/changes/unreleased/2261-memory-durable-stage-spend.md b/docs/changes/unreleased/2261-memory-durable-stage-spend.md new file mode 100644 index 0000000000..2dba16cbf2 --- /dev/null +++ b/docs/changes/unreleased/2261-memory-durable-stage-spend.md @@ -0,0 +1,12 @@ +--- +kind: fixed +title: codeaf retains memory spending after a stage closes +pr: 2261 +surface: [engine] +invalidates: + - "A closed stage could lose paid memory from its saved conversation total." +--- + +Memory that finishes after a stage closes now appends its paid usage to the +existing conversation journal. Reopening restores that spending along with the +stage's other calls, so the next spending update retains the paid memory. diff --git a/docs/changes/unreleased/2261-memory-paused-admission.md b/docs/changes/unreleased/2261-memory-paused-admission.md new file mode 100644 index 0000000000..6d783bb139 --- /dev/null +++ b/docs/changes/unreleased/2261-memory-paused-admission.md @@ -0,0 +1,14 @@ +--- +kind: fixed +title: codeaf keeps paused job memory pending without spending +pr: 2261 +surface: [engine] +invalidates: + - "A memory pass could keep spending after its job paused without appearing in the job's spending." +--- + +Pause and Stop end a job's background memory work, including work left behind +by a finished step. Paid requests already sent still charge the job when their +receipts arrive within the existing settlement window. Paused jobs admit no new +memory calls. Pending sources remain for a later finished turn after Resume; +Resume itself does not start another memory pass. Completed steps stay done. diff --git a/docs/changes/unreleased/2261-memory-quit-settlement.md b/docs/changes/unreleased/2261-memory-quit-settlement.md new file mode 100644 index 0000000000..ceb04954b8 --- /dev/null +++ b/docs/changes/unreleased/2261-memory-quit-settlement.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: codeaf gives paid job calls a bounded settlement on quit +pr: 2261 +surface: [engine] +invalidates: + - "Quitting cancelled the worker's final accounting request before it could report paid calls." +--- + +After cancelling work, a quitting worker gives its existing accounting receipt +up to five additional seconds to report paid calls before removing its local job +copy. This applies to ordinary stage calls and memory alike. Later receipts are +not guaranteed to reach the owning job, and pending recovery is not durable. diff --git a/docs/changes/unreleased/2261-memory-spend-counted.md b/docs/changes/unreleased/2261-memory-spend-counted.md new file mode 100644 index 0000000000..cbc3ffe2fc --- /dev/null +++ b/docs/changes/unreleased/2261-memory-spend-counted.md @@ -0,0 +1,40 @@ +--- +kind: fixed +title: codeaf counts memory spending and keeps job learning inside its turn +pr: 2261 +surface: [engine] +invalidates: + - "Memory compiler costs were kept only in its private receipts, so Spending and job headers understated provider spending. Memory calls now enter the usage ledger with the memory role, and owned calls charge the job's existing meter." + - "The process worker could import job conversations and continue compiling them after Pause or completion. Job sources now belong to the job turn and are excluded from independent background compilation. Idle polling no longer launches refinement without pending sources or retries the same failed input without new sources or an explicit Build." +--- + +Saved knowledge, citations, search and page organization remain available. +A finished job turn waits at most two seconds for optional memory. Unfinished +compilation continues with its job across steps. Pause, Stop and job end cancel +it, while paid receipts still charge that job. Later turns do not stack compilers. +Quit leaves unfinished sources pending for the next finished owned turn. Compilation can require multiple dependent model requests +for extraction, organization and evidence review. Unchanged input reuses validated +receipts. Large finished turns drain their finite captured snapshot before +publication. Older job sources are reclassified from their persisted ownership. +Existing historical ledger gaps are not reconstructed. + +Old job chats with factory-owner.json remain owned even after their roster +entry disappears, so background memory does not spend for them. + +Delegated task reports compile with their owning job turn, keep their citation +identities, and charge the same job instead of remaining pending indefinitely. + +Memory spending cannot revoke a successful stage result. Admission to the next +step still sees the full job spend, including late memory receipts. + +Pending provider receipt recovery still has the ordinary-chat quit limit: it is +not durable, and quitting before the receipt arrives can leave a charge unpriced. + +The next admitted job pass includes earlier steps skipped while another pass was +running, preserving their searchable evidence and original citations. + +The worker keeps reporting while final memory receipts settle, so late prices +reach the job header as well as Spending without extending a step’s memory wait. + +Stop also reconciles final paid memory through the existing accounting receipt, +without reopening the stopped work or counting an acknowledged price twice. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index f0a1c1e2b7..e0963ede6c 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -1102,6 +1102,18 @@ time. `search.all` retains its existing `bad_params` error for an empty query. | `settings.models.set` | `in.TaskModelsPatchWire` | `TaskModelsResult` | | `settings.set` | `in.SettingsPatchWire` | `SettingsResult` | +An engine whose hello carries `settings.memory` includes `memory` in Settings +(`settings.get`, `settings.set`, and `settings.changed`): +`{enabled: bool, source: "default"|"saved", cost_note: string}`. No saved choice +means off with source `default`. A saved word (`on`/`off`) or boolean is respected. +`settings.set` takes optional `memory_enabled: bool`; any non-boolean, including +null, is `bad_params`. Its reply and `settings.changed` carry the updated Settings. +The switch applies live at the next memory step. Off prevents automatic learning +calls; saved knowledge and explicit keeps remain available. Calls already in +flight finish and are billed. Terminal Build saves an explicit on. +`cost_note` is the shared `config.MemoryCostNote` disclosure, including the example +of one small DeepSeek V4.1 Flash job at 39% ($0.0124 of $0.0315). + An engine whose hello carries the feature `settings.models.auxiliary` adds a fourth role to the task models: `roles.auxiliary` in `settings.models.get` (and in `settings.changed`'s `task_models`) is `{pin, warning, eligible_count}` like diff --git a/docs/knowledge.md b/docs/knowledge.md index 9c47e29736..41d5f34d29 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -38,6 +38,67 @@ updates affected guides or creates distinct concepts. Source eligibility and concept organization are separate responsibilities. Saved task reports can supply evidence; internal control traffic and deleted history remain excluded. +## Memory spending and job lifetime + +The compiler uses the provider funnel with `lane.RoleMemory`. Session startup +connects `knowledge.SetUsageRecorder` to the shared usage ledger. The compiler +arms this sink before sending a request, so paid malformed answers and discarded +attempts are charged independently of validated compiler-cache receipts. +An owned handle binds admission and billing through `Vault.WithCalls`; its sink +banks each provider receipt through the agent's existing ledger and job meter. +The standalone application worker uses the same ledger with no invented job. + +Job journals carry a durable `.knowledge-job` marker. Pre-upgrade +`factory-owner.json` markers also retain ownership without a roster entry. History inventory marks +those journals as owned, and source admission retains a scheduling-only `job` flag. The +process compiler excludes owned sources. Persisted factory team memberships +also identify older job journals, including closed teams, so existing pending +sources are reclassified without model calls. Delegated `task::...` +reports keep their source identities and compile with the owner’s finished turn, +charged to that job. Each finished owned turn can start one compilation. `jobKnowledgeWait` caps its +stage-end wait at two seconds; unfinished compilation continues under the job's +lifetime across stage conversation closes. Only one compiler per job runs at a +time, and later finished turns do not queue compilers. History groups sibling +stage journals by their durable job owner. The next admitted pass captures those +journals and selects their original session IDs through `WithOwnerSessions`, so +a skipped stage remains eligible without changing its citations. The runner cancels that +lifetime on termination. Owner admission is checked before requests and retries; +Pause and Stop cancel the active call, and no new call starts for a paused or +ended job. Quit drops compilation, retaining source journals and checkpoints for +the next finished owned turn. The process worker never takes over owned sources. +A standalone owned conversation cancels its compilation on Close. + +One compilation can need several dependent requests: extraction and discovery +already require two for a small new concept; guide writing and evidence review +add their existing bounded stages. Their output and the 8,000-token response +ceiling are preserved. The process worker does not launch another refinement +pass merely because the previous pass left editorial work pending. New source +input or an explicit build can advance that work. The projection records the +Build request it consumed, so a forced flush alone cannot buy another refinement +pass on unchanged sources. Failed input is attempted once; +new sources or an explicit Build permit another attempt. Historical unrecorded costs +are not backfilled. + +A successful stage result is retained even when deferred memory spending reaches +its cap. The runner records the finished phase before checking admission for the +next unfinished step. Existing admission checks measured spend, not a prediction +of the next call's price, so an admitted call can cross the remaining budget. + +After job work finishes, the existing worker report path stays open while memory +receipts settle, up to the provider recovery ceiling plus one second. This keeps +late prices in the job header without extending a step's two-second memory wait. +Ordinary worker report acknowledgments retain their existing recovery digest. +After Stop revokes a lease, that digest admits only outstanding accounting through +the existing recovery route; stopped work is not resumed or replaced. Ordinary +reported deltas advance the same receipt atomically, so a lost report response +cannot cause recovery to charge its cost twice. + +Receipt recovery has the ordinary turn's existing quit limit: generation IDs and +pending recovery live in the provider client's in-memory queue. Neither ordinary +turns nor memory persist that queue or join it during application shutdown. A +receipt arriving while the process remains alive is charged to its owner; quitting +before recovery finishes can leave that paid call unpriced in the ledger. + ## Storage authority | Stored data | Authority | diff --git a/internal/factory/api/codec.go b/internal/factory/api/codec.go index 072e17a0a3..dd36054d70 100644 --- a/internal/factory/api/codec.go +++ b/internal/factory/api/codec.go @@ -109,6 +109,7 @@ func ToSnapshot(q Queue, jobs []Job) factory.Snapshot { // Lost names the Go fields the wire does not carry, and why. The round-trip // test reads it, so a field is either carried or named here. var Lost = map[string]string{ + "Item.MemorySpent": "runner-local provenance already included in total cost; completed-round checks compare only charges since local admission, so readers and a newly admitted worker need no historical split", "Snapshot.People": "the queue projects identities onto mentioned handles only; unrelated cached profiles do not cross", "Item.PlannedFiles": "the cache never crosses: the wire's planned_files is derived from the plan step's result (factory.PlanFilesOf), which does", "Item.BriefSeen": "each person's own: the caller's crosses as brief_seen (ForViewer), and nobody else's ever does", diff --git a/internal/factory/report_receipt.go b/internal/factory/report_receipt.go index ff91bb0016..e63ecdce8b 100644 --- a/internal/factory/report_receipt.go +++ b/internal/factory/report_receipt.go @@ -6,6 +6,7 @@ import "time" // payloads or machine-local paths. It is not writable through worker reports. type WorkerReportReceipt struct { Digest string + Lease string `json:",omitempty"` At time.Time AdditionalCost float64 `json:",omitempty"` } diff --git a/internal/factory/run/chatstage.go b/internal/factory/run/chatstage.go index 1fe9203b78..394fd6e721 100644 --- a/internal/factory/run/chatstage.go +++ b/internal/factory/run/chatstage.go @@ -59,6 +59,9 @@ type StageDoor interface { // ConversationSpec is everything the maker needs to open one stage round. type ConversationSpec struct { + // JobContext owns optional work that may outlive this stage conversation. + JobContext context.Context + PendingSpend func() func() // VerifyRepair makes successful PR-repair reports pass the concrete receipt guard. // Read-only readiness reviews retain their separate current-PR snapshot contract. VerifyRepair bool @@ -264,6 +267,8 @@ func (e chatExecutor) Run(ctx context.Context, job Job) (factory.StageResult, er log = func(string) {} } spec := ConversationSpec{ + JobContext: job.Lifetime, + PendingSpend: job.PendingSpend, Item: job.Item, Team: itemTeam(job.Item), Name: stageChatName(job), @@ -504,7 +509,9 @@ loop: made.Edit = res.Edit res = made } - if stop, ok := spendStopOf(waitErr); ok { + // A final successful result survives optional work charged as the turn + // settles. The next stage still reads the complete live job spend. + if stop, ok := spendStopOf(waitErr); ok && !(stop.Job && stop.Memory && !stop.Daily && res.Done) { res.JobBudgetStopped = stop.Job res.Reports, res.Inferred = 0, false res.Done, res.Exit, res.Output = false, 1, stop.Error() diff --git a/internal/factory/run/contract.go b/internal/factory/run/contract.go index bfbefe609a..c24dd3dfef 100644 --- a/internal/factory/run/contract.go +++ b/internal/factory/run/contract.go @@ -48,6 +48,11 @@ type ModelSelection struct { } type Job struct { + // Lifetime ends with the job, rather than with one stage round. + Lifetime context.Context + // PendingSpend retains the existing worker report path while owned work + // can still bank a receipt. Calling the returned function settles one unit. + PendingSpend func() func() // Item is the item as it stands when the round starts. Item factory.Item // Stage is the stage being run and Index its place in Item.Stages. diff --git a/internal/factory/run/loop.go b/internal/factory/run/loop.go index 3f29accbe6..5ef695f14e 100644 --- a/internal/factory/run/loop.go +++ b/internal/factory/run/loop.go @@ -29,6 +29,7 @@ import ( "github.com/Agent-Field/codeaf/internal/factory/record" "github.com/Agent-Field/codeaf/internal/factory/store" "github.com/Agent-Field/codeaf/internal/factory/verification" + "github.com/Agent-Field/codeaf/internal/provider" ) // loopLogMost is how many lines a stream keeps, the newest last: the same @@ -97,6 +98,7 @@ type loopCtl struct { id int ctx context.Context cancel context.CancelFunc + spending sync.WaitGroup steer chan string modelSelections chan ModelSelection modelMu sync.Mutex @@ -468,6 +470,7 @@ func (lp *floorLoop) say(c *loopCtl, tone, text string) { // order, and lands the item. func (lp *floorLoop) drive(c *loopCtl) { defer lp.release(c) + defer c.settleSpend() if c.reverify { lp.reverifyRun(c) return @@ -880,7 +883,7 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { if !lp.held(c) { return false } - if !lp.capCheck(c, i, name) { + if !lp.capCheck(c, i, name, -1) { return false } if factory.IsCIRepairStage(st) && !lp.refreshPRRound(c, i) { @@ -1013,7 +1016,13 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { } res = unchangedWrite(st, job, res) lp.fold(c, it, st, i, res) - if !lp.capCheck(c, i, name) { + // Completed work is recorded before the next step asks for admission. + // Optional memory charges cannot turn this phase back into waiting. + memoryBefore := -1.0 + if factory.Met(st, res) { + memoryBefore = job.Item.MemorySpent + } + if !lp.capCheck(c, i, name, memoryBefore) { return false } if res.JobBudgetStopped { @@ -1065,7 +1074,7 @@ func (lp *floorLoop) runPhase(c *loopCtl, i int) bool { } if readiness { if res.Repair != nil && res.Repair.Decision == "needs_repair" && strings.TrimSpace(res.Repair.Reason) != "" { - if !lp.capCheck(c, i, name) { + if !lp.capCheck(c, i, name, -1) { return false } if e := lp.write(c, func(cur *factory.Item) error { @@ -1515,6 +1524,8 @@ func (lp *floorLoop) job(c *loopCtl, st factory.Stage, index, i, round int) Job } } return Job{ + Lifetime: c.ctx, + PendingSpend: c.pendingSpend, Item: it, Stage: st, Index: index, @@ -1567,11 +1578,16 @@ func (lp *floorLoop) spender(c *loopCtl) func(usd float64) { // capCheck parks the item when its budget is reached after a round: yes raises // the budget to the next allowed amount above spend, no stops it. // It answers false when it stopped. -func (lp *floorLoop) capCheck(c *loopCtl, i int, name string) bool { +func (lp *floorLoop) capCheck(c *loopCtl, i int, name string, memoryBefore float64) bool { it, err := lp.r.opts.Store.Get(c.id) if err != nil { return false } + if memoryBefore >= 0 && it.Stream != nil { + stream := *it.Stream + stream.Spent -= max(0, it.MemorySpent-memoryBefore) + it.Stream = &stream + } reached := it.Cap > 0 && it.Stream != nil && it.Stream.Spent >= it.Cap q, kind := (*Money)(nil).CapQuestion(it) if p := lp.r.opts.Pool; p != nil { @@ -2752,3 +2768,24 @@ func (lp *floorLoop) retailTail(c *loopCtl) { return nil }) } + +func (c *loopCtl) pendingSpend() func() { + c.spending.Add(1) + return c.spending.Done +} + +// Work is already visibly finished. Cancel compilation first, then leave its +// existing periodic worker reports alive until the resulting receipts land. +func (c *loopCtl) settleSpend() { + c.cancel() + settled := make(chan struct{}) + go func() { c.spending.Wait(); close(settled) }() + // ReceiptWait starts when a cancelled call queues its receipt. One second + // lets that call unwind before the provider's own bounded schedule starts. + timer := time.NewTimer(provider.ReceiptWait + time.Second) + defer timer.Stop() + select { + case <-settled: + case <-timer.C: + } +} diff --git a/internal/factory/run/stagereport.go b/internal/factory/run/stagereport.go index 2d1196be31..89eb92de5b 100644 --- a/internal/factory/run/stagereport.go +++ b/internal/factory/run/stagereport.go @@ -97,9 +97,11 @@ func stageRetry(why string) string { // limit's own words. A round that ends on one is a stop, not a step that never // reported, and the loop stops the job on it ([spendStopped]). type SpendStop struct { - Job bool - Daily bool - Said string + // Memory marks a job cap crossed only by optional learning during this round. + Memory bool + Job bool + Daily bool + Said string } func (s SpendStop) Error() string { diff --git a/internal/factory/serve/workers.go b/internal/factory/serve/workers.go index 03ab6cf4b0..e3a397a59a 100644 --- a/internal/factory/serve/workers.go +++ b/internal/factory/serve/workers.go @@ -1,11 +1,13 @@ package serve import ( + "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" "errors" + "io" "net/http" "reflect" "strings" @@ -524,6 +526,9 @@ func (h *Handler) reportWorker(w http.ResponseWriter, r *http.Request) { if !h.workerReady(w) { return } + var raw bytes.Buffer + defer r.Body.Close() + r.Body = io.NopCloser(io.TeeReader(r.Body, &raw)) var body api.WorkerReport if !h.decode(w, r, &body, false) { return @@ -547,11 +552,35 @@ func (h *Handler) reportWorker(w http.ResponseWriter, r *http.Request) { h.applyWorker(w, r, false, func(it *factory.Item) error { return factory.MergeThread(it, report.Thread, report.Delivered) }) return } + headers := http.Header{} + for _, name := range []string{"Idempotency-Key", api.StepModelsHeader, api.StepChatsHeader, api.StepRuntimeHeader} { + if value := r.Header.Get(name); value != "" { + headers.Set(name, value) + } + } + _, digest, _ := retainedWorkerReport(api.WorkerRecovery{OriginalLease: r.PathValue("lease"), Body: raw.Bytes(), Headers: headers}) h.applyWorker(w, r, false, func(it *factory.Item) error { before := failureKey(it.Failure) if err := run.ApplyReport(it, report, h.workers.now()); err != nil { return err } + if digest != "" { + // Advance earlier acknowledgments with ordinary deltas too. Recovery + // can then name cumulative spend despite an ambiguous later response. + for i := range it.ReportReceipts { + if it.ReportReceipts[i].Lease == r.PathValue("lease") { + it.ReportReceipts[i].AdditionalCost += report.Cost + } + } + // Only the initial empty report creates a settlement anchor. Later + // reports advance it without displacing it from bounded history. + if reflect.DeepEqual(report, factory.RunReport{}) { + it.ReportReceipts = append(it.ReportReceipts, factory.WorkerReportReceipt{Digest: digest, Lease: r.PathValue("lease"), At: h.workers.now()}) + if len(it.ReportReceipts) > factory.WorkerReportReceiptsMost { + it.ReportReceipts = append([]factory.WorkerReportReceipt(nil), it.ReportReceipts[len(it.ReportReceipts)-factory.WorkerReportReceiptsMost:]...) + } + } + } after := it.Failure // A PERSON'S STOP IS NOT A STEP THAT ERRORED: a worker whose run // stopped on a person's no reports the cut step with the stop on it, diff --git a/internal/factory/spend.go b/internal/factory/spend.go index f8dbbe1bdb..3093f84903 100644 --- a/internal/factory/spend.go +++ b/internal/factory/spend.go @@ -348,3 +348,12 @@ func SpentAt(it Item) time.Time { } return it.Created } + +// AddMemorySpend books optional learning in the total and identifies its share. +func AddMemorySpend(it *Item, usd float64) { + if it == nil || usd <= 0 { + return + } + AddChatSpend(it, usd) + it.MemorySpent += usd +} diff --git a/internal/factory/types.go b/internal/factory/types.go index 9a5ccaaa8d..5bb8d021f9 100644 --- a/internal/factory/types.go +++ b/internal/factory/types.go @@ -702,6 +702,8 @@ type Item struct { // zeroes it (run's freshStream). A job with a stream books its chat's // spend there instead ([AddChatSpend]), so [JobSpent] reads one figure. ChatSpent float64 `json:",omitempty"` + // MemorySpent is the memory portion already included in the job total. + MemorySpent float64 `json:",omitempty"` // ChatModels is the model split of ChatSpent. The next start carries it // onto the run with the dollars and clears it (run's freshStream). ChatModels []ModelUse `json:",omitempty"` diff --git a/internal/factory/worker/worker.go b/internal/factory/worker/worker.go index 7127bc1f89..04b9d48621 100644 --- a/internal/factory/worker/worker.go +++ b/internal/factory/worker/worker.go @@ -22,6 +22,7 @@ import ( "github.com/Agent-Field/codeaf/internal/factory/run" "github.com/Agent-Field/codeaf/internal/factory/store" factorythread "github.com/Agent-Field/codeaf/internal/factory/thread" + "github.com/Agent-Field/codeaf/internal/provider" ) // Options is one machine's worker against one factory. @@ -291,18 +292,20 @@ type job struct { id int sent factory.Item // the mirror as last reported - mu sync.Mutex - replying bool - replyCancel context.CancelFunc - accepted map[string]bool - acknowledged map[string]bool - thread []factory.Message - pending *pendingDelivery - blocked string - withdrawn bool - control *client.PreparedWorkerReport - controlTries int - controlSent bool + mu sync.Mutex + replying bool + replyCancel context.CancelFunc + accepted map[string]bool + acknowledged map[string]bool + thread []factory.Message + pending *pendingDelivery + blocked string + withdrawn bool + control *client.PreparedWorkerReport + controlTries int + controlSent bool + spendReport *client.PreparedWorkerReport + spendBaseline float64 } // work runs one pulled job to its end, or until the lease is gone. @@ -339,6 +342,25 @@ func (o Options) work(ctx context.Context, w *client.WorkerClient, pulled api.Wo }) defer o.Runner.SetStopAfterStepReader(it.ID, nil) } + // Establish an accounting acknowledgment before the first paid call. + key, err := deliveryKey() + if err != nil { + j.refuse(ctx, err) + return + } + anchor, err := client.PrepareWorkerReport(api.WorkerReport{}, key) + if err == nil { + err = j.sendRequest(ctx, anchor) + } + if err != nil { + j.refuse(ctx, err) + return + } + j.spendReport = &anchor + if it.Stream != nil { + j.spendBaseline = it.Stream.Spent + } + defer j.settleAccounting(ctx) if err := start(o.Runner, it.ID); err != nil { j.refuse(ctx, err) return @@ -691,6 +713,43 @@ func gone(err error) bool { return errors.Is(err, client.ErrLeaseGone) || errors.Is(err, client.ErrNotAttached) } +// A revoked lease cannot carry execution reports. Its acknowledged request can +// still reconcile accounting alone through the existing receipt door. +func (j *job) settleAccounting(ctx context.Context) { + if j.spendReport == nil { + return + } + // Quit can arrive before or during settlement. Keep its existing receipt + // door alive for five seconds after cancellation, before mirror cleanup. + parent := ctx + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + defer cancel() + stop := context.AfterFunc(parent, func() { time.AfterFunc(StopWait, cancel) }) + defer stop() + deadline := time.Now().Add(provider.ReceiptWait + time.Second) + for j.o.Runner.Running(j.id) && time.Now().Before(deadline) { + if !pause(ctx, 20*time.Millisecond) { + return + } + } + cur, err := j.o.Mirror.Get(j.id) + if err != nil || cur.Stream == nil { + return + } + additional := max(0, cur.Stream.Spent-j.spendBaseline) + if additional == 0 { + return + } + for attempt := range deliveryRecoveryLimit { + requestCtx, cancel := context.WithTimeout(ctx, deliveryRequestTimeout) + _, err := j.w.Recover(requestCtx, j.id, j.lease, *j.spendReport, additional) + cancel() + if err == nil || !reportRetryable(err) || !pause(ctx, time.Duration(attempt+1)*time.Second) { + return + } + } +} + func (j *job) lostLease(err error) bool { if errors.Is(err, client.ErrLeaseWithdrawn) { j.withdrawn = true diff --git a/internal/knowledge/accounting.go b/internal/knowledge/accounting.go new file mode 100644 index 0000000000..3c60977844 --- /dev/null +++ b/internal/knowledge/accounting.go @@ -0,0 +1,57 @@ +package knowledge + +import ( + "context" + "sync" + + "github.com/Agent-Field/codeaf/internal/provider" +) + +var usageRecorder struct { + sync.RWMutex + record func(context.Context, provider.Billed) +} + +// SetUsageRecorder connects the application's ledger without making the vault +// depend on the session engine. An owned call's billing sink takes precedence. +func SetUsageRecorder(record func(context.Context, provider.Billed)) { + usageRecorder.Lock() + defer usageRecorder.Unlock() + usageRecorder.record = record +} + +func memoryBilling(ctx context.Context) context.Context { + if provider.BillingSinkFrom(ctx) != nil { + return ctx + } + usageRecorder.RLock() + record := usageRecorder.record + usageRecorder.RUnlock() + if record == nil { + return ctx + } + base := ctx + ctx = provider.WithBilling(ctx, func(b provider.Billed) { record(base, b) }) + ctx = provider.WithReconcile(ctx, func(r provider.Reconciled) { + if r.Found { + record(base, r.Billed) + } + }) + return provider.WithUnmeteredReceipts(ctx) +} + +// WithCalls binds compilation to its owner's admission and billing context. +// The returned handle shares durable state with other handles of this vault. +func (v *Vault) WithCalls(calls func(context.Context) (context.Context, error)) *Vault { + copy := *v + copy.calls = calls + return © +} + +type ownerSessionsKey struct{} + +// WithOwnerSessions admits the saved conversations of one job to a compilation. +// Citation identities and the initiating call's billing session remain unchanged. +func WithOwnerSessions(ctx context.Context, sessions []string) context.Context { + return context.WithValue(ctx, ownerSessionsKey{}, append([]string(nil), sessions...)) +} diff --git a/internal/knowledge/build.go b/internal/knowledge/build.go index 6438505d17..aac241e7e5 100644 --- a/internal/knowledge/build.go +++ b/internal/knowledge/build.go @@ -36,6 +36,9 @@ type Journal struct { Error string Size int64 Modified time.Time + // JobOwner groups stage and task journals without changing source identities. + JobOwner string + Job bool } type JournalCursor struct { Offset int64 @@ -72,10 +75,8 @@ func (v *Vault) StartBuild() error { if err != nil { return err } - if textValue(job["phase"]) == string(BuildRunning) { - return nil - } job["backfill"] = true + job["request"] = time.Now().UTC().Format(time.RFC3339Nano) job["phase"] = string(BuildRunning) job["error"] = "" if err = saveState(tx, "job", job); err != nil { @@ -134,7 +135,7 @@ func (v *Vault) Run(ctx context.Context, history History, key func() string) err return loadErr } job["phase"] = string(BuildFailed) - job["error"] = "Knowledge build paused: " + err.Error() + ". Completed progress is retained; retrying automatically." + job["error"] = "Knowledge build paused: " + err.Error() + ". Completed progress is retained; use Build to retry or add new sources." saveErr := saveState(db, "job", job) db.Close() if saveErr != nil { @@ -169,6 +170,13 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, return err } defer db.Close() + for _, journal := range journals { + if journal.Job { + if _, err = db.Exec("UPDATE events SET payload=json_set(payload,'$.job',json('true')) WHERE partition='profile' AND json_extract(payload,'$.session')=? AND coalesce(json_extract(payload,'$.job'),0)=0", journal.Session); err != nil { + return err + } + } + } settings, err := stateDocument(db, "config") if err != nil { return err @@ -265,6 +273,7 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, return err } for _, source := range batch.Sources { + source.Job = journal.Job if err = validateCapture(source); err == nil { err = v.captureTx(tx, source) } @@ -311,6 +320,16 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile'").Scan(&total); err != nil { return err } + var latest int64 + var admitted int + var arrived float64 + if err = db.QueryRow("SELECT coalesce(max(rowid),0),count(*),coalesce(max(arrived),0) FROM events WHERE partition='profile' AND coalesce(json_extract(payload,'$.job'),0)=0").Scan(&latest, &admitted, &arrived); err != nil { + return err + } + attemptInput := identity([]any{latest, admitted, arrived, job["request"], knowledgeProjectionVersion}) + if job["failed_input"] == attemptInput { + return nil + } job["problems"] = problems job["sources"] = total job["chats"] = len(journals) @@ -320,14 +339,10 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, if err = ctx.Err(); err != nil { return err } - if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND status IN ('pending','revisit')").Scan(&pending); err != nil { - return err - } - status, err := stateDocument(db, "status") - if err != nil { + if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND status IN ('pending','revisit') AND coalesce(json_extract(payload,'$.job'),0)=0").Scan(&pending); err != nil { return err } - if pending == 0 && !flag(status["needs_update"]) { + if pending == 0 && !flag(job["backfill"]) { projection, err := stateDocument(db, "projection") if err != nil { return err @@ -341,13 +356,20 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, return err } if err = v.Flush(ctx, key(), flag(job["backfill"])); err != nil { + job["failed_input"] = attemptInput + if saveErr := saveState(db, "job", job); saveErr != nil { + return saveErr + } return err } var remaining int - if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND status IN ('pending','revisit')").Scan(&remaining); err != nil { + if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND status IN ('pending','revisit') AND coalesce(json_extract(payload,'$.job'),0)=0").Scan(&remaining); err != nil { return err } - if remaining > 0 && remaining >= pending { + if remaining == 0 { + break + } + if remaining >= pending { return nil } } diff --git a/internal/knowledge/compiler.go b/internal/knowledge/compiler.go index 2599a233d7..237943a525 100644 --- a/internal/knowledge/compiler.go +++ b/internal/knowledge/compiler.go @@ -7,11 +7,13 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "unicode/utf8" "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/provider" ) func knowledgeModel(profile string) string { return config.KnowledgeModelAt(profile) } @@ -25,6 +27,12 @@ func (v *Vault) compile(ctx context.Context, key string, force bool) (err error) if err = ctx.Err(); err != nil { return err } + if v.calls != nil { + ctx, err = v.calls(ctx) + if err != nil { + return err + } + } lock, err := os.OpenFile(filepath.Join(v.root, ".compiler.lock"), os.O_CREATE|os.O_RDWR, 0600) if err != nil { return err @@ -51,7 +59,7 @@ func (v *Vault) compile(ctx context.Context, key string, force bool) (err error) return err } defer db.Close() - model := compilerModel{ctx: ctx, root: v.root, key: key, model: knowledgeModel(v.profile), db: db} + model := compilerModel{ctx: ctx, root: v.root, key: key, model: knowledgeModel(v.profile), db: db, calls: v.calls} defer func() { message := "" if err != nil { @@ -75,31 +83,46 @@ func (v *Vault) compile(ctx context.Context, key string, force bool) (err error) if err = v.compilerStatus(db, model.model, "", true); err != nil { return err } - boundary, err := stateDocument(db, "organization_boundary") + boundaryKey := "organization_boundary" + if owner := provider.SessionFrom(ctx); owner != "" { + boundaryKey += ":" + owner + } + boundary, err := stateDocument(db, boundaryKey) if err != nil { return err } through := int64(number(boundary["through"])) - if through == 0 { + if through == 0 || provider.SessionFrom(ctx) != "" { if err = db.QueryRow("SELECT coalesce(max(rowid),0) FROM events WHERE partition='profile'").Scan(&through); err != nil { return err } - if err = saveState(db, "organization_boundary", document{"through": through}); err != nil { + if err = saveState(db, boundaryKey, document{"through": through}); err != nil { return err } } - vault, err := v.compilePending(db, "profile", model, force, through) + before, err := pendingCompilerSources(db, ctx, through) if err != nil { return err } - // First-build organization needs the complete admitted history, rather than - // a tree whose page boundaries reflect extraction batches. - var pending int - if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND rowid<=? AND status IN ('pending','revisit')", through).Scan(&pending); err != nil { - return err - } - if pending > 0 { - return nil + var vault document + for { + vault, err = v.compilePending(db, "profile", model, force, through) + if err != nil { + return err + } + remaining, err := pendingCompilerSources(db, ctx, through) + if err != nil { + return err + } + if remaining == 0 { + break + } + // Only an owned turn drains its finite snapshot. Ordinary Flush keeps + // its bounded single-batch admission even when forced. + if !force || provider.SessionFrom(ctx) == "" || remaining >= before { + return nil + } + before = remaining } projection, err := stateDocument(db, "projection") if err != nil { @@ -110,9 +133,24 @@ func (v *Vault) compile(ctx context.Context, key string, force bool) (err error) if err != nil { return err } - needsLearning := textValue(projection["input_hash"]) != digest || hasPendingRefinement(learning) + intent, err := stateDocument(db, "job") + if err != nil { + return err + } + request := textValue(intent["request"]) + explicitBuild := request != "" && request != textValue(projection["build_request"]) + sameInput := textValue(projection["input_hash"]) == digest + if sameInput && projection["version"] == knowledgeProjectionVersion && !explicitBuild { + _, err = db.Exec("DELETE FROM state WHERE key=?", boundaryKey) + return err + } + needsLearning := !sameInput || (explicitBuild && hasPendingRefinement(learning)) if !needsLearning && projection["version"] == knowledgeProjectionVersion { - _, err = db.Exec("DELETE FROM state WHERE key='organization_boundary'") + projection["build_request"] = request + if err = saveState(db, "projection", projection); err != nil { + return err + } + _, err = db.Exec("DELETE FROM state WHERE key=?", boundaryKey) return err } if needsLearning { @@ -142,7 +180,11 @@ func (v *Vault) compile(ctx context.Context, key string, force bool) (err error) if err != nil { return err } - return v.commitPublication(published, learning, registry, digest) + if err = v.commitPublication(published, learning, registry, digest, request); err != nil { + return err + } + _, err = db.Exec("DELETE FROM state WHERE key=?", boundaryKey) + return err } func (v *Vault) compilerStatus(db *sql.DB, model, message string, needsUpdate bool) error { var cost float64 @@ -170,7 +212,8 @@ func (v *Vault) compilePending(db *sql.DB, partition string, model compilerModel previous = repaired revision++ } - rows, err := db.Query("SELECT e.id,e.payload,e.arrived,e.status,coalesce(w.length,0) FROM events e LEFT JOIN source_windows w ON w.partition=e.partition AND w.id=e.id WHERE e.partition=? AND e.rowid<=? AND e.status IN ('pending','revisit') ORDER BY e.arrived,e.id LIMIT 64", partition, through) + selection, args := compilerSourceSelection(model.ctx, "e.payload") + rows, err := db.Query("SELECT e.id,e.payload,e.arrived,e.status,coalesce(w.length,0) FROM events e LEFT JOIN source_windows w ON w.partition=e.partition AND w.id=e.id WHERE e.partition=? AND e.rowid<=? AND e.status IN ('pending','revisit') AND ("+selection+") ORDER BY e.arrived,e.id LIMIT 64", append([]any{partition, through}, args...)...) if err != nil { return nil, err } @@ -192,6 +235,7 @@ func (v *Vault) compilePending(db *sql.DB, partition string, model compilerModel rows.Close() return nil, err } + delete(source, "job") n := utf8.RuneCountInString(textValue(source["text"])) if windowLength > 0 { n = windowLength @@ -262,3 +306,30 @@ func (v *Vault) compilePending(db *sql.DB, partition string, model compilerModel } return vault, nil } + +func pendingCompilerSources(db *sql.DB, ctx context.Context, through int64) (int, error) { + var pending int + selection, args := compilerSourceSelection(ctx, "payload") + err := db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND rowid<=? AND status IN ('pending','revisit') AND ("+selection+")", append([]any{through}, args...)...).Scan(&pending) + return pending, err +} + +func compilerSourceSelection(ctx context.Context, payload string) (string, []any) { + owners, bound := ctx.Value(ownerSessionsKey{}).([]string) + if !bound && provider.SessionFrom(ctx) != "" { + owners = []string{provider.SessionFrom(ctx)} + } + if bound && len(owners) == 0 { + return "0", nil + } + if len(owners) == 0 { + return "coalesce(json_extract(" + payload + ",'$.job'),0)=0", nil + } + clauses := make([]string, 0, len(owners)) + args := make([]any, 0, len(owners)*3) + for _, owner := range owners { + clauses = append(clauses, "(json_extract("+payload+",'$.session')=? OR substr(json_extract("+payload+",'$.session'),1,length(?))=?)") + args = append(args, owner, "task:"+owner+":", "task:"+owner+":") + } + return strings.Join(clauses, " OR "), args +} diff --git a/internal/knowledge/compiler_batch_regression_test.go b/internal/knowledge/compiler_batch_regression_test.go new file mode 100644 index 0000000000..e159191ccf --- /dev/null +++ b/internal/knowledge/compiler_batch_regression_test.go @@ -0,0 +1,42 @@ +package knowledge_test + +import ( + "context" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/knowledge" +) + +func TestKnowledgeFlushKeepsNextUnownedBatchPending(t *testing.T) { + v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } + for _, source := range []knowledge.Source{ + {ID: "first", Session: "chat", Role: "user", TS: "2026-10-25T12:00:00Z", Text: strings.Repeat("a", 12001)}, + {ID: "next", Session: "chat", Role: "user", TS: "2026-10-25T12:00:01Z", Text: strings.Repeat("b", 12001)}, + } { + if err := v.Capture(source); err != nil { + t.Fatal(err) + } + } + calls := 0 + withKnowledgeModel(t, func(p map[string]any) map[string]any { + if p["messages"] != nil { + calls++ + return map[string]any{"claims": []any{}} + } + return map[string]any{"subjects": []any{}} + }) + if err := v.Flush(context.Background(), "fixture", true); err != nil { + t.Fatal(err) + } + status, err := v.Status() + if err != nil { + t.Fatal(err) + } + if calls != 1 || status.Pending != 1 { + t.Fatalf("Flush consumed another bounded batch: extraction calls=%d pending=%d; want 1/1", calls, status.Pending) + } +} diff --git a/internal/knowledge/compiler_provider.go b/internal/knowledge/compiler_provider.go index 108c8cd3a2..b5e38a93cf 100644 --- a/internal/knowledge/compiler_provider.go +++ b/internal/knowledge/compiler_provider.go @@ -33,6 +33,7 @@ type compilerModel struct { ctx context.Context root, key, model string db *sql.DB + calls func(context.Context) (context.Context, error) } func (m compilerModel) call(role string, payload document) (document, error) { @@ -52,11 +53,26 @@ func (m compilerModel) call(role string, payload document) (document, error) { if err != nil { return nil, err } + ctx := m.ctx + if m.calls != nil { + ctx, err = m.calls(ctx) + if err != nil { + return nil, err + } + } + if err = ctx.Err(); err != nil { + return nil, err + } sources := modelsource.Set{}.OrDefault(m.key, config.DefaultBaseURL) settings := config.ClientConfigFor(sources, m.model) settings.Timeout = 240 * time.Second settings.HTTPClient = http.DefaultClient settings.RouteGate = func(ctx context.Context, model, tag string) error { + if m.calls != nil { + if _, err := m.calls(ctx); err != nil { + return err + } + } if model != m.model { return fmt.Errorf("knowledge requested model %q; refusing fallback %q", m.model, model) } @@ -67,10 +83,11 @@ func (m compilerModel) call(role string, payload document) (document, error) { return nil, err } served := &provider.ServedEndpoint{} - ctx := provider.WithRole(m.ctx, lanes.RoleMemory) + ctx = provider.WithRole(ctx, lanes.RoleMemory) + ctx = memoryBilling(ctx) ctx = provider.WithCacheKey(ctx, "knowledge:"+identity([]any{m.root, m.model, role})) ctx = provider.WithServedEndpoint(ctx, served) - ctx = provider.WithCallTag(ctx, "memory") + ctx = provider.WithCallTag(ctx, string(lanes.RoleMemory)) ctx = provider.WithRequiredReasoningEffort(ctx, provider.Effort("none")) messages := []ai.Message{{Role: "system", Content: []ai.ContentPart{{Type: "text", Text: prompt}}}, {Role: "user", Content: []ai.ContentPart{{Type: "text", Text: mustJSON(wire.payload)}}}} started := time.Now() diff --git a/internal/knowledge/knowledge.go b/internal/knowledge/knowledge.go index dea7d954da..ad866c4a65 100644 --- a/internal/knowledge/knowledge.go +++ b/internal/knowledge/knowledge.go @@ -28,6 +28,7 @@ type Source struct { TS string `json:"ts"` Text string `json:"text"` Kind string `json:"kind"` + Job bool `json:"job,omitempty"` } type Link struct { ID string `json:"id"` @@ -97,7 +98,10 @@ type Result struct { Page Page Snippet string } -type Vault struct{ root, profile string } +type Vault struct { + root, profile string + calls func(context.Context) (context.Context, error) +} func hash(s string) string { sum := sha256.Sum256([]byte(s)); return hex.EncodeToString(sum[:]) } func Open(profile string) *Vault { diff --git a/internal/knowledge/refinement_concurrency_test.go b/internal/knowledge/refinement_concurrency_test.go index be0bc59923..977f18d553 100644 --- a/internal/knowledge/refinement_concurrency_test.go +++ b/internal/knowledge/refinement_concurrency_test.go @@ -144,6 +144,10 @@ func TestKnowledgeRefinementOverlapsTwoJobsAndPublishesTogether(t *testing.T) { t.Fatalf("completed %s was not published: %v", scope, refreshed) } } + // Only an explicit Build may advance unfinished refinement without new input. + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } if err := v.Flush(context.Background(), "fixture", true); err != nil { t.Fatal(err) } diff --git a/internal/knowledge/store.go b/internal/knowledge/store.go index 5c83485010..b106343f8d 100644 --- a/internal/knowledge/store.go +++ b/internal/knowledge/store.go @@ -273,13 +273,27 @@ func (v *Vault) captureTx(tx *sql.Tx, s Source) error { } } source := document{"id": part.ID, "session": part.Session, "role": part.Role, "ts": part.TS, "text": part.Text, "provenance": document{"kind": part.Kind, "content_hash": identity(part.Text), "recorded_at": part.TS, "valid_at": part.TS, "locator": document{"session": part.Session, "message": part.ID, "project": part.Project}, "revision": nil, "independently_verified": false}} + if part.Job { + source["job"] = true + } var raw string err := tx.QueryRow("SELECT payload FROM events WHERE partition='profile' AND id=?", part.ID).Scan(&raw) if err == nil { var prior document - if json.Unmarshal([]byte(raw), &prior) != nil || identity(prior) != identity(source) { + if json.Unmarshal([]byte(raw), &prior) != nil { + return errors.New("knowledge immutable source changed") + } + priorEvidence, evidence := clone(prior), clone(source) + delete(priorEvidence, "job") + delete(evidence, "job") + if identity(priorEvidence) != identity(evidence) { return errors.New("knowledge immutable source changed") } + if part.Job && !flag(prior["job"]) { + if _, err = tx.Exec("UPDATE events SET payload=? WHERE partition='profile' AND id=?", mustJSON(source), part.ID); err != nil { + return err + } + } continue } if err != sql.ErrNoRows { @@ -544,7 +558,7 @@ func storePages(tx *sql.Tx, pages []Page) error { } return nil } -func (v *Vault) commitPublication(vault, learning, registry document, digest string) error { +func (v *Vault) commitPublication(vault, learning, registry document, digest, request string) error { if err := validatePublication(vault); err != nil { return err } @@ -593,10 +607,7 @@ func (v *Vault) commitPublication(vault, learning, registry document, digest str if err = saveState(tx, "topics", registry); err != nil { return err } - if err = saveState(tx, "projection", document{"input_hash": digest, "version": knowledgeProjectionVersion}); err != nil { - return err - } - if _, err = tx.Exec("DELETE FROM state WHERE key='organization_boundary'"); err != nil { + if err = saveState(tx, "projection", document{"input_hash": digest, "version": knowledgeProjectionVersion, "build_request": request}); err != nil { return err } return tx.Commit() diff --git a/internal/manual/chat/commands.md b/internal/manual/chat/commands.md index fbb20c4fa2..6469e069f0 100644 --- a/internal/manual/chat/commands.md +++ b/internal/manual/chat/commands.md @@ -1955,8 +1955,8 @@ Settings now groups controls by what you want to change: `General · Models · Memory · Tasks · AI teams · Permissions · Spending · Connections · Devices · Privacy` General holds appearance, input and updates. Models holds model selection and -advanced routing/context controls. Memory holds `notice things to remember` — -whether codeaf may keep a line on its own after a turn you typed — and a link to +advanced routing/context controls. Memory holds `learn from chats and jobs` — +automatic memory learning, off by default, with its extra model-call cost — and a link to the Memories tab. Tasks controls delegated work; AI teams sets defaults for groups of AI chats. Permissions controls approvals. Spending holds limits. Connections holds accounts, keys, web search and remote connection settings. diff --git a/internal/manual/chat/desktop-sync.md b/internal/manual/chat/desktop-sync.md index a6058aebf5..030bdd0a33 100644 --- a/internal/manual/chat/desktop-sync.md +++ b/internal/manual/chat/desktop-sync.md @@ -228,3 +228,9 @@ send no Devices event when the displayed computer list has not changed. A model choice is applied here only when its provider is set up on this computer; otherwise this computer keeps its own model. Setting up that provider later does not apply an older skipped choice; only a newer choice on the other computer travels. + +## Saved-key sync when the local vault is unreadable + +A removal check returns a local vault read error rather than stopping the +background sync worker. Background sync can retry on a later round. This does +not repair a damaged vault or change the saved keys shared between computers. diff --git a/internal/manual/chat/factory-stage-conversations.md b/internal/manual/chat/factory-stage-conversations.md index ca3d84b6af..50fd3b0e17 100644 --- a/internal/manual/chat/factory-stage-conversations.md +++ b/internal/manual/chat/factory-stage-conversations.md @@ -909,6 +909,14 @@ An accepted “no code change needed” assessment (`repair.decision: no_change` successful outcome: Follow PR returns to watching GitHub for new activity. No code change does not require a new approval. +## Memory charges after a finished step reaches its budget + +Model and tool spending that reaches the job cap still asks for more budget, +including when a step reports completion early. If the step finishes under its +cap and only optional memory crosses it, the finished step stays done. The next +step asks before starting, using the full job spend. On the last step, the job +finishes with its true spend shown and no question for work it will not run. + ## Stop spending after this step in a job's Spend menu **Stop spending after this step** lets the current step finish before you diff --git a/internal/manual/chat/places.md b/internal/manual/chat/places.md index cf70741bb2..a8146b5f80 100644 --- a/internal/manual/chat/places.md +++ b/internal/manual/chat/places.md @@ -648,7 +648,7 @@ These view switches underline while hovered. Build and Search controls highlight under the pointer; the surrounding description stays passive. If saved chats already exist, click **Build knowledge base** or press Enter. -The first build uses saved chats across all projects, including archived chats +Build turns automatic memory on and saves that choice. The first build uses saved chats across all projects, including archived chats and saved task reports. **Building knowledge base… You can keep chatting.** shows that work continues in the background. You can leave Memory, change chats, and return later; pages @@ -672,8 +672,8 @@ and completed publications are saved together. Opening another window does not start a duplicate compiler for the same profile. If there is no chat history, the reader says **Knowledge will populate as you -chat.** Future saved chats populate it automatically, without a separate build -step. A profile has one knowledge base; changing project does not change vaults. +chat.** Automatic learning is off by default. Turn **learn from chats and jobs** +on in Settings or use **Build knowledge base** to populate it from saved chats. A profile has one knowledge base; changing project does not change vaults. ## Reading knowledge pages, links and sources @@ -722,10 +722,27 @@ updates the same page, while a distinct concept can receive a new page. Issue and work-session details remain linked evidence. Organization follows the actual subjects in your chats, without fixed project categories. -The application process owns the background worker. Closing Memory or changing -chats does not stop it. When codeaf stops, captured sources and checkpoints stay +## Memory spending and factory job pauses + +Memory model requests appear in Spending under the `memory` role. Requests +owned by a factory job also count toward that job's spending and budget. + +A memory charge cannot undo a successful step. Completed work stays done; +the next unfinished step checks the job's full spending before it starts. +Delegated task reports feed their owning job's memory pass and keep their citations. +Older job ownership markers still exclude paused and ended jobs from idle learning. + +If a call is interrupted before its price arrives, codeaf asks for the provider's +receipt in the background. Quitting before that recovery finishes can leave the +charge missing from Spending. This limit also applies to ordinary chat calls; +pending receipt recovery does not survive application quit. + +The application process owns the background worker for ordinary saved chats. +Closing Memory or changing chats does not stop it. When codeaf stops, captured sources and checkpoints stay in the database; the next running codeaf process resumes unfinished work. +## Memory background compiler batching and retries + The initial build processes a fixed collection of admitted history before organizing it. Independent page drafts and refreshes can run two at a time; source updates and publication remain ordered. Large concepts stay on one page; @@ -736,15 +753,46 @@ its own step and source-reading budget, so total work grows with the evidence. A final pass keeps the core overview concise by selecting reviewed paragraphs; the complete source detail stays in linked evidence rather than expanding the overview after every update. -Later updates combine saved turns into +## Factory job memory waits, pauses and skipped steps + +Factory jobs wait at most two seconds for memory after a finished turn. An +unfinished pass continues with that job while the next step works. A job has at +most one pass running; later turns do not queue more. Pause, Stop and the job's +end cancel it, including a request in flight. Paid receipts still charge the job. +While a job stays paused, no new paid memory request is admitted. This also +applies to memory from the job's own conversation outside a stage. Resume does +not start a memory burst: pending sources wait for the next finished job turn, +using the same one-pass rule. +The worker can remain occupied briefly after work finishes to report those prices. +Paid memory that finishes after its stage closes stays in that stage’s saved +spending total. Reopening after that memory finishes restores the full paid total. +Quit cancels the pass. The worker allows up to five additional seconds to settle +paid calls through its existing accounting receipt before deleting its local job +copy. Receipts arriving after shutdown are not guaranteed to reach the job; pending +receipt recovery does not survive quit. Saved sources stay pending for its next finished turn, and +idle learning never takes over a job's sources. The next admitted pass also +includes earlier steps skipped while another pass was running. Unchanged source input +reuses completed receipts. Extraction, page organization and evidence review can +require separate model requests in that compilation. Sources and completed +progress survive cancellation for a later active turn. + +Ordinary memory with no owning job belongs to the application process. Closing +one chat does not stop its background build; quitting the application cancels +that build. Its saved sources remain available for recovery on a later launch. + +## Ordinary saved chat memory batching + +Later updates to ordinary saved chats combine turns into normal model requests, without OpenRouter's Batch API. A batch becomes eligible after 30 seconds without new sources, when its oldest source reaches 120 seconds, or when its context limit is reached. These thresholds do not include provider time or retries. Each extraction uses at most 64 source spans and 24,000 source characters. -Search runs against the database index. Failed requests retain pending work and -retry with backoff up to 120 seconds. Unreadable chats do not prevent other chats +Search runs against the database index. Idle background checks do not launch +refinement without pending sources. Failed ordinary-chat requests retain pending +work. The same failed input does not spend again automatically; choose Build +to retry or add new saved sources. Unreadable chats do not prevent other chats from being processed; their incomplete status remains visible. The Knowledge view over `--host` is not supported. @@ -752,9 +800,8 @@ Settings → Models → **knowledge** (`knowledge.model`) selects the OpenRouter model for the next compilation. The default is `deepseek/deepseek-v4.1-flash`. The compiler checks the returned model ID and rejects substitutions. -Failures retain pending work and retry automatically, with backoff up to 120 -seconds. Stable source IDs prevent duplicate records during retry or journal -recovery. A failed or interrupted provider request can still incur charges; +Stable source IDs prevent duplicate records during an explicit retry or +journal recovery. A failed or interrupted provider request can still incur charges; retry safety does not mean every network request is billed only once. ## Updating existing knowledge pages and sections diff --git a/internal/manual/chat/running-on-another-machine.md b/internal/manual/chat/running-on-another-machine.md index 1efffcb938..0836f7f871 100644 --- a/internal/manual/chat/running-on-another-machine.md +++ b/internal/manual/chat/running-on-another-machine.md @@ -503,9 +503,10 @@ name the connected machine, and say to change the crew there: The connection's welcome says whether the connected engine has its memory store open, and the memory commands use that engine's store. `/remember` and the Memories tab write there; -`/memories` and `/memory ` read there. Memory has no off switch — the far -machine's `notice things to remember` row decides only whether that engine keeps lines -on its own — so the one refusal is a store that would not open: +`/memories` and `/memory ` read there. The far machine's +`learn from chats and jobs` row controls automatic learning and defaults off. +Saved lines remain readable and `/remember` still works while off. A store that +would not open says: `memory is unavailable on this machine · the store could not be opened`. ## Does subharness know whether the remote machine has saved programs? diff --git a/internal/manual/chat/settings.md b/internal/manual/chat/settings.md index 8ca259696b..e11ef4a708 100644 --- a/internal/manual/chat/settings.md +++ b/internal/manual/chat/settings.md @@ -30,7 +30,7 @@ not undo the write already in progress. ## Change settings by asking in chat You can ask codeaf to change supported preferences: “hide hints,” “expand completed -tool details,” “turn timestamps off,” or “stop noticing things to remember.” It reads the +tool details,” “turn timestamps off,” or “turn memory on.” It reads the `settings` tool first, then uses `change_setting` to save the requested value through the same registry as the panel. If these tools are not loaded, codeaf loads the `settings` capability group. It does not need to edit config files. @@ -98,20 +98,29 @@ An individual team's own Settings controls its supported overrides. Global defaults do not erase explicit team overrides. New-subteam budget share applies when a subteam is created; it does not redistribute existing budgets. -## Memory settings — notice things to remember, inspect and forget +## Memory settings — turn memory on, why isn't it learning, what does memory cost -**Memory → notice things to remember** decides one thing: whether, after a turn -you typed, codeaf may keep a line you said that should hold in every conversation -— and tell you in the conversation. Off stops only that noticing. Lines kept -with `/remember`, by asking, or on the Memories tab still -ride every conversation, and nothing saved is deleted. It applies from the next -turn, in every open chat, with no restart. +Memory is **off by default**. Turn it on in Settings → Memory: set **learn from +chats and jobs** to **on** (`memory.enabled`). In the desktop app you can also do +it from a Factory job's spend area, under Spent: "Memory is off · Turn on…". This +one switch per home covers noticing useful lines, learning from saved chats, and +job memory, whose calls count in the job's cost. A saved on or off choice is kept. +Terminal **Build knowledge base** also saves on and imports saved chats. -There is no switch that turns memory off altogether: what you kept is yours to -forget, line by line. Use `/memory`, `/memories` or the Memories tab (`alt+8`) to -see every line, and fix, move, forget or put back any of them there. The row's -`Open the Memories tab` door goes straight to it. Input history and unfinished -drafts are separate Privacy choices. +When on, memory learns from your chats and jobs so later work can use it, and +makes extra model calls on your key. On one small test job, memory was **39% of +the cost ($0.0124 of $0.0315)**, measured live on DeepSeek V4.1 Flash; earlier runs +of that same job measured **27%** and **38%**. It varies by job, so read these as +examples and not a forecast for your work. Memory requests appear in Spending. + +Off stops automatic memory model calls from the next memory step, with no restart. +A call already in flight finishes and is billed. Saved knowledge stays readable; +reading it costs nothing. Kept lines and `/remember` still work. Use `/memory` +or `/memories` to inspect, fix or forget saved lines. + +An older home with no saved choice now reads off, even if automatic capture was +previously armed. Nothing saved is deleted. Older terminal Builds with no saved +choice also read off: they cannot be distinguished from automatic opt-ins. ## Permissions, task timers and automatic work @@ -189,4 +198,4 @@ A saved change applies to the next compilation without restarting your chats. A compilation already running finishes with its original model. The compiler verifies the model reported by the provider and rejects substitutions; it never falls back to a different model. Existing notes stay intact when you change this -setting. Knowledge capture must still be enabled from Memory for each project. +setting. Turn automatic learning on with the shared Memory setting. diff --git a/internal/manual/chat/skills-from-other-tools.md b/internal/manual/chat/skills-from-other-tools.md index 3f77f51959..fabbdee5bc 100644 --- a/internal/manual/chat/skills-from-other-tools.md +++ b/internal/manual/chat/skills-from-other-tools.md @@ -71,8 +71,8 @@ is known by its own folder name, not Claude Code's `plugin:skill` spelling. ## Do skills work with memory off -Yes. Turning `memory.enabled` (`notice things to remember`) off stops only -codeaf keeping lines about you on its own; it does not take your skills away, +Yes. Turning `memory.enabled` (`learn from chats and jobs`) off stops automatic +memory model calls; it does not take your skills away, and the shelf stays in the memory database. If that database could not be opened at all, codeaf builds the shelf from the folders for that conversation's process and discards it when the process ends. diff --git a/internal/manual/chat/staying-on-that-machine.md b/internal/manual/chat/staying-on-that-machine.md index 62a4b47637..c2f940976f 100644 --- a/internal/manual/chat/staying-on-that-machine.md +++ b/internal/manual/chat/staying-on-that-machine.md @@ -662,8 +662,9 @@ Over `--host`, these same doors reach the other machine's store. When a machine's memory store could not be opened, the page and the commands say so (`memory is unavailable in this conversation · the store could not be opened` here, `memory is unavailable on this machine · …` from a far engine). The -`notice things to remember` row is read on every turn, so even an engine that -has been running for days follows a change from the next turn. +`learn from chats and jobs` row is read at every memory step, so even an engine +that has been running for days follows a change without a restart. Off stops +automatic memory model calls; saved lines remain available. ## Executable handoff on Windows diff --git a/internal/manual/chat/use-skill.md b/internal/manual/chat/use-skill.md index df23cc81b7..7cf3435558 100644 --- a/internal/manual/chat/use-skill.md +++ b/internal/manual/chat/use-skill.md @@ -39,8 +39,8 @@ opens the one a request fits, even when the request shares no words with it. ## Can you use skills with memory off -Yes. Skills do not depend on the `notice things to remember` setting (the -`memory.enabled` row): turning it off stops only codeaf keeping lines on its own, +Yes. Skills do not depend on the `learn from chats and jobs` setting (the +`memory.enabled` row): turning it off stops automatic memory model calls, and the shelf lives in the same database memory uses either way. Only when that database could not be opened at all does codeaf build a shelf of its own at launch from the skill folders on disk, and throw it away when the diff --git a/internal/manual/chat/what-i-remember.md b/internal/manual/chat/what-i-remember.md index f5c3701a83..0490697a4f 100644 --- a/internal/manual/chat/what-i-remember.md +++ b/internal/manual/chat/what-i-remember.md @@ -244,7 +244,8 @@ earlier build may not know its conversation, and then the card simply says less. ## How does something get remembered without me asking? Noticing -After each turn **you typed**, one small call reads what you said and the answer +With **learn from chats and jobs** on (off by default), after each turn +**you typed**, one small call reads what you said and the answer and may keep **one** line: a preference, a standing rule, a correction, a decision, or a lasting fact. It is shown what is already kept, so it does not keep the same thing twice. It never keeps something only the assistant said, the @@ -260,7 +261,7 @@ If you have moved to another conversation or home by then, the line waits and is drawn in its own conversation when you come back to it. The tab marks it `noticed`, and `→` then `f` (or `delete`) there forgets it. Most -turns keep nothing. Turn noticing off with the `notice things to remember` row +turns keep nothing. Turn noticing off with the `learn from chats and jobs` row (see "How do I turn memory off"). @@ -343,25 +344,17 @@ itself runs locally without a model or embedding call; searching a memory does not require re-adding it. -## How do I turn memory off? The notice things to remember setting +## How do I turn memory off? Stop learning from chats and jobs -`/settings` → Memory → **notice things to remember** governs one thing: whether -codeaf keeps lines on its own after your turns. Off, nothing is noticed. It -applies from the next turn, with no restart. - -It does not stop memory selection or explicit pins from riding, and it does -not stop `/remember` or the Memories tab. To stop a line, forget it there. - -If memory was off before this build, off used to keep your saved lines out of -every prompt; now they ride again. The first conversation after the upgrade says -so once on this machine — `N kept memory lines now ride every conversation they -hold in — memory off now only stops noticing · see or forget them on the -Memories tab` — and nothing about the lines changes. -Saved lines are never deleted by the setting. - -Conversation search and the bash fix hints are separate and keep working either -way (see their sections below). +`/settings` → Memory → **learn from chats and jobs** controls automatic memory +model calls. It is off by default. On learns from chats and jobs at extra cost +on your key; the row explains an example cost. Off applies from the next memory +step without restarting. A call already in flight finishes and is billed. +Saved knowledge remains readable at no cost. Kept lines still ride in prompts, +and `/remember` and the Memories tab still work. Forget a line there to remove it. +Nothing saved is deleted by the setting. Conversation search and bash fix hints +are separate and keep working either way. ## It said memory is off, or memory is unavailable in this conversation diff --git a/internal/session/artifact_append_law_test.go b/internal/session/artifact_append_law_test.go index 0034dc9e25..dea512846b 100644 --- a/internal/session/artifact_append_law_test.go +++ b/internal/session/artifact_append_law_test.go @@ -5,6 +5,7 @@ import ( "go/parser" "go/token" "path/filepath" + "slices" "testing" ) @@ -27,20 +28,20 @@ func TestArtifactIndexHasOneAppendFunnel(t *testing.T) { } func artifactAppendViolations(path string, file *ast.File) (funnels int, violations []token.Pos) { - otherStores := map[string]string{ - "answers.go": "deliverAnswer", - "groundladder.go": "hideFurrowMarker", - "memory.go": "appendMemoryFailureLog", - "memoryledger.go": "appendMemoryRow", - "plandb_plan.go": "TaskGraph.planNote", - "question.go": "Agent.recordDecision", - "renderlog.go": "renderLedger.append", - "sessionfile.go": "openSessionFile", - "task_index.go": "appendTaskIndex", + otherStores := map[string][]string{ + "answers.go": {"deliverAnswer"}, + "groundladder.go": {"hideFurrowMarker"}, + "memory.go": {"appendMemoryFailureLog"}, + "memoryledger.go": {"appendMemoryRow"}, + "plandb_plan.go": {"TaskGraph.planNote"}, + "question.go": {"Agent.recordDecision"}, + "renderlog.go": {"renderLedger.append"}, + "sessionfile.go": {"openSessionFile", "sessionFile.appendLine"}, + "task_index.go": {"appendTaskIndex"}, // A remote task's mirrored money and step events are their own ledgers. - "task_remote_money.go": "appendRemoteUsage", - "task_remote_record.go": "remoteStepMirror.append", - "usage_ledger.go": "openUsageLedger", + "task_remote_money.go": {"appendRemoteUsage"}, + "task_remote_record.go": {"remoteStepMirror.append"}, + "usage_ledger.go": {"openUsageLedger"}, } ast.Inspect(file, func(node ast.Node) bool { flag, ok := node.(*ast.Ident) @@ -50,7 +51,7 @@ func artifactAppendViolations(path string, file *ast.File) (funnels int, violati owner := enclosingFunction(file, flag.Pos()) if path == "artifact_append.go" && owner == "appendArtifactIndex" { funnels++ - } else if owner == "" || otherStores[path] != owner { + } else if owner == "" || !slices.Contains(otherStores[path], owner) { violations = append(violations, flag.Pos()) } return true diff --git a/internal/session/factory_contract.go b/internal/session/factory_contract.go index fc29c15eb5..87985b1ee1 100644 --- a/internal/session/factory_contract.go +++ b/internal/session/factory_contract.go @@ -222,6 +222,12 @@ type JobModelSpend interface { SpendModel(usd float64, model string, calls, input, output int) } +// JobMemorySpend is a job budget that keeps optional learning's portion of the +// job's spend apart, beside the model that answered the call. +type JobMemorySpend interface { + SpendMemoryModel(usd float64, model string, calls, input, output int) +} + // JobChat is one job's chat's half of the job's money. type JobChat interface { // Spend adds usd, one call's dollars, to the job's spend. THE CALL IS diff --git a/internal/session/knowledge.go b/internal/session/knowledge.go index c4ead8753e..5b6070f3c7 100644 --- a/internal/session/knowledge.go +++ b/internal/session/knowledge.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "os" "strings" "sync" "sync/atomic" @@ -57,6 +58,8 @@ func (a *Agent) KnowledgeCaptureError() string { } type knowledgeJournal struct { + ctx context.Context + compile *jobKnowledgeCompile vault *knowledge.Vault project string scope string @@ -77,7 +80,7 @@ func (a *Agent) initKnowledge() { if strings.TrimSpace(a.config.transcriptPath()) == "" || a.config.InTask || a.config.taskID != 0 { return } - a.knowledgeVault = knowledge.Open(config.ProfilePath(a.config.ProfileDir, "")) + a.knowledgeVault = knowledge.Open(config.ProfilePath(a.config.ProfileDir, "")).WithCalls(a.knowledgeCallContext) } // wakeKnowledge is called by the journal writer's hook. It never blocks or @@ -99,11 +102,24 @@ func (a *Agent) startKnowledge() { return } ctx, cancel := context.WithCancel(context.Background()) - j := &knowledgeJournal{cancel: cancel, done: make(chan struct{}), vault: vault, project: a.config.Workspace} + j := &knowledgeJournal{ctx: ctx, cancel: cancel, done: make(chan struct{}), vault: vault, project: a.config.Workspace} a.knowledgeMu.Lock() a.knowledgeJournal = j a.knowledgeMu.Unlock() sessionID := a.id + if a.jobBudgetOf() != nil { + // The marker survives a restart: recovery must not turn a stage into an + // independent source of spending after its job has stopped. + marker := "turn-owned\n" + if named, ok := a.jobBudgetOf().(JobLedgerName); ok { + marker = "job:" + named.FactoryJob() + "\n" + } + if err := os.WriteFile(path+".knowledge-job", []byte(marker), 0600); err != nil { + j.lastError = err.Error() + } + close(j.done) + return + } go func() { defer close(j.done) safety := time.NewTicker(knowledgeSafetyTick) @@ -171,7 +187,7 @@ func (a *Agent) rebindKnowledgeLocked(project string) { old.cancel() a.knowledgeRetired = append(a.knowledgeRetired, old) } - a.knowledgeVault = knowledge.Open(config.ProfilePath(a.config.ProfileDir, "")) + a.knowledgeVault = knowledge.Open(config.ProfilePath(a.config.ProfileDir, "")).WithCalls(a.knowledgeCallContext) a.knowledgeJournal = nil a.knowledgeMu.Unlock() a.startKnowledge() @@ -185,6 +201,9 @@ func (a *Agent) stopKnowledge() { a.knowledgeMu.RUnlock() for _, j := range workers { j.cancel() + if a.config.JobContext == nil { + a.cancelJobKnowledge() + } } timer := time.NewTimer(closeGrace) defer timer.Stop() @@ -262,6 +281,7 @@ func (a *Agent) scanKnowledge(ctx context.Context, j *knowledgeJournal, path, se if stamp.Before(since) { continue } + source.Job = a.jobBudgetOf() != nil if e = j.vault.Capture(source); e != nil { return e } diff --git a/internal/session/knowledge_history.go b/internal/session/knowledge_history.go index 4098d2cea1..8e7640f652 100644 --- a/internal/session/knowledge_history.go +++ b/internal/session/knowledge_history.go @@ -17,11 +17,13 @@ import ( "sync" "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/teams" ) type knowledgeHistory struct { - root string - cache *knowledgeHistoryCache + root string + profile string + cache *knowledgeHistoryCache } type knowledgeHistoryCache struct { @@ -43,10 +45,29 @@ type knowledgeHistoryRecord struct { // KnowledgeHistory reads every saved conversation under the named places root, // across projects and including archives and their saved task reports. -func KnowledgeHistory(placesRoot string) knowledge.History { - return knowledgeHistory{root: placesRoot, cache: &knowledgeHistoryCache{}} +func KnowledgeHistory(placesRoot string, profile ...string) knowledge.History { + h := knowledgeHistory{root: placesRoot, cache: &knowledgeHistoryCache{}} + if len(profile) > 0 { + h.profile = profile[0] + } + return h } func (h knowledgeHistory) Inventory(ctx context.Context) ([]knowledge.Journal, error) { + owned := map[string]bool{} + if h.profile != "" { + roster, _, err := teams.ReadSnapshot(ctx, h.profile) + if err != nil { + return nil, err + } + for _, team := range roster.Teams { + for _, member := range append(append([]teams.Member{}, team.Members...), team.FormerMembers...) { + if member.FactoryStage || (team.FactoryManaged && member.Key == team.Manager) { + owned[filepath.Clean(member.File)] = true + owned[filepath.Clean(member.Key)] = true + } + } + } + } var previous map[string]knowledgeHistoryFile current := make(map[string]knowledgeHistoryFile) if h.cache != nil { @@ -91,11 +112,22 @@ func (h knowledgeHistory) Inventory(ctx context.Context) ([]knowledge.Journal, e journals := []knowledge.Journal{} seen := map[string]bool{} owners := map[string]string{} + jobOwners := map[string]string{} deletedOwners := map[string]bool{} add := func(path, session, project string, task bool) error { if seen[path] { return nil } + owner, marked, err := knowledgeJobOwner(path) + if err != nil { + return err + } + job := owned[filepath.Clean(path)] || marked + if inherited := jobOwners[filepath.Clean(path)]; inherited != "" { + owner, job = inherited, true + } + owned[filepath.Clean(path)] = job + jobOwners[filepath.Clean(path)] = owner info, err := os.Stat(path) if os.IsNotExist(err) { return nil @@ -121,7 +153,7 @@ func (h knowledgeHistory) Inventory(ctx context.Context) ([]knowledge.Journal, e } } seen[path] = true - journals = append(journals, knowledge.Journal{ID: path, Session: session, Project: project, Size: info.Size(), Modified: info.ModTime()}) + journals = append(journals, knowledge.Journal{ID: path, Session: session, Project: project, Size: info.Size(), Modified: info.ModTime(), Job: job, JobOwner: owner}) return nil } addTasks := func(dir, owner, file, project, run string) error { @@ -173,7 +205,14 @@ func (h knowledgeHistory) Inventory(ctx context.Context) ([]knowledge.Journal, e if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jsonl") || (run == "" && (id == "" || deleted[id])) || blocked[path] { continue } + jobOwners[filepath.Clean(path)] = jobOwners[filepath.Clean(owners[owner])] identity := "task:" + owner + ":" + run + ":" + strings.TrimSuffix(entry.Name(), ".jsonl") + if owned[filepath.Clean(owners[owner])] { + owned[filepath.Clean(path)] = true + } + if _, err := os.Stat(owners[owner] + ".knowledge-job"); err == nil { + owned[filepath.Clean(path)] = true + } if err = add(path, identity, project, true); err != nil { return err } @@ -452,3 +491,32 @@ func readKnowledgeLine(reader *bufio.Reader) ([]byte, error) { return line, err } } + +// Both markers are durable ownership facts. The old factory marker identifies +// sibling stages even after their team roster has been replaced. +func knowledgeJobOwner(path string) (owner string, job bool, err error) { + data, err := os.ReadFile(path + ".knowledge-job") + if err == nil { + job = true + owner = path + if strings.HasPrefix(strings.TrimSpace(string(data)), "job:") { + owner = strings.TrimSpace(string(data)) + } + } else if !os.IsNotExist(err) { + return "", false, err + } + data, err = os.ReadFile(filepath.Join(filepath.Dir(path), "factory-owner.json")) + if err == nil { + job = true + var saved struct { + Root string + Job int + } + if json.Unmarshal(data, &saved) == nil && saved.Root != "" && saved.Job > 0 { + owner = fmt.Sprintf("%s:%d", filepath.Clean(saved.Root), saved.Job) + } + } else if !os.IsNotExist(err) { + return "", false, err + } + return owner, job, nil +} diff --git a/internal/session/knowledge_usage.go b/internal/session/knowledge_usage.go new file mode 100644 index 0000000000..989713fcc6 --- /dev/null +++ b/internal/session/knowledge_usage.go @@ -0,0 +1,255 @@ +package session + +import ( + "context" + "errors" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/knowledge" + "github.com/Agent-Field/codeaf/internal/lane" + "github.com/Agent-Field/codeaf/internal/provider" +) + +func init() { + knowledge.SetUsageRecorder(func(ctx context.Context, billed provider.Billed) { + RecordUsage(UsageLedgerPath(), withUsageBilling(UsageLine{ + At: time.Now(), Model: billed.Model, Role: string(lane.RoleMemory), Session: provider.SessionFrom(ctx), + Calls: 1, Input: billed.PromptTokens, Output: billed.CompletionTokens, USD: billed.Cost, + Price: usagePriceProvider, + }, billed.Billing)) + }) +} + +func (a *Agent) knowledgeCallContext(ctx context.Context) (context.Context, error) { + a.mu.Lock() + closed, stopped := a.closed, a.workStopped + a.mu.Unlock() + continuing, _ := ctx.Value(jobKnowledgeContextKey{}).(bool) + if stopped || (closed && (!continuing || a.config.JobContext == nil)) { + return nil, context.Canceled + } + if owner, ok := a.jobBudgetOf().(interface{ MemoryRefusal() string }); ok { + if why := owner.MemoryRefusal(); why != "" { + return nil, errors.New(why) + } + } + if err := a.admitJobCall(ctx); err != nil { + return nil, err + } + if a.config.JobContext != nil && a.config.JobContext.Err() != nil { + return nil, context.Canceled + } + ctx = provider.WithSession(ctx, a.id) + ctx = provider.WithCallNode(ctx, a.id) + bill := func(b provider.Billed) { + used := Usage{Input: b.PromptTokens, Output: b.CompletionTokens, CacheRead: b.CachedTokens, CostUSD: b.Cost, Calls: 1} + a.bank(bankedCall{used: used, model: b.Model, role: string(lane.RoleMemory), ledger: true, late: true, billing: b.Billing, price: usagePriceProvider}) + a.file.appendUsage(used, b.Model, true, string(lane.RoleMemory)) + a.stampSpend() + } + ctx = provider.WithBilling(ctx, bill) + if a.config.JobPendingSpend != nil { + ctx = provider.WithReceiptPending(ctx, a.config.JobPendingSpend) + } + ctx = provider.WithReconcile(ctx, func(r provider.Reconciled) { + if r.Found { + bill(r.Billed) + } else { + a.recordUnbilledReceipt(r.Model) + } + }) + return provider.WithUnmeteredReceipts(ctx), nil +} + +// jobKnowledgeWait limits the part of optional learning a stage waits for. +const jobKnowledgeWait = 2 * time.Second + +type jobKnowledgeContextKey struct{} +type jobKnowledgeCompile struct { + cancel context.CancelFunc + done chan struct{} +} + +var jobKnowledgeCompilers = struct { + sync.Mutex + active map[string]bool +}{active: make(map[string]bool)} + +// A later finished turn leaves its durable journal for the next pass instead +// of stacking another compiler behind the current one. +func (a *Agent) finishJobKnowledge(ctx context.Context) { + if a.jobBudgetOf() == nil || ctx.Err() != nil { + return + } + a.knowledgeMu.RLock() + j := a.knowledgeJournal + a.knowledgeMu.RUnlock() + if j == nil || !j.vault.Enabled() { + return + } + owner, _, err := knowledgeJobOwner(a.file.journalName()) + if err != nil { + return + } + if owner == "" { + owner = a.id + } + key := config.ProfilePath(a.config.ProfileDir, "") + ":" + owner + jobKnowledgeCompilers.Lock() + if jobKnowledgeCompilers.active[key] { + jobKnowledgeCompilers.Unlock() + return + } + jobKnowledgeCompilers.active[key] = true + jobKnowledgeCompilers.Unlock() + parent := j.ctx + if a.config.JobContext != nil { + parent = a.config.JobContext + } + compileCtx, cancel := context.WithCancel(context.WithValue(context.WithoutCancel(ctx), jobKnowledgeContextKey{}, true)) + detach := context.AfterFunc(parent, cancel) + work := &jobKnowledgeCompile{cancel: cancel, done: make(chan struct{})} + j.mu.Lock() + j.compile = work + j.mu.Unlock() + settled := func() {} + if a.config.JobPendingSpend != nil { + settled = a.config.JobPendingSpend() + } + go func() { + defer settled() + defer close(work.done) + defer cancel() + defer detach() + defer func() { + jobKnowledgeCompilers.Lock() + delete(jobKnowledgeCompilers.active, key) + jobKnowledgeCompilers.Unlock() + }() + a.compileJobKnowledge(compileCtx, j) + }() + go func() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-work.done: + return + case <-compileCtx.Done(): + return + case <-ticker.C: + if owner, ok := a.jobBudgetOf().(interface{ MemoryRefusal() string }); ok && owner.MemoryRefusal() != "" { + cancel() + return + } + } + } + }() + timer := time.NewTimer(jobKnowledgeWait) + defer timer.Stop() + select { + case <-work.done: + case <-timer.C: + case <-ctx.Done(): + cancel() + } +} + +func (a *Agent) cancelJobKnowledge() { + a.knowledgeMu.RLock() + j := a.knowledgeJournal + a.knowledgeMu.RUnlock() + if j == nil { + return + } + j.mu.Lock() + work := j.compile + j.mu.Unlock() + if work != nil { + work.cancel() + } +} + +func (a *Agent) compileJobKnowledge(ctx context.Context, j *knowledgeJournal) { + sessions, err := a.captureJobKnowledge(ctx, j) + if err != nil { + return + } + key := a.config.Sources.OrDefault(a.config.APIKey, a.config.BaseURL).Default().Key + ctx = knowledge.WithOwnerSessions(provider.WithSession(ctx, a.id), sessions) + if err := j.vault.Flush(ctx, key, true); err != nil && ctx.Err() == nil { + j.mu.Lock() + j.lastError = err.Error() + j.mu.Unlock() + } +} + +// The next finished turn includes sibling stages skipped during an earlier +// pass. Sources retain their original session IDs and searchable citations. +func (a *Agent) captureJobKnowledge(ctx context.Context, j *knowledgeJournal) ([]string, error) { + if a.config.Place.Dir == "" { + for { + j.mu.Lock() + before := j.offset + j.mu.Unlock() + if err := a.scanKnowledge(ctx, j, a.file.journalName(), a.id, true); err != nil { + return nil, err + } + j.mu.Lock() + after := j.offset + j.mu.Unlock() + if before == after { + break + } + } + return []string{a.id}, nil + } + owner, _, err := knowledgeJobOwner(a.file.journalName()) + if err != nil { + return nil, err + } + history := KnowledgeHistory(filepath.Dir(filepath.Dir(a.config.Place.Dir)), a.config.ProfileDir) + journals, err := history.Inventory(ctx) + if err != nil { + return nil, err + } + sessions := []string{a.id} + _, since := j.vault.ConfirmConsent() + for _, journal := range journals { + if journal.JobOwner != owner || owner == "" { + continue + } + if !strings.HasPrefix(journal.Session, "task:") && journal.Session != a.id { + sessions = append(sessions, journal.Session) + } + cursor := knowledge.JournalCursor{Scope: journal.Project} + for cursor.Offset < journal.Size { + batch, err := history.Read(ctx, journal, cursor) + if err != nil { + return nil, err + } + for _, source := range batch.Sources { + stamp, err := time.Parse(time.RFC3339Nano, source.TS) + if err != nil { + return nil, err + } + if stamp.Before(since) { + continue + } + source.Job = true + if err := j.vault.Capture(source); err != nil { + return nil, err + } + } + if batch.Cursor.Offset <= cursor.Offset { + break + } + cursor = batch.Cursor + } + } + return sessions, nil +} diff --git a/internal/session/loop.go b/internal/session/loop.go index 7cf6cd0788..4023e986a8 100644 --- a/internal/session/loop.go +++ b/internal/session/loop.go @@ -336,7 +336,12 @@ func (a *Agent) turnCue(user userMessage) string { // call, and nobody interrupted and nothing failed. Only that outcome may drain // a follow-up (agent.go) — an interrupted or faulted turn must not be the thing // that starts the next one. -func (a *Agent) runTurn(ctx context.Context, hub *eventHub, user userMessage) bool { +func (a *Agent) runTurn(ctx context.Context, hub *eventHub, user userMessage) (finished bool) { + defer func() { + if finished { + a.finishJobKnowledge(ctx) + } + }() if user.bash == "" { resolved, ready := a.prepareModel(hub) if !ready { @@ -4513,13 +4518,16 @@ func (a *Agent) bank(call bankedCall) { // charge at their own call boundary; their later folded totals do not. if call.used.CostUSD > 0 { if budget := a.jobBudgetOf(); budget != nil && call.ledger { - if split, ok := budget.(JobModelSpend); ok && strings.TrimSpace(call.model) != "" { - calls := call.used.Calls - if calls < 1 { - calls = 1 - } + calls := max(call.used.Calls, 1) + model := strings.TrimSpace(call.model) + memory, isMemory := budget.(JobMemorySpend) + split, hasSplit := budget.(JobModelSpend) + switch { + case isMemory && call.role == string(lanes.RoleMemory): + memory.SpendMemoryModel(call.used.CostUSD, call.model, calls, call.used.Input, call.used.Output) + case hasSplit && model != "": split.SpendModel(call.used.CostUSD, call.model, calls, call.used.Input, call.used.Output) - } else { + default: budget.Spend(call.used.CostUSD) } } diff --git a/internal/session/session.go b/internal/session/session.go index 70e8b020c2..196a892bab 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -2498,6 +2498,14 @@ type Config struct { // Children inherit it without inheriting manager or stage authority. JobBudget JobChat + // JobContext is the owning pipeline's lifetime across stage conversations. + // Optional knowledge may finish after a stage closes, but never after this + // context ends. A standalone job conversation uses its own close lifetime. + JobContext context.Context + + // JobPendingSpend keeps the worker report path open for paid memory receipts. + JobPendingSpend func() func() + // Stage is the door a factory stage's own conversation reports through // (stage_contract.go). NIL IS NO DOOR: `stage_result` and `plan_edit` are off // the belt. Only the factory's runner sets it, on a stage round it opened. diff --git a/internal/session/sessionfile.go b/internal/session/sessionfile.go index 3636aa8af3..8831ecfc48 100644 --- a/internal/session/sessionfile.go +++ b/internal/session/sessionfile.go @@ -3442,7 +3442,8 @@ func (s *sessionFile) writeLine(entry any) bool { // the model exactly as they wrote it. The broader shape scrub belongs to the // records people share — the call log and the debug record — not here. payload = trace.ScrubRegistered(payload) - if !s.appendLine(payload) { + entryLine, _ := entry.(sessionEntry) + if !s.appendLine(payload, entryLine.Type == "usage") { return false } if s.onWrite != nil { @@ -3451,11 +3452,22 @@ func (s *sessionFile) writeLine(entry any) bool { return true } -func (s *sessionFile) appendLine(payload []byte) bool { +func (s *sessionFile) appendLine(payload []byte, usage bool) bool { s.mu.Lock() defer s.mu.Unlock() if s.closed { - return false + if !usage { + return false + } + // Paid work can outlive its stage. Keep its usage in the same journal + // under this file's mutex so reopening restores the full paid total. + file, err := os.OpenFile(s.file.Name(), os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return false + } + defer file.Close() + _, err = file.Write(append(payload, '\n')) + return err == nil } _, err := s.file.Write(append(payload, '\n')) return err == nil @@ -3469,7 +3481,7 @@ func (s *sessionFile) appendLine(payload []byte) bool { // flock on its own — but it states the release at the place a reader looks for // it, and it puts the release before the close rather than as a side effect of // it. Nothing can slip into the gap: closed is already set, so no write reaches -// the file after this point. +// the file after this point, except paid usage appended by path. func (s *sessionFile) Close() error { s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/session/stopwork.go b/internal/session/stopwork.go index 0e142536d9..a6b1e354a0 100644 --- a/internal/session/stopwork.go +++ b/internal/session/stopwork.go @@ -57,6 +57,7 @@ func (a *Agent) stopWorkFor(why string, door StopDoor) error { graph.mu.Unlock() } a.mu.Unlock() + a.cancelJobKnowledge() if memoryStop != nil { memoryStop() } From ceb50829d2fc5898af39f6fece3dc29efafe9c5a Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:52:09 -0500 Subject: [PATCH 16/28] memory: off by default, one switch per home (#2266) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: require explicit memory consent through settings Co-Authored-By: Claude Opus 5.5 * fix: require explicit consent for automatic memory Use memory.enabled for both noticing and knowledge compilation. Empty history no longer grants consent; saved choices remain live across open chats and jobs. Co-Authored-By: Claude Opus 5.5 * test: cover fresh app memory calls through job and chat Co-Authored-By: Claude Opus 5.5 * fix: stop memory requests while keeping saved knowledge available Gate compiler requests and noticing retries on the live setting. Keep saved pages readable while off and retain pending builds across a switch. Prepare explicit-on billing fixtures and a capped live off test. Co-Authored-By: Claude Opus 5.5 * test: require zero new memory requests after switching off Align remaining manual references with the shared learning switch and count request starts exactly. Co-Authored-By: Claude Opus 5.5 * test: verify fresh app leaves knowledge consent unset Co-Authored-By: Claude Opus 5.5 * memory: cost words say it varies, and the manual names the job's switch Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * test: the live memory measure waits for the last brief before reading the header Co-Authored-By: Claude Opus 5.5 * test: fixtures that exercise automatic memory turn it on Co-Authored-By: Claude Opus 5.5 * test: the switch test reads the one cost note, and its job step is plain work Co-Authored-By: Claude Opus 5.5 * memory: a store that can't open says why, not that memory is off The switch makes off a real state, so this refusal must not read as the person's choice. Co-Authored-By: Claude Opus 5.5 * test: a first start with memory off must still be able to join Red: the off pass records its job as unbuilt, and the join check only knows waiting. Co-Authored-By: Claude Opus 5.5 * identity: a memory-off first start counts as a first start A person who starts fresh with memory off could not join an organization, because the home read as holding work. Co-Authored-By: Claude Opus 5.5 * settings: memory's detail says again that nothing saved is deleted Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/app.go | 7 +- cmd/codeaf/appfactory_memory_live_e2e_test.go | 58 +++- cmd/codeaf/appfactory_memory_spend_test.go | 18 ++ cmd/codeaf/appfactory_memory_switch_test.go | 291 ++++++++++++++++++ cmd/codeaf/appsettings.go | 23 +- .../unreleased/2266-memory-off-default.md | 19 ++ internal/appwire/server.go | 1 + internal/appwire/setup.go | 7 + internal/appwire/taskmodels.go | 13 + internal/appwire/wire.go | 27 +- internal/config/settings.go | 70 +++-- internal/config/settings_presentation.go | 2 +- internal/identity/l4fix2_startup_test.go | 41 ++- internal/identity/startup_database.go | 10 +- internal/knowledge/build.go | 28 +- internal/knowledge/compiler_provider.go | 27 +- internal/knowledge/consent_test.go | 17 +- internal/knowledge/edit.go | 3 - internal/knowledge/recall_test.go | 3 + internal/knowledge/store.go | 17 +- internal/manual/memory_settings_test.go | 21 ++ internal/session/knowledge_consent_test.go | 19 +- internal/session/knowledge_history_test.go | 3 + internal/session/knowledge_tools.go | 4 +- internal/session/memory.go | 15 +- internal/session/memory_test.go | 9 +- internal/session/nochange_test.go | 3 + internal/session/stop_auxiliary_test.go | 3 + internal/tui3/knowledge.go | 8 +- 29 files changed, 648 insertions(+), 119 deletions(-) create mode 100644 cmd/codeaf/appfactory_memory_switch_test.go create mode 100644 docs/changes/unreleased/2266-memory-off-default.md create mode 100644 internal/manual/memory_settings_test.go diff --git a/cmd/codeaf/app.go b/cmd/codeaf/app.go index 435f88a5b4..5f21ea7ce1 100644 --- a/cmd/codeaf/app.go +++ b/cmd/codeaf/app.go @@ -179,8 +179,11 @@ func runApp(args []string) error { // woken by its manager must not wait for the window to ask for crews. backend.memoryStore = func() (*store.Store, error) { proc, err := openAppEngineProcess() - if err != nil || proc.Memory == nil { - return nil, errors.New("memory is off") + if err != nil { + return nil, err + } + if proc.Memory == nil { + return nil, errors.New("this engine has no saved memory store") } return proc.Memory, nil } diff --git a/cmd/codeaf/appfactory_memory_live_e2e_test.go b/cmd/codeaf/appfactory_memory_live_e2e_test.go index 5d442a84d8..fcf4d80bae 100644 --- a/cmd/codeaf/appfactory_memory_live_e2e_test.go +++ b/cmd/codeaf/appfactory_memory_live_e2e_test.go @@ -33,14 +33,18 @@ import ( ) func TestLiveMemoryStageMeasure(t *testing.T) { - liveMemoryStageMeasure(t, false) + liveMemoryStageMeasure(t, false, true) } func TestLiveMemoryPauseMeasure(t *testing.T) { - liveMemoryStageMeasure(t, true) + liveMemoryStageMeasure(t, true, true) } -func liveMemoryStageMeasure(t *testing.T, pause bool) { +func TestLiveMemoryOffMakesNoCalls(t *testing.T) { + liveMemoryStageMeasure(t, false, false) +} + +func liveMemoryStageMeasure(t *testing.T, pause, memoryOn bool) { if os.Getenv(liveFactoryOptIn) != "1" { t.Skipf("opt-in live lane: set %s=1 (spends a few cents on OpenRouter)", liveFactoryOptIn) } @@ -56,13 +60,17 @@ func liveMemoryStageMeasure(t *testing.T, pause bool) { t.Setenv(calllog.BodiesEnvVar, "") t.Setenv("HOME", userHome) t.Setenv(home.EnvVar, appHome) - profile := v3Profile(t, map[string]any{ + values := map[string]any{ config.KeyTaskModelsMode: "one", config.KeyTaskModelsOne: liveFactoryModel, "models.roles": "", "models.tiers.low": liveFactoryModel, "models.tiers.high": liveFactoryModel, "models.tiers.reflex": liveFactoryModel, "models.tiers.mastermind": liveFactoryModel, "models.tiers.worker": liveFactoryModel, "task.model": liveFactoryModel, "models.fallbacks": "", config.KeyKnowledgeModel: liveFactoryModel, - }) + } + if memoryOn { + values[config.KeyMemoryEnabled] = config.MemoryOn + } + profile := v3Profile(t, values) t.Setenv(config.ProfileDirEnv, profile) t.Setenv("CODEAF_PROFILE_DIR", profile) // NO KEY IN THE ENVIRONMENT: the only road to one is the app's memory. @@ -149,8 +157,10 @@ func liveMemoryStageMeasure(t *testing.T, pause bool) { t.Fatal(err) } assertFactoryPin(t, launch.Config, liveFactoryModel) - if err := knowledge.Open(profile).StartBuild(); err != nil { - t.Fatal(err) + if memoryOn { + if err := knowledge.Open(profile).StartBuild(); err != nil { + t.Fatal(err) + } } stageSettle = 300 * time.Millisecond @@ -285,6 +295,7 @@ func liveMemoryStageMeasure(t *testing.T, pause bool) { // Let the real app's final debounced brief enter the measured call log. t.Logf("allowing %s for the final automatic brief", factory.BriefDebounce+time.Second) time.Sleep(factory.BriefDebounce + time.Second) + memoryAwaitQuiet(t, callPath) memoryAwaitReceipts(t, callPath) it, err = floor.Get(id) if err != nil { @@ -298,7 +309,7 @@ func liveMemoryStageMeasure(t *testing.T, pause bool) { lateUSD += cost } receiptMu.Unlock() - memoryMeasureTable(t, callPath, it, header.Cost, lateUSD) + memoryMeasureTable(t, callPath, it, header.Cost, lateUSD, memoryOn) recorder.mu.Lock() defer recorder.mu.Unlock() t.Logf("models requested=%v answered=%v provider_total=%.8f problems=%v", recorder.requests, recorder.replies, recorder.cost, recorder.problems) @@ -309,7 +320,7 @@ func liveMemoryStageMeasure(t *testing.T, pause bool) { // Tables come from the shipped call log, ledger and stage records. They are // printed only; no measurement artifact becomes part of the repository. -func memoryMeasureTable(t *testing.T, path string, it factory.Item, headerUSD, receiptUSD float64) { +func memoryMeasureTable(t *testing.T, path string, it factory.Item, headerUSD, receiptUSD float64, memoryOn bool) { t.Helper() if !session.FlushUsage() { t.Error("usage ledger did not flush") @@ -352,7 +363,7 @@ func memoryMeasureTable(t *testing.T, path string, it factory.Item, headerUSD, r calls = append(calls, row) total++ providerUSD += row.Cost - if row.Tag == "memory" { + if row.Tag == "memory" || row.Tag == "reflex" { memory++ } } @@ -480,13 +491,38 @@ func memoryMeasureTable(t *testing.T, path string, it factory.Item, headerUSD, r if providerUSD >= 1 || ledgerUSD >= 1 || headerUSD >= 1 { t.Error("live run reached $1") } - if memory == 0 { + if memoryOn && memory == 0 { t.Error("no memory call measured") } + if !memoryOn && memory != 0 { + t.Errorf("memory off made %d calls", memory) + } } // Receipt recovery is measured after the job, outside every stage-tail interval. // The live fixture does not turn the two-second user wait into a receipt wait. +// memoryAwaitQuiet waits until the call log has stopped growing, so a brief +// that started just before the debounce ended has finished before the header +// is read. +func memoryAwaitQuiet(t *testing.T, path string) { + t.Helper() + const quiet = 8 * time.Second + size, since := int64(-1), time.Now() + for deadline := time.Now().Add(time.Minute); time.Now().Before(deadline); time.Sleep(250 * time.Millisecond) { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Size() != size { + size, since = info.Size(), time.Now() + } + if time.Since(since) >= quiet { + return + } + } + t.Error("the call log kept growing after the job ended") +} + func memoryAwaitReceipts(t *testing.T, path string) { t.Helper() deadline := time.Now().Add(provider.ReceiptWait) diff --git a/cmd/codeaf/appfactory_memory_spend_test.go b/cmd/codeaf/appfactory_memory_spend_test.go index 3cb3437151..962d828d50 100644 --- a/cmd/codeaf/appfactory_memory_spend_test.go +++ b/cmd/codeaf/appfactory_memory_spend_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/factory" "github.com/Agent-Field/codeaf/internal/factory/api" "net/http" @@ -125,6 +126,23 @@ func TestMemorySpendAndLifecycle(t *testing.T) { if err != nil || len(hits) != 1 { t.Fatalf("memory output lost: %v %v", hits, err) } + offProfile := filepath.Join(s.home, "profile") + if err := config.WriteMemoryEnabled(offProfile, false); err != nil { + t.Fatal(err) + } + saved, err := vault.Snapshot() + if err != nil || saved.Enabled || len(saved.Pages) != 1 { + t.Fatalf("off hid saved knowledge: %+v %v", saved, err) + } + if _, err := vault.Read(hits[0].Page.ID); err != nil { + t.Fatal(err) + } + if offHits, err := vault.Search("queues", 5); err != nil || len(offHits) != 1 { + t.Fatalf("off search lost knowledge: %v %v", offHits, err) + } + if err := config.WriteMemoryEnabled(offProfile, true); err != nil { + t.Fatal(err) + } before := calls.Load() if err := vault.Flush(context.Background(), "fixture", true); err != nil { t.Fatal(err) diff --git a/cmd/codeaf/appfactory_memory_switch_test.go b/cmd/codeaf/appfactory_memory_switch_test.go new file mode 100644 index 0000000000..e07bd081b6 --- /dev/null +++ b/cmd/codeaf/appfactory_memory_switch_test.go @@ -0,0 +1,291 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/Agent-Field/codeaf/internal/calllog" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/pipeline" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/store" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/knowledge" +) + +func TestMemorySwitchDefaultsOffAndChangesLive(t *testing.T) { + t.Run("fresh app job and chat", freshMemoryOffApp) + t.Run("saved words and booleans", func(t *testing.T) { + for _, value := range []any{"on", "off", true, false} { + profile := v3Profile(t, map[string]any{config.KeyMemoryEnabled: value}) + want := value == "on" || value == true + got, saved := config.MemorySettingAt(profile) + if got != want || !saved || knowledge.Open(profile).Enabled() != want { + t.Fatalf("saved %v = (%v,%v), want (%v,true)", value, got, saved, want) + } + } + }) + t.Run("legacy consent without saved choice", func(t *testing.T) { + profile := t.TempDir() + vault := knowledge.Open(profile) + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(config.BudgetConfigPath(profile), []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + state, err := vault.Snapshot() + if err != nil { + t.Fatal(err) + } + if vault.Enabled() || state.Enabled { + t.Fatal("legacy vault consent enabled memory without a saved choice") + } + }) + s := newLocalSetup(t) + s.backend.emit = s.server + profile := config.ProfileDir() + vault := knowledge.Open(profile) + if config.MemoryEnabledAt(profile) || vault.Enabled() { + t.Error("fresh home enables automatic memory without consent") + } + read := func(method string, params any, enabled bool, source string) { + t.Helper() + var reply struct { + Settings struct { + Memory *struct { + Enabled bool `json:"enabled"` + Source string `json:"source"` + CostNote string `json:"cost_note"` + } `json:"memory"` + } `json:"settings"` + } + s.must(method, params, &reply) + m := reply.Settings.Memory + if m == nil || m.Enabled != enabled || m.Source != source || m.CostNote != config.MemoryCostNote { + t.Fatalf("%s memory = %+v, want enabled=%v source=%s and cost disclosure", method, m, enabled, source) + } + } + read("settings.get", struct{}{}, false, "default") + if !strings.Contains(s.hello, "settings.memory") { + t.Error("hello omits settings.memory") + } + read("settings.set", map[string]any{"memory_enabled": true}, true, "saved") + if !config.MemoryEnabledAt(profile) || !vault.Enabled() { + t.Fatal("on did not reach the existing vault") + } + read("settings.set", map[string]any{"memory_enabled": false}, false, "saved") + if config.MemoryEnabledAt(profile) || vault.Enabled() { + t.Fatal("off did not reach the existing vault") + } + for _, bad := range []any{"on", 1, nil} { + s.mustFail("settings.set", map[string]any{"memory_enabled": bad}, appwire.CodeBadParams) + } + if err := vault.Flush(context.Background(), "fixture", false); err != knowledge.ErrDisabled { + t.Fatalf("off compile = %v, want disabled", err) + } + if err := vault.StartBuild(); err != nil { + t.Fatal(err) + } + read("settings.get", struct{}{}, true, "saved") + s.mu.Lock() + defer s.mu.Unlock() + found := false + for _, line := range s.lines { + if strings.Contains(line, "settings.changed") && strings.Contains(line, `"memory"`) { + found = true + } + } + if !found { + t.Error("settings.changed omits memory") + } +} + +func freshMemoryOffApp(t *testing.T) { + s := newLocalSetup(t) + var posts atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + fmt.Fprint(w, `{"data":[]}`) + return + } + posts.Add(1) + var body struct { + Tools []struct { + Function struct{ Name string } `json:"function"` + } `json:"tools"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + return + } + delta := map[string]any{"content": "Cedar queues jobs."} + finish := "stop" + for _, tool := range body.Tools { + if tool.Function.Name == "stage_result" { + delta = map[string]any{"tool_calls": []any{map[string]any{"index": 0, "id": "report", "type": "function", "function": map[string]any{"name": "stage_result", "arguments": `{"done":true,"output":"Cedar queues jobs."}`}}}} + finish = "tool_calls" + } + } + raw, _ := json.Marshal(map[string]any{"model": knowledge.Model, "choices": []any{map[string]any{"delta": delta, "finish_reason": finish}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + })) + t.Cleanup(server.Close) + originalClient := http.DefaultClient + http.DefaultClient = &http.Client{Transport: roundTripper(func(r *http.Request) (*http.Response, error) { + clone := r.Clone(r.Context()) + u := *r.URL + clone.URL = &u + if strings.Contains(u.Host, "openrouter.ai") { + clone.URL.Scheme = "http" + clone.URL.Host = server.Listener.Addr().String() + } + return http.DefaultTransport.RoundTrip(clone) + })} + t.Cleanup(func() { http.DefaultClient = originalClient }) + profile := v3Profile(t, map[string]any{config.KeyTaskModelsMode: "one", config.KeyTaskModelsOne: knowledge.Model, "models.tiers.low": knowledge.Model, "models.tiers.reflex": knowledge.Model, "models.roles": ""}) + t.Setenv(config.ProfileDirEnv, profile) + t.Setenv("OPENROUTER_API_KEY", "fixture") + t.Setenv("CODEAF_BASE_URL", server.URL) + t.Setenv("CODEAF_MODEL_POOL", "off") + callPath := filepath.Join(t.TempDir(), "calls.jsonl") + t.Setenv(calllog.EnvVar, callPath) + freshEngineProcess(t) + g := newRunRig(t) + proc, err := openV3Process("chat") + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.closeAll) + launch, err := openV3Launch(proc, v3Options{Model: knowledge.Model, OneModel: true, Workspace: g.workspace}) + if err != nil { + t.Fatal(err) + } + stageSettle = 20 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + s.backend.open = appOpenInProcess + local := &s.backend.local + local.parent = func() session.Config { return launch.Config } + local.rules = func(context.Context, string, string) (pipeline.File, error) { return pipeline.File{}, os.ErrNotExist } + local.seamHook = func(seam factory.Seam) factory.Seam { + seam.GitHub = func(context.Context) (factory.GitHubLink, error) { return factory.GitHubLink{Login: "owner"}, nil } + return seam + } + st := s.start() + floor := local.host.floor + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + id, err := floor.Add(context.Background(), factory.Item{Repo: "acme/api", Title: "Cedar job", Cap: 1, Tier: factory.TierOwner, Model: knowledge.Model, Stages: []factory.Stage{{Name: "work", Kind: factory.StageChat, On: true, Until: "done", Ask: "Call stage_result with done true and output Cedar queues jobs."}, {Name: factory.ApproveName, Kind: factory.StageGate, On: true}}}) + if err != nil { + t.Fatal(err) + } + var job api.Job + s.must("factory.job", map[string]any{"factory": st.Factory, "job": id}, &job) + s.must("factory.job_start", map[string]any{"factory": st.Factory, "job": id, "expect": map[string]any{"version": job.Version}}, &appwire.FactoryJobActed{}) + deadline := time.Now().Add(15 * time.Second) + for { + it, err := floor.Get(id) + if err != nil { + t.Fatal(err) + } + if it.State == factory.StateLanded { + break + } + if it.Failure != nil || it.Question != "" || time.Now().After(deadline) { + t.Fatalf("job did not finish: %+v", it) + } + time.Sleep(10 * time.Millisecond) + } + agent, err := session.New(launch.Config) + if err != nil { + t.Fatal(err) + } + events, err := agent.Submit(context.Background(), "Cedar queues my jobs.") + if err != nil { + t.Fatal(err) + } + for range events { + } + t.Cleanup(func() { _ = agent.Close() }) + raw, err := os.ReadFile(callPath) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(raw), "\n") { + var row calllog.Record + if json.Unmarshal([]byte(line), &row) == nil && (row.Tag == "memory" || row.Tag == "reflex") { + t.Errorf("fresh app made memory call: %s", line) + } + } + if posts.Load() < 2 { + t.Fatalf("job and chat did not call stub: %d", posts.Load()) + } + vault := knowledge.Open(profile) + if vault.Enabled() || !vault.EnabledSince().IsZero() { + t.Error("background worker armed fresh memory or wrote enabled consent") + } + s.must("settings.set", map[string]any{"memory_enabled": true}, nil) + events, err = agent.Submit(context.Background(), "Remember that I prefer Cedar queues.") + if err != nil { + t.Fatal(err) + } + for range events { + } + deadline = time.Now().Add(5 * time.Second) + for { + raw, _ = os.ReadFile(callPath) + if strings.Contains(string(raw), `"tag":"reflex"`) { + break + } + if time.Now().After(deadline) { + t.Fatal("wire on did not start memory in existing chat") + } + time.Sleep(10 * time.Millisecond) + } + s.must("settings.set", map[string]any{"memory_enabled": false}, nil) + // Let the already-started request finish before measuring the next turn. + time.Sleep(50 * time.Millisecond) + raw, err = os.ReadFile(callPath) + if err != nil { + t.Fatal(err) + } + before := strings.Count(string(raw), `"phase":"start","tag":"reflex"`) + events, err = agent.Submit(context.Background(), "Explain Cedar again.") + if err != nil { + t.Fatal(err) + } + for range events { + } + if err := agent.Close(); err != nil { + t.Fatal(err) + } + raw, _ = os.ReadFile(callPath) + if after := strings.Count(string(raw), `"phase":"start","tag":"reflex"`); after != before { + t.Fatalf("wire off admitted another memory request: before=%d after=%d", before, after) + } + +} + +// The switch makes off a real state, so a store that can't open must say why +// and never read as the person's choice. +func TestMemoryStoreFailureNamesItsReasonNotOff(t *testing.T) { + b := &appBackend{memoryStore: func() (*store.Store, error) { return nil, errors.New("no provider is set up") }} + _, err := b.memoryOf() + if err == nil || strings.Contains(err.Error(), "memory is off") || !strings.Contains(err.Error(), "no provider is set up") { + t.Fatalf("memoryOf error = %v", err) + } +} diff --git a/cmd/codeaf/appsettings.go b/cmd/codeaf/appsettings.go index 8865af6e57..8970b0a627 100644 --- a/cmd/codeaf/appsettings.go +++ b/cmd/codeaf/appsettings.go @@ -63,6 +63,12 @@ func tierShown(value string) string { // settingsExtra fills the wave 8 rows of settings. func settingsExtra(settings *appwire.Settings, file appStateFile, profile string) { + enabled, saved := config.MemorySettingAt(profile) + source := "default" + if saved { + source = "saved" + } + settings.Memory = appwire.MemorySettings{Enabled: enabled, Source: source, CostNote: config.MemoryCostNote} settings.Appearance, settings.TextSize = appwire.AppearanceSystem, appwire.TextMedium if file.Appearance != "" { settings.Appearance = file.Appearance @@ -84,6 +90,11 @@ func settingsExtra(settings *appwire.Settings, file appStateFile, profile string // applyExtra writes the wave 8 rows a patch names: the profile's rows first, // because they can refuse, then the app's own. func (b *appBackend) applyExtra(patch appwire.SettingsPatch, profile string) error { + if patch.MemoryEnabled != nil { + if err := config.WriteMemoryEnabled(profile, *patch.MemoryEnabled); err != nil { + return appEngineError(err) + } + } for _, row := range []struct { key string value *string @@ -254,11 +265,14 @@ func (b *appBackend) ForgetMemory(id string) error { // memoryOf is the memory store, or the refusal that says there is none. func (b *appBackend) memoryOf() (*store.Store, error) { if b.memoryStore == nil { - return nil, appwire.Errorf(appwire.CodeEngine, "memory is off") + return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read: this engine has no saved memory store") } brain, err := b.memoryStore() - if err != nil || brain == nil { - return nil, appwire.Errorf(appwire.CodeEngine, "memory is off") + if err != nil { + return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read: %v", err) + } + if brain == nil { + return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read: this engine has no saved memory store") } return brain, nil } @@ -296,3 +310,6 @@ func (b *appBackend) ExportSpend(from, to time.Time) (string, error) { } return path, nil } + +// MemorySettings advertises the live memory switch on the app wire. +func (b *appBackend) MemorySettings() {} diff --git a/docs/changes/unreleased/2266-memory-off-default.md b/docs/changes/unreleased/2266-memory-off-default.md new file mode 100644 index 0000000000..44f24944f6 --- /dev/null +++ b/docs/changes/unreleased/2266-memory-off-default.md @@ -0,0 +1,19 @@ +--- +kind: changed +title: codeaf asks before learning from chats and jobs +pr: 2266 +surface: [engine, chat] +invalidates: + - "Fresh homes automatically armed capture, while memory.enabled governed only noticing and defaulted on." +--- + +Memory is off by default. One live setting controls automatic learning from +chats, history and jobs, with its extra model-call cost explained in Settings. +Explicit saved on and off choices are kept. Terminal Build saves on. Saved +knowledge, kept lines and /remember remain available while off; reading saved +knowledge costs nothing. A call already in flight finishes and is billed. + +Homes automatically armed by the old empty-history path, without a saved memory +choice, now read off. No saved knowledge is deleted. Older homes that used +terminal Build without a saved choice also read off, because those two cases +cannot be distinguished. Turn memory on in Settings or use Build to opt in. diff --git a/internal/appwire/server.go b/internal/appwire/server.go index b520e64fdf..1c0e967d35 100644 --- a/internal/appwire/server.go +++ b/internal/appwire/server.go @@ -214,6 +214,7 @@ func (s *Server) Serve(ctx context.Context, in io.Reader, hello Hello, engine En hello = withUnreadFeature(hello, engine) hello = withErrandFeature(hello, engine) hello = withTaskModelsAuxiliaryFeature(hello, engine) + hello = withMemorySettingsFeature(hello, engine) hello = withTaskModelsInsightFeature(hello, engine) hello = withProcessFeatures(hello, engine) hello = withProcessActionFeature(hello, engine) diff --git a/internal/appwire/setup.go b/internal/appwire/setup.go index 767f902046..def16cd30a 100644 --- a/internal/appwire/setup.go +++ b/internal/appwire/setup.go @@ -198,6 +198,7 @@ type ProviderParams struct { // optional, and a field present is changed. A money field sent as null means // none. [decodeSettingsPatch] reads exactly these fields and refuses any other. type SettingsPatchWire struct { + MemoryEnabled *bool `json:"memory_enabled,omitempty"` DefaultModel *string `json:"default_model,omitempty"` AsksBefore *string `json:"asks_before,omitempty"` WorkFolder *string `json:"work_folder,omitempty"` @@ -328,6 +329,12 @@ func decodeSettingsPatch(raw json.RawMessage) (SettingsPatch, error) { err = oneOf(name, &patch.TextSize, TextSmall, TextMedium, TextLarge) case "open_at_login": err = flag(name, &patch.OpenAtLogin) + case "memory_enabled": + if strings.TrimSpace(string(fields[name])) == "null" { + err = Errorf(CodeBadParams, "memory_enabled must be true or false") + } else { + err = flag(name, &patch.MemoryEnabled) + } case "dock_badge": err = flag(name, &patch.DockBadge) case "checks_model": diff --git a/internal/appwire/taskmodels.go b/internal/appwire/taskmodels.go index a8f55a543b..3957f58154 100644 --- a/internal/appwire/taskmodels.go +++ b/internal/appwire/taskmodels.go @@ -560,3 +560,16 @@ func decodeTaskModelsPatch(raw json.RawMessage) (TaskModelsPatch, error) { } return patch, nil } + +// FeatureSettingsMemory announces the shared automatic-memory setting. +const FeatureSettingsMemory = "settings.memory" + +// MemorySettingsServer marks an engine that serves the memory setting. +type MemorySettingsServer interface{ MemorySettings() } + +func withMemorySettingsFeature(h Hello, e Engine) Hello { + if _, ok := e.(MemorySettingsServer); !ok { + return h + } + return withFeature(h, FeatureSettingsMemory) +} diff --git a/internal/appwire/wire.go b/internal/appwire/wire.go index ce749f071c..9ae38ace99 100644 --- a/internal/appwire/wire.go +++ b/internal/appwire/wire.go @@ -664,16 +664,24 @@ const ( // WorkFolder is where an agent created with no workspace works; // ProposedTaskClockS is how long a proposed task waits before it starts on its // own; IdleCloseDays is how long an agent nobody touches stays open. +// MemorySettings is the live automatic-learning choice and its cost disclosure. +type MemorySettings struct { + Enabled bool `json:"enabled"` + Source string `json:"source" enum:"default,saved"` + CostNote string `json:"cost_note"` +} + type Settings struct { - DefaultModel string `json:"default_model"` - BudgetUSD *float64 `json:"budget_usd"` - AsksBefore string `json:"asks_before"` - DayCapUSD *float64 `json:"day_cap_usd"` - WorkFolder string `json:"work_folder"` - TaskCapUSD *float64 `json:"task_cap_usd"` - ProposedTaskClockS int `json:"proposed_task_clock_s"` - IdleCloseDays int `json:"idle_close_days"` - Effort string `json:"effort,omitempty" enum:"low,medium,high"` + Memory MemorySettings `json:"memory"` + DefaultModel string `json:"default_model"` + BudgetUSD *float64 `json:"budget_usd"` + AsksBefore string `json:"asks_before"` + DayCapUSD *float64 `json:"day_cap_usd"` + WorkFolder string `json:"work_folder"` + TaskCapUSD *float64 `json:"task_cap_usd"` + ProposedTaskClockS int `json:"proposed_task_clock_s"` + IdleCloseDays int `json:"idle_close_days"` + Effort string `json:"effort,omitempty" enum:"low,medium,high"` // The General rows the window draws by (Wave 8): kept here so every window // reads the same answer. Appearance string `json:"appearance" enum:"system,light,dark"` @@ -697,6 +705,7 @@ type Settings struct { // money figure sent as null is cleared, which the settings that always have a // figure refuse. type SettingsPatch struct { + MemoryEnabled *bool DefaultModel *string BudgetUSD *Figure AsksBefore *string diff --git a/internal/config/settings.go b/internal/config/settings.go index a7dd84bd15..65470ae8c1 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -332,13 +332,8 @@ const ( // as the default, silently, the way any word this build does not know reads // ([TaskStartAt]). KeyTaskStart = "task.start" - // KeyMemoryEnabled is whether codeaf NOTICES things worth keeping on its - // own (internal/session's memory.go): after a turn the person typed, one - // small call may keep one line they said, and says so. It governs nothing - // else — the lines a person kept ride every conversation whether it is on - // or off, and /remember works either way — and it is read live, so a change - // applies from the next turn without a restart. The key keeps its old - // spelling because profiles on disk carry it. + // KeyMemoryEnabled controls automatic learning from chats and jobs. + // Saved knowledge and explicit keeps remain available while it is off. KeyMemoryEnabled = "memory.enabled" KeyModelRoles = "models.roles" @@ -691,14 +686,13 @@ const ( MemoryOn = "on" ) -// MemoryModes lists them, on first — which is the default. A colleague who -// forgot every preference you stated the moment you closed the window would be -// one you had to brief again every morning, and the calls that carry this are -// the cheapest the surface makes. -var MemoryModes = []string{MemoryOn, MemoryOff} +// MemoryModes lists off first, the default until a person opts in. +var MemoryModes = []string{MemoryOff, MemoryOn} -// DefaultMemory is on. -const DefaultMemory = MemoryOn +const DefaultMemory = MemoryOff + +// MemoryCostNote is the cost disclosure shared by settings surfaces. +const MemoryCostNote = "Off by default. When on, it learns from your chats and jobs so later work can use it, and makes extra model calls on your key. On one small test job, memory was 39% of the cost ($0.0124 of $0.0315); it varies by job." // The prompt profile's three answers, and they are three rather than two // because the honest default is not a size at all: it is "work it out". A row @@ -2369,15 +2363,8 @@ func (s *Settings) build() []Setting { }, Setting{ Key: KeyMemoryEnabled, Category: CategoryPractice, Kind: SettingChoice, - Label: "notice things to remember", Choices: MemoryModes, - // THE ONE SPELLING OF WHAT THIS ROW DOES: the settings panel reads it - // as the row's detail and the settings tool reads it as its hint, so - // the two cannot disagree. Off comes first because it is what a - // person scanning the row needs to know before anything else. - Hint: "Off stops only the noticing: what you keep with /remember still rides every " + - "conversation, and nothing saved is deleted. On, after a turn you typed, codeaf may " + - "keep one line you said that should hold everywhere — a preference, a rule, a " + - "correction — and tells you; the Memories tab shows, forgets and puts back any of it.", + Label: "learn from chats and jobs", Choices: MemoryModes, + Hint: MemoryCostNote + " Off stops automatic learning; nothing saved is deleted, and saved knowledge and /remember stay available. A call already in flight finishes and is billed.", read: func() string { return MemoryAt(dir) }, write: func(raw string) error { return writeChoice(dir, KeyMemoryEnabled, raw, MemoryModes) }, }, @@ -3918,14 +3905,38 @@ func TaskStartAt(profileDir string) string { return DefaultTaskStart } -// MemoryAt resolves the memory row to its word, default on. +// MemoryAt resolves the one automatic-memory switch, absent means off. func MemoryAt(profileDir string) string { + enabled, _ := MemorySettingAt(profileDir) + if enabled { + return MemoryOn + } + return MemoryOff +} + +// MemorySettingAt reads saved words and booleans, and whether one was saved. +func MemorySettingAt(profileDir string) (enabled, saved bool) { if value, ok := persistedString(profileDir, KeyMemoryEnabled); ok { - if value = strings.TrimSpace(value); value != "" { - return value + switch strings.TrimSpace(value) { + case MemoryOn: + return true, true + case MemoryOff: + return false, true } } - return DefaultMemory + if value, ok := persistedBool(profileDir, KeyMemoryEnabled); ok { + return value, true + } + return false, false +} + +// WriteMemoryEnabled saves an explicit choice through the shared profile writer. +func WriteMemoryEnabled(profileDir string, enabled bool) error { + word := MemoryOff + if enabled { + word = MemoryOn + } + return writeProfileValue(profileDir, KeyMemoryEnabled, word) } // PromptProfileAt resolves the prompt-profile row to its word: the environment @@ -3955,10 +3966,7 @@ func PromptProfileAt(profileDir string) string { return DefaultPromptProfile } -// MemoryEnabledAt is [MemoryAt] as the bool the v3 door reads before it opens a -// brain at all: memory off is a session handed no store, which is what makes -// "no block and no calls" a property of the wiring rather than a branch every -// caller has to remember (internal/session's memory.go states the law). +// MemoryEnabledAt reads automatic learning live; saved lines remain readable. func MemoryEnabledAt(profileDir string) bool { return MemoryAt(profileDir) == MemoryOn } diff --git a/internal/config/settings_presentation.go b/internal/config/settings_presentation.go index 57b94e83d8..647ff487b3 100644 --- a/internal/config/settings_presentation.go +++ b/internal/config/settings_presentation.go @@ -45,7 +45,7 @@ func (s Setting) ChatPresentation() SettingPresentation { p.Hidden = true // Resident-only controls have no live chat consumer. case KeyMemoryEnabled: // The detail is the row's own Hint (settings.go), not a second copy. - p.Category, p.Label = "Memory", "notice things to remember" + p.Category, p.Label = "Memory", s.Label p.Activation = "Applies from the next turn" case KeyUpdateAuto: p.Label = "automatic updates" diff --git a/internal/identity/l4fix2_startup_test.go b/internal/identity/l4fix2_startup_test.go index 199bc61330..61d058a74a 100644 --- a/internal/identity/l4fix2_startup_test.go +++ b/internal/identity/l4fix2_startup_test.go @@ -3,6 +3,7 @@ package identity_test import ( "context" "database/sql" + "github.com/Agent-Field/codeaf/internal/config" "os" "path/filepath" "testing" @@ -77,7 +78,9 @@ func firstStart(t *testing.T) string { <-done t.Fatal(err) } - if snapshot.Build.Phase == knowledge.BuildWaiting { + // Memory is off on a first start, so its worker records an empty inventory + // as unbuilt and stays that way. + if snapshot.Build.Phase == knowledge.BuildUnbuilt { break } if ctx.Err() != nil { @@ -187,3 +190,39 @@ func TestL4Fix2GraphMemoryRemainsJoinable(t *testing.T) { t.Fatalf("unknown work event: %v %v", ok, err) } } + +// A first start where the person turned memory on is still a first start. +func TestL4Fix2MemoryOnFirstStartCanJoin(t *testing.T) { + home := t.TempDir() + if _, err := identity.EnsureSolo(home); err != nil { + t.Fatal(err) + } + if err := config.WriteMemoryEnabled(home, true); err != nil { + t.Fatal(err) + } + vault := knowledge.Open(home) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done := make(chan error, 1) + guard.Go("identity/startup-fixture-on", func() { + done <- vault.Run(ctx, session.KnowledgeHistory(filepath.Join(home, "v3/projects")), func() string { return "" }) + }) + for { + snapshot, err := vault.Status() + if err != nil { + t.Fatal(err) + } + if snapshot.Build.Phase == knowledge.BuildWaiting { + break + } + if ctx.Err() != nil { + t.Fatal("memory-on worker did not reach waiting", <-done) + } + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + if ok, err := identity.Joinable(home); err != nil || !ok { + t.Fatalf("a first start with memory on cannot join: %v %v", ok, err) + } +} diff --git a/internal/identity/startup_database.go b/internal/identity/startup_database.go index 2a335a527e..5a4696866f 100644 --- a/internal/identity/startup_database.go +++ b/internal/identity/startup_database.go @@ -118,15 +118,19 @@ func automationStartup(path string) (bool, error) { func knowledgeStartup(path string) (bool, error) { out := startupTables("state", "events", "publications", "receipts", "overrides", "pages", "history_errors", "cursors", "proposals", "source_windows", "claim_keys", "proposal_sources", "watch_terms", "watches", "wakes") - // Vault.runPass enables an empty inventory and records its waiting job. No - // other state document, error, source, or person-edited configuration is startup. + // Vault.runPass records a job with nothing in it: unbuilt while memory is off, + // waiting once it is on. No other state document, error, source, or + // person-edited configuration is startup. out["state"] = `NOT coalesce(json_valid(payload) AND ( (key='config' AND json_extract(payload,'$.enabled')=1 AND json_extract(payload,'$.schema')=1 AND NOT EXISTS(SELECT 1 FROM json_each(payload) WHERE key NOT IN ('enabled','schema','created_at'))) OR (key='job' AND json_extract(payload,'$.phase')='waiting' AND json_extract(payload,'$.backfill')=0 AND json_extract(payload,'$.sources')=0 AND json_extract(payload,'$.problems')=0 AND json_extract(payload,'$.error')='' - AND NOT EXISTS(SELECT 1 FROM json_each(payload) WHERE key NOT IN ('backfill','chats','error','imported','inventory_hash','phase','problems','sources')))), 0)` + AND NOT EXISTS(SELECT 1 FROM json_each(payload) WHERE key NOT IN ('backfill','chats','error','imported','inventory_hash','phase','problems','sources'))) + OR (key='job' AND json_extract(payload,'$.phase')='unbuilt' + AND json_extract(payload,'$.chats')=0 AND json_extract(payload,'$.problems')=0 AND json_extract(payload,'$.error')='' + AND NOT EXISTS(SELECT 1 FROM json_each(payload) WHERE key NOT IN ('chats','error','inventory_hash','phase','problems')))), 0)` // Vault.runPass saves offsets even when the history reader admits no sources. // Every source and publication table above must still be empty. out["cursors"] = "skip <> 0 OR offset < 0" diff --git a/internal/knowledge/build.go b/internal/knowledge/build.go index aac241e7e5..b07c57303e 100644 --- a/internal/knowledge/build.go +++ b/internal/knowledge/build.go @@ -8,6 +8,7 @@ import ( "path/filepath" "time" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/filelock" ) @@ -58,6 +59,9 @@ type History interface { } func (v *Vault) StartBuild() error { + if err := config.WriteMemoryEnabled(v.profile, true); err != nil { + return err + } db, err := v.database() if err != nil { return err @@ -177,17 +181,12 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, } } } - settings, err := stateDocument(db, "config") - if err != nil { - return err - } job, err := stateDocument(db, "job") if err != nil { return err } - if !flag(settings["enabled"]) { - // A successful empty inventory may arm future capture. Existing history - // always waits for an explicit Build action before sources enter the vault. + if !v.Enabled() { + // Discovery is local. No inventory, including an empty one, opts in. fingerprint := identity(journals) if textValue(job["inventory_hash"]) == fingerprint { return nil @@ -226,14 +225,9 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, if err = saveState(db, "job", job); err != nil { return err } - if available > 0 || problems > 0 { - return nil - } - if _, err = db.Exec("INSERT OR IGNORE INTO state(key,payload) VALUES('config',?)", mustJSON(document{"enabled": true, "schema": 1, "created_at": time.Now().UTC().Format(time.RFC3339Nano)})); err != nil { - return err - } - v.forgetConsent() + return nil } + imported := int(number(job["imported"])) problems := int(number(job["problems"])) fingerprint := identity(journals) @@ -339,6 +333,9 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, if err = ctx.Err(); err != nil { return err } + if !v.Enabled() { + return nil + } if err = db.QueryRow("SELECT count(*) FROM events WHERE partition='profile' AND status IN ('pending','revisit') AND coalesce(json_extract(payload,'$.job'),0)=0").Scan(&pending); err != nil { return err } @@ -356,6 +353,9 @@ func (v *Vault) runPass(ctx context.Context, history History, key func() string, return err } if err = v.Flush(ctx, key(), flag(job["backfill"])); err != nil { + if errors.Is(err, ErrDisabled) { + return nil + } job["failed_input"] = attemptInput if saveErr := saveState(db, "job", job); saveErr != nil { return saveErr diff --git a/internal/knowledge/compiler_provider.go b/internal/knowledge/compiler_provider.go index b5e38a93cf..66d8ef497f 100644 --- a/internal/knowledge/compiler_provider.go +++ b/internal/knowledge/compiler_provider.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "strings" "time" @@ -53,12 +54,9 @@ func (m compilerModel) call(role string, payload document) (document, error) { if err != nil { return nil, err } - ctx := m.ctx - if m.calls != nil { - ctx, err = m.calls(ctx) - if err != nil { - return nil, err - } + ctx, err := m.callContext(m.ctx) + if err != nil { + return nil, err } if err = ctx.Err(); err != nil { return nil, err @@ -68,10 +66,8 @@ func (m compilerModel) call(role string, payload document) (document, error) { settings.Timeout = 240 * time.Second settings.HTTPClient = http.DefaultClient settings.RouteGate = func(ctx context.Context, model, tag string) error { - if m.calls != nil { - if _, err := m.calls(ctx); err != nil { - return err - } + if _, err := m.callContext(ctx); err != nil { + return err } if model != m.model { return fmt.Errorf("knowledge requested model %q; refusing fallback %q", m.model, model) @@ -184,3 +180,14 @@ func (m compilerModel) cached(role string, payload document) (document, bool, er } return nil, false, nil } + +// callContext gates every memory model step, including recall and provider retries. +func (m compilerModel) callContext(ctx context.Context) (context.Context, error) { + if !config.MemoryEnabledAt(filepath.Dir(m.root)) { + return nil, ErrDisabled + } + if m.calls != nil { + return m.calls(ctx) + } + return ctx, nil +} diff --git a/internal/knowledge/consent_test.go b/internal/knowledge/consent_test.go index e1370aa5bb..668cdc2db6 100644 --- a/internal/knowledge/consent_test.go +++ b/internal/knowledge/consent_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/sqliteuri" ) @@ -82,6 +83,9 @@ func TestCaptureConsentRechecksExternalWritesAndKeepsFreshReaders(t *testing.T) if got := opens.Load(); got != 1 { t.Fatalf("initial consent opened database %d times; want 1", got) } + if err := config.WriteMemoryEnabled(v.profile, true); err != nil { + t.Fatal(err) + } written := now.Add(time.Second) uri := sqliteuri.File(path) db, err := sql.Open("sqlite", uri.String()) @@ -102,8 +106,8 @@ func TestCaptureConsentRechecksExternalWritesAndKeepsFreshReaders(t *testing.T) before := opens.Load() for _, advance := range []time.Duration{0, consentRecheck - time.Nanosecond} { now = now.Add(advance) - if enabled, since := v.CaptureConsent(); enabled || !since.IsZero() { - t.Fatalf("external opt-in visible before recheck: (%v, %v)", enabled, since) + if enabled, since := v.CaptureConsent(); !enabled || !since.Equal(time.Unix(0, 0)) { + t.Fatalf("live setting must enable capture before timestamp refresh: (%v, %v)", enabled, since) } } if got := opens.Load(); got != before { @@ -160,7 +164,7 @@ func (consentEmptyHistory) Read(context.Context, Journal, JournalCursor) (Journa return JournalBatch{}, nil } -func TestCaptureConsentRefreshesWhenEmptyHistoryArmsCapture(t *testing.T) { +func TestCaptureConsentStaysOffWithEmptyHistory(t *testing.T) { v := Open(t.TempDir()) now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) previous := consentNow @@ -174,8 +178,8 @@ func TestCaptureConsentRefreshesWhenEmptyHistoryArmsCapture(t *testing.T) { t.Fatal(err) } enabled, since := v.CaptureConsent() - if !enabled || since.IsZero() || !since.Equal(v.EnabledSince()) { - t.Fatalf("empty inventory did not immediately refresh frozen-clock consent: (%v, %v)", enabled, since) + if enabled || !since.IsZero() { + t.Fatalf("empty inventory armed capture: (%v, %v)", enabled, since) } } @@ -195,6 +199,9 @@ func TestConfirmConsentReadsNowAndSharesTheAnswer(t *testing.T) { if enabled, _ := v.CaptureConsent(); enabled { t.Fatal("fresh profile already enabled") } + if err := config.WriteMemoryEnabled(v.profile, true); err != nil { + t.Fatal(err) + } written := now.Add(time.Second) uri := sqliteuri.File(path) db, err := sql.Open("sqlite", uri.String()) diff --git a/internal/knowledge/edit.go b/internal/knowledge/edit.go index 9dddb08786..bca53e42f3 100644 --- a/internal/knowledge/edit.go +++ b/internal/knowledge/edit.go @@ -75,9 +75,6 @@ func (v *Vault) DeletePage(id, expectedRevision string) error { return v.changePage(id, expectedRevision, pageOverride{ID: id, Deleted: true}) } func (v *Vault) changePage(id, revision string, override pageOverride) error { - if !v.Enabled() { - return ErrDisabled - } db, err := v.database() if err != nil { return err diff --git a/internal/knowledge/recall_test.go b/internal/knowledge/recall_test.go index dc4ebb9943..5989cd5d67 100644 --- a/internal/knowledge/recall_test.go +++ b/internal/knowledge/recall_test.go @@ -129,6 +129,9 @@ func TestKnowledgeRefinesTwoDirtySubjectsWithoutUnchangedRetries(t *testing.T) { func TestKnowledgeFrozenReplayReportsFailedSourceContracts(t *testing.T) { v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } withKnowledgeModel(t, func(p map[string]any) map[string]any { return map[string]any{"claims": []any{}} }) raw, err := v.RunSupportContracts(context.Background(), "fixture") if err == nil { diff --git a/internal/knowledge/store.go b/internal/knowledge/store.go index b106343f8d..047fbb2ff6 100644 --- a/internal/knowledge/store.go +++ b/internal/knowledge/store.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/sqliteuri" ) @@ -89,6 +90,15 @@ func (v *Vault) ConfirmConsent() (enabled bool, since time.Time) { // consent answers from the shared view while it is younger than maxAge and no // consent write of this process came after it; maxAge 0 always reads. func (v *Vault) consent(maxAge time.Duration) (enabled bool, since time.Time) { + // The setting is read for every memory step, independently of cached metadata. + defer func() { + enabled = v.Enabled() + if !enabled { + since = time.Time{} + } else if since.IsZero() { + since = time.Unix(0, 0).UTC() + } + }() view := v.captureConsentView() view.mu.Lock() defer view.mu.Unlock() @@ -200,8 +210,7 @@ func (v *Vault) state(key string) (document, error) { return stateDocument(db, key) } func (v *Vault) Enabled() bool { - value, err := v.state("config") - return err == nil && flag(value["enabled"]) + return config.MemoryEnabledAt(v.profile) } func (v *Vault) EnabledSince() time.Time { value, err := v.state("config") @@ -340,7 +349,7 @@ func (v *Vault) status(db *sql.DB) (Snapshot, error) { return snapshot, err } snapshot.Revision = textValue(revision["value"]) - snapshot.Enabled = flag(settings["enabled"]) + snapshot.Enabled = v.Enabled() snapshot.LastError = textValue(status["last_error"]) snapshot.NeedsUpdate = flag(status["needs_update"]) snapshot.CostUSD = number(status["cost_usd"]) @@ -381,7 +390,7 @@ func (v *Vault) Snapshot() (Snapshot, error) { } defer db.Exec("ROLLBACK") snapshot, err := v.status(db) - if err != nil || !snapshot.Enabled { + if err != nil { return snapshot, err } snapshot.Pages, err = readPages(db) diff --git a/internal/manual/memory_settings_test.go b/internal/manual/memory_settings_test.go new file mode 100644 index 0000000000..919f37782b --- /dev/null +++ b/internal/manual/memory_settings_test.go @@ -0,0 +1,21 @@ +package manual + +import ( + "strings" + "testing" +) + +func TestMemorySettingQuestionsReachCostAndConsent(t *testing.T) { + for _, question := range []string{"turn memory on", "why isn't it learning", "what does memory cost"} { + found := false + for _, section := range Chat().Search(question, DefaultResults) { + if strings.Contains(section.Body, "off by default") && strings.Contains(section.Body, "39%") && strings.Contains(section.Body, "27%") && strings.Contains(section.Body, "38%") { + found = true + break + } + } + if !found { + t.Errorf("%q does not reach memory consent and measured costs", question) + } + } +} diff --git a/internal/session/knowledge_consent_test.go b/internal/session/knowledge_consent_test.go index f3791b3300..3ca82f9d2c 100644 --- a/internal/session/knowledge_consent_test.go +++ b/internal/session/knowledge_consent_test.go @@ -4,13 +4,13 @@ import ( "context" "database/sql" "fmt" - "os" "path/filepath" "sync/atomic" "testing" "time" "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/knowledge" "github.com/Agent-Field/codeaf/internal/sqliteuri" ) @@ -58,6 +58,9 @@ func TestKnowledgeTickersShareOneConsentRead(t *testing.T) { if err != nil { t.Fatal(err) } + if err := config.WriteMemoryEnabled(profile, true); err != nil { + t.Fatal(err) + } since := time.Now().UTC().Format(time.RFC3339Nano) _, err = db.Exec("INSERT INTO state(key,payload) VALUES('config',?)", fmt.Sprintf(`{"enabled":true,"schema":1,"created_at":%q}`, since)) closeErr := db.Close() @@ -103,6 +106,9 @@ func consentChat(t *testing.T, profile string) *Agent { // writeConsent is another process writing this profile's consent row. func writeConsent(t *testing.T, path string, at time.Time) { t.Helper() + if err := config.WriteMemoryEnabled(filepath.Dir(filepath.Dir(path)), true); err != nil { + t.Fatal(err) + } uri := sqliteuri.File(path) db, err := sql.Open("sqlite", uri.String()) if err != nil { @@ -139,10 +145,10 @@ func TestKnowledgeAdmitsSourcesOnTheVaultsCurrentOptIn(t *testing.T) { } } -// A vault deleted elsewhere withdraws consent at once for what is admitted: +// A switch changed elsewhere withdraws consent at once for what is admitted: // the chat's sources wait in its journal, and are captured when consent that // covers them comes back. -func TestKnowledgeKeepsSourcesWhileTheVaultIsGone(t *testing.T) { +func TestKnowledgeKeepsSourcesWhileMemoryIsOff(t *testing.T) { profile := t.TempDir() agent := consentChat(t, profile) vault := agent.Knowledge() @@ -154,11 +160,10 @@ func TestKnowledgeKeepsSourcesWhileTheVaultIsGone(t *testing.T) { t.Fatal("own opt-in not seen") } path := filepath.Join(vault.Directory(), "knowledge.sqlite") - for _, name := range []string{path, path + "-wal", path + "-shm"} { - if err := os.Remove(name); err != nil && !os.IsNotExist(err) { - t.Fatal(err) - } + if err := config.WriteMemoryEnabled(profile, false); err != nil { + t.Fatal(err) } + collect(t, mustSubmit(t, agent, "Explain how the queue keeps work durable")) time.Sleep(2500 * time.Millisecond) snapshot, err := vault.Snapshot() diff --git a/internal/session/knowledge_history_test.go b/internal/session/knowledge_history_test.go index 8af612c3a0..d50f5d0067 100644 --- a/internal/session/knowledge_history_test.go +++ b/internal/session/knowledge_history_test.go @@ -330,6 +330,9 @@ func TestKnowledgeInitialBuildImportsAllProjectsAndAutopopulates(t *testing.T) { func TestKnowledgeWithoutHistoryWaitsAndLearnsFromFirstSavedChat(t *testing.T) { root := t.TempDir() v := knowledge.Open(t.TempDir()) + if err := v.StartBuild(); err != nil { + t.Fatal(err) + } calls := knowledgeProviderFixture(t, nil, nil) runKnowledgeWorker(t, v, root) state := waitKnowledgeState(t, v, func(s knowledge.Snapshot) bool { return s.Build.Phase == knowledge.BuildWaiting }) diff --git a/internal/session/knowledge_tools.go b/internal/session/knowledge_tools.go index fd8482ef74..71a17cf31d 100644 --- a/internal/session/knowledge_tools.go +++ b/internal/session/knowledge_tools.go @@ -137,7 +137,7 @@ func (a *Agent) knowledgeTools() []bare.Tool { return err.Error(), true, nil } vault := a.Knowledge() - if vault == nil || !vault.Enabled() { + if vault == nil { return "Knowledge is not enabled. The user can create a vault in Memory.", false, nil } hits, err := vault.Search(p.Query, 8) @@ -180,7 +180,7 @@ func (a *Agent) knowledgeTools() []bare.Tool { return err.Error(), true, nil } vault := a.Knowledge() - if vault == nil || !vault.Enabled() { + if vault == nil { return "Knowledge is not enabled. The user can create a vault in Memory.", false, nil } page, err := vault.Read(p.ID) diff --git a/internal/session/memory.go b/internal/session/memory.go index 6ee485aa62..ff8bd351d4 100644 --- a/internal/session/memory.go +++ b/internal/session/memory.go @@ -27,10 +27,9 @@ package session // does it by hand, and the model's `forget` is the one place it is asked // for in words. // -// - THE SWITCH GOVERNS NOTICING, NOT MEMORY. The person's `memory.enabled` -// row turns the noticing pass off; saved lines remain available and -// /remember goes on working. It is read live, at the end of every -// turn, so a long-lived engine follows a change without a restart. +// - THE SWITCH GOVERNS AUTOMATIC LEARNING. The person's memory.enabled +// row stops noticing and knowledge compilation. Saved lines remain +// available and /remember goes on working. Each memory step reads it live. // // A READ-ONLY POSTURE (an automation's run, [Config.bindingOnlyMemory]) reads // the block and does nothing else: no noticing, no verbs, no lines said. @@ -572,7 +571,7 @@ func memoryOffUpgradeWords(count int) string { if count == 1 { lines = "1 kept memory line now rides every conversation it holds in" } - return lines + " — memory off now only stops noticing · see or forget them on the Memories tab" + return lines + " — memory off stops automatic learning · see or forget them on the Memories tab" } // memoryOverflowWords is that one line. @@ -941,6 +940,9 @@ func (a *Agent) reportNoticeFailure(err error) { // set stay together, so a correction never applies to a newer turn's memories. // The job remains owned by memoryCtx and memoryJobs, including its backoff. func (a *Agent) noticeWithRecovery(client reflex.Completer, words, answer string, texts []string, askMemory bool) (reflex.NoticeResult, error) { + if !config.MemoryEnabledAt(a.config.ProfileDir) { + return reflex.NoticeResult{}, nil + } found, err := reflex.Notice(a.memoryCtx, client, words, answer, texts, askMemory) kind, _ := provider.RouteFailureOf(err) if err == nil || kind != provider.RouteTransient || a.memoryCtx.Err() != nil { @@ -956,6 +958,9 @@ func (a *Agent) noticeWithRecovery(client reflex.Completer, words, answer string if err := a.memoryCtx.Err(); err != nil { return reflex.NoticeResult{}, err } + if !config.MemoryEnabledAt(a.config.ProfileDir) { + return reflex.NoticeResult{}, nil + } return reflex.Notice(a.memoryCtx, client, words, answer, texts, askMemory) } diff --git a/internal/session/memory_test.go b/internal/session/memory_test.go index 5dc2fe570e..eef84bd8e7 100644 --- a/internal/session/memory_test.go +++ b/internal/session/memory_test.go @@ -127,6 +127,7 @@ func brainAgent(t *testing.T, completer Completer, mutate func(*Config)) (*Agent // A profile the test owns, so the noticing row is never read from the // machine's own configuration. config.ProfileDir = t.TempDir() + writeProfileRow(t, config.ProfileDir, "memory.enabled", "on") if mutate != nil { mutate(config) } @@ -525,14 +526,14 @@ func TestTheSwitchTurnsNoticingOffAndLeavesEverythingElse(t *testing.T) { if !strings.Contains(script.system(0), "prefers tabs over spaces") { t.Fatal("with noticing off the person's lines stopped riding the turn") } - if len(script.asked) != 1 || script.asked[0] { + if len(script.asked) != 0 { t.Fatalf("with noticing off the post-turn call asked about memory: %v", script.asked) } if got := activeTexts(t, brain, store.OwnerUser); len(got) != 1 { t.Fatalf("with noticing off something was kept: %v", got) } - if !strings.Contains(agent.stateCardText(), "tidy the build") { - t.Fatal("with noticing off the state card stopped being kept") + if strings.Contains(agent.stateCardText(), "tidy the build") { + t.Fatal("off made an automatic state-card call") } if _, err := agent.Remember("a line kept by hand"); err != nil { t.Fatalf("with noticing off /remember stopped working: %v", err) @@ -541,7 +542,7 @@ func TestTheSwitchTurnsNoticingOffAndLeavesEverythingElse(t *testing.T) { writeProfileRow(t, agent.config.ProfileDir, config.KeyMemoryEnabled, config.MemoryOn) collect(t, mustSubmit(t, agent, "another")) settleMemory(agent) - if len(script.asked) != 2 || !script.asked[1] { + if len(script.asked) != 1 || !script.asked[0] { t.Fatalf("the row's change was not followed without a restart: %v", script.asked) } } diff --git a/internal/session/nochange_test.go b/internal/session/nochange_test.go index 0e720ed692..dcde7474f6 100644 --- a/internal/session/nochange_test.go +++ b/internal/session/nochange_test.go @@ -16,6 +16,7 @@ package session import ( "context" "fmt" + "github.com/Agent-Field/codeaf/internal/config" "strings" "sync/atomic" "testing" @@ -119,6 +120,7 @@ func TestACarryOnAnsweredNoChangeEndsItAndKeepsTheAnswer(t *testing.T) { } completer := &scriptedCompleter{steps: steps} agent := checkpointAgent(t, completer, func(config *Config) { config.Memory = openTestBrain(t) }) + writeProfileRow(t, agent.config.ProfileDir, config.KeyMemoryEnabled, config.MemoryOn) learned := observeMemoryExtraction(completer) stubbedGraph(agent, func(node *TaskNode) {}) @@ -232,6 +234,7 @@ func TestNoChangeAfterCallsMadeForTheNoteIsReadAgain(t *testing.T) { } completer := &scriptedCompleter{steps: steps} agent := checkpointAgent(t, completer, func(config *Config) { config.Memory = openTestBrain(t) }) + writeProfileRow(t, agent.config.ProfileDir, config.KeyMemoryEnabled, config.MemoryOn) learned := observeMemoryExtraction(completer) stubbedGraph(agent, func(node *TaskNode) {}) diff --git a/internal/session/stop_auxiliary_test.go b/internal/session/stop_auxiliary_test.go index 6c9b2ac583..f3575b3f69 100644 --- a/internal/session/stop_auxiliary_test.go +++ b/internal/session/stop_auxiliary_test.go @@ -2,6 +2,7 @@ package session import ( "context" + "github.com/Agent-Field/codeaf/internal/config" "sync/atomic" "testing" "testing/synctest" @@ -30,6 +31,8 @@ func TestConversationStopPreventsBackgroundMemoryRetryAndRestoresOnlyForFreshInp t.Cleanup(stopMemory) t.Cleanup(stopTitle) agent := &Agent{memoryCtx: memory, memoryStop: stopMemory, titleCtx: title, titleStop: stopTitle} + agent.config.ProfileDir = t.TempDir() + writeProfileRow(t, agent.config.ProfileDir, config.KeyMemoryEnabled, config.MemoryOn) oldMemory, oldTitle := agent.memoryCtx, agent.titleCtx probe := &stoppedMemoryRetryProbe{first: make(chan struct{})} if !agent.startMemoryJob() { diff --git a/internal/tui3/knowledge.go b/internal/tui3/knowledge.go index d25cda3dde..f7cda9915a 100644 --- a/internal/tui3/knowledge.go +++ b/internal/tui3/knowledge.go @@ -358,7 +358,7 @@ func (a *app) knowledgeOwns(msg tea.KeyPressMsg) (tea.Cmd, bool) { p.open = false } case "enter": - if !p.reading.Enabled { + if !p.reading.Enabled && len(p.reading.Pages) == 0 { return a.knowledgeRead(true), true } if p.links { @@ -513,7 +513,7 @@ func (a *app) knowledgeBody(width, room int) []placeRow { var rows []placeRow add := func(s string) { rows = append(rows, placeRow{text: " " + fit(s, max(1, width-2)), hit: -1}) } add(a.pal.bold(a.pal.muted("Knowledge"))) - if p.reading.Enabled { + if p.reading.Enabled || len(p.reading.Pages) > 0 { stats := a.pal.data(fmt.Sprint(len(p.reading.Pages))) + a.pal.dim(" pages") if p.reading.Pending > 0 { stats += a.pal.dim(" · ") + a.pal.data(fmt.Sprint(p.reading.Pending)) + a.pal.dim(" queued") @@ -539,7 +539,7 @@ func (a *app) knowledgeBody(width, room int) []placeRow { if p.loading { add(a.pal.dim("Reading knowledge…")) } - if !p.reading.Enabled { + if !p.reading.Enabled && len(p.reading.Pages) == 0 { if p.reading.Build.Error != "" { add(a.pal.bad(p.reading.Build.Error)) } @@ -848,7 +848,7 @@ func (placeMemory) pressAt(a *app, x, y int) (tea.Cmd, bool) { return nil, true } if hit, ok := hits[y].(knowledgeHit); ok { - if hit.create && !p.reading.Enabled && x >= 1 && x < 1+ansi.StringWidth(knowledgeBuildLabel) { + if hit.create && !p.reading.Enabled && len(p.reading.Pages) == 0 && x >= 1 && x < 1+ansi.StringWidth(knowledgeBuildLabel) { return a.knowledgeRead(true), true } for _, link := range hit.links { From d0d2ce330fd4594c34a931189446c33438215f82 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:52:22 -0500 Subject: [PATCH 17/28] keys: a failed vault read returns its error instead of panicking (#2265) * test: reproduce tombstone lookup panic on vault read failure Co-Authored-By: Claude Opus 5.5 * fix: return vault read errors before checking tombstones Deleted dereferenced a nil document when a valid vault handle failed to read its backing file. Return the read error so sync can retry instead of terminating its worker. Co-Authored-By: Claude Opus 5.5 * docs: clarify previous and current vault read failure behavior Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * manual: name the vault read failure section by what it covers Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../2265-vault-deleted-read-error.md | 12 ++++++++++ internal/keys/deleted_error_test.go | 23 +++++++++++++++++++ internal/keys/vault.go | 5 +++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 docs/changes/unreleased/2265-vault-deleted-read-error.md create mode 100644 internal/keys/deleted_error_test.go diff --git a/docs/changes/unreleased/2265-vault-deleted-read-error.md b/docs/changes/unreleased/2265-vault-deleted-read-error.md new file mode 100644 index 0000000000..868c60921e --- /dev/null +++ b/docs/changes/unreleased/2265-vault-deleted-read-error.md @@ -0,0 +1,12 @@ +--- +kind: fixed +title: Saved-key sync returns vault read failures instead of panicking +pr: 2265 +surface: [engine, app] +invalidates: + - "Checking whether a saved key was removed previously could panic on a local vault read failure; it now returns the read error." +--- + +A failed local vault read during a removal check now returns its error, allowing +background sync to retry rather than ending its worker. This does not repair a +damaged vault or change which saved keys travel between computers. diff --git a/internal/keys/deleted_error_test.go b/internal/keys/deleted_error_test.go new file mode 100644 index 0000000000..1dfc67ff5e --- /dev/null +++ b/internal/keys/deleted_error_test.go @@ -0,0 +1,23 @@ +package keys_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/keys" +) + +func TestDeletedReturnsVaultReadError(t *testing.T) { + home := t.TempDir() + vault, err := keys.Open(home) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, "vault.enc"), []byte("damaged vault"), 0600); err != nil { + t.Fatal(err) + } + if gone, err := vault.Deleted("missing-slot"); err == nil || gone { + t.Fatalf("Deleted = %v, %v; want false and a vault read error", gone, err) + } +} diff --git a/internal/keys/vault.go b/internal/keys/vault.go index 479abaee0f..7822f6e663 100644 --- a/internal/keys/vault.go +++ b/internal/keys/vault.go @@ -180,7 +180,10 @@ func (v *Vault) Get(id string) (Entry, error) { // that was removed from one that was never there. func (v *Vault) Deleted(id string) (bool, error) { d, err := v.read() - return d.Secrets[id].Deleted, err + if err != nil { + return false, err + } + return d.Secrets[id].Deleted, nil } // Delete removes the secret under id and leaves a tombstone, so merging with a From ade1233ff0c268728f7327d816f2cfa92b451659 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:13:54 -0500 Subject: [PATCH 18/28] app: one engine on a home; a second waits for the first to end (#2274) * test: two engines on one home must not both start Co-Authored-By: Claude Opus 5.5 * app: one engine on a home; a second waits for the first to end Co-Authored-By: Claude Opus 5.5 * test: a waiting engine starts nothing, and a child doesn't hold the home Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/app.go | 3 + cmd/codeaf/app_home.go | 51 ++++ cmd/codeaf/app_one_engine_test.go | 281 +++++++++++++++++++++ docs/changes/unreleased/2274-one-engine.md | 13 + docs/design/app-v3/PROTOCOL.md | 5 + internal/identity/l4fix2_startup_test.go | 2 +- internal/identity/pristine.go | 1 + internal/manual/chat/desktop-sync.md | 6 + 8 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 cmd/codeaf/app_home.go create mode 100644 cmd/codeaf/app_one_engine_test.go create mode 100644 docs/changes/unreleased/2274-one-engine.md diff --git a/cmd/codeaf/app.go b/cmd/codeaf/app.go index 5f21ea7ce1..90a8596e1e 100644 --- a/cmd/codeaf/app.go +++ b/cmd/codeaf/app.go @@ -123,6 +123,9 @@ func runApp(args []string) error { return err } } + if err := holdAppHome(); err != nil { + return err + } // STDOUT IS THE PROTOCOL'S AND NOBODY ELSE'S. The wire keeps the real // descriptor, and anything else in this process that prints to stdout // lands on stderr, where the app collects logs. diff --git a/cmd/codeaf/app_home.go b/cmd/codeaf/app_home.go new file mode 100644 index 0000000000..a1a51541c0 --- /dev/null +++ b/cmd/codeaf/app_home.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/home" +) + +// appHomeLock stays reachable until process death. Closing it when runApp +// returns would admit a new engine while main still finishes saving. The OS +// releases it on any exit, so a crash leaves nothing to clean up. +var appHomeLock *os.File + +// appHomeWait is how long an engine waits for one that is closing the same home. +// It outlasts the longest careful wrap-up, so an update's replacement engine +// starts as soon as the old one ends. +var appHomeWait = appLeaveWait + 10*time.Second + +const appHomePoll = 50 * time.Millisecond + +// holdAppHome makes this the one engine on this home. Two engines would both +// spend on the same work. It runs before anything that could make a paid call. +func holdAppHome() error { + if err := os.MkdirAll(home.Dir(), 0700); err != nil { + return err + } + file, err := os.OpenFile(home.Join("app.lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return err + } + deadline := time.Now().Add(appHomeWait) + for { + err := filelock.Lock(file, true, true) + if err == nil { + appHomeLock = file + return nil + } + if !filelock.IsBusy(err) { + _ = file.Close() + return fmt.Errorf("hold codeaf app home: %w", err) + } + if time.Now().After(deadline) { + _ = file.Close() + return fmt.Errorf("another codeaf app is still closing this home after %s; nothing was started. Quit it, then open the app again", appHomeWait) + } + time.Sleep(appHomePoll) + } +} diff --git a/cmd/codeaf/app_one_engine_test.go b/cmd/codeaf/app_one_engine_test.go new file mode 100644 index 0000000000..bcbda85d1a --- /dev/null +++ b/cmd/codeaf/app_one_engine_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/home" +) + +// The command's real signal and wire doors are exercised in separate processes. +// Only final disk saving is replaced, by a disk that never answers its caller. +func TestAppOneEngine(t *testing.T) { + if mode := os.Getenv("CODEAF_TEST_ONE_ENGINE"); mode != "" { + switch mode { + case "stuck": + flushFinalSaves = func(context.Context) error { + _ = os.WriteFile(home.Join("saving"), nil, 0600) + select {} + } + case "impatient": + appHomeWait = 300 * time.Millisecond + case "parent": + t.Setenv(home.EnvVar, os.Getenv("CODEAF_TEST_ONE_HOME")) + if err := holdAppHome(); err != nil { + t.Fatal(err) + } + child := exec.Command("sleep", "60") + if err := child.Start(); err != nil { + t.Fatal(err) + } + fmt.Printf("child %d\n", child.Process.Pid) + select {} + } + root := os.Getenv("CODEAF_TEST_ONE_HOME") + t.Setenv(home.EnvVar, root) + os.Args = []string{os.Args[0], "app", "--home", root} + if mode == "schema" { + os.Args = append(os.Args, "--schema") + } + if mode == "version" { + os.Args = []string{os.Args[0], "version"} + } + main() + os.Exit(0) + } + closing := func(t *testing.T, root string) *oneEngineProcess { + t.Helper() + first := startOneEngine(t, root, "stuck") + first.hello(t) + _ = first.cmd.Process.Signal(syscall.SIGTERM) + waitOneEngineFile(t, filepath.Join(root, "saving")) + return first + } + t.Run("the second engine waits and starts when the first ends", func(t *testing.T) { + root := t.TempDir() + first := closing(t, root) + second := startOneEngine(t, root, "ordinary") + select { + case line := <-second.lines: + t.Fatalf("second engine admitted while the first lives: %s", line) + case <-second.done: + t.Fatalf("second engine exited instead of waiting: %s", second.stderr.String()) + case <-time.After(2 * time.Second): + } + // A crash releases ownership without a stale-pid recovery or a deleted lock. + first.kill(t) + second.hello(t) + second.leave(t) + }) + t.Run("a crash leaves nothing to wait for", func(t *testing.T) { + root := t.TempDir() + first := startOneEngine(t, root, "ordinary") + first.hello(t) + first.kill(t) + began := time.Now() + next := startOneEngine(t, root, "ordinary") + next.hello(t) + next.leave(t) + if waited := time.Since(began); waited > 8*time.Second { + t.Fatalf("relaunch after a crash took %s", waited) + } + }) + t.Run("the wait ends with a clear message and starts nothing", func(t *testing.T) { + root := t.TempDir() + closing(t, root) + second := startOneEngine(t, root, "impatient") + select { + case line := <-second.lines: + t.Fatalf("engine spoke after giving up: %s", line) + case <-second.done: + if !strings.Contains(second.stderr.String(), "another codeaf app is still closing this home") { + t.Fatalf("missing message: %s", second.stderr.String()) + } + case <-time.After(10 * time.Second): + t.Fatal("the wait had no bound") + } + }) + t.Run("a waiting engine starts nothing and asks for nothing", func(t *testing.T) { + root := t.TempDir() + held, err := os.OpenFile(filepath.Join(root, "app.lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer held.Close() + if err := filelock.Lock(held, true, true); err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + provider := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer provider.Close() + t.Setenv("CODEAF_BASE_URL", provider.URL) + t.Setenv("OPENROUTER_API_KEY", "fixture") + waiting := startOneEngine(t, root, "ordinary") + select { + case line := <-waiting.lines: + t.Fatalf("engine spoke while waiting: %s", line) + case <-waiting.done: + t.Fatalf("engine left while waiting: %s", waiting.stderr.String()) + case <-time.After(2 * time.Second): + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || requests.Load() != 0 { + t.Fatalf("while waiting: home holds %v, %d provider requests", entries, requests.Load()) + } + if err := filelock.Unlock(held); err != nil { + t.Fatal(err) + } + waiting.hello(t) + waiting.leave(t) + }) + t.Run("a child that outlives the engine doesn't hold the home", func(t *testing.T) { + root := t.TempDir() + parent := startOneEngine(t, root, "parent") + var pid int + select { + case line := <-parent.lines: + if _, err := fmt.Sscanf(line, "child %d", &pid); err != nil { + t.Fatalf("no child line: %s", line) + } + case <-time.After(10 * time.Second): + t.Fatalf("parent didn't start its child: %s", parent.stderr.String()) + } + t.Cleanup(func() { + if raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)); err == nil && bytes.HasPrefix(raw, []byte("sleep")) { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + }) + parent.kill(t) + next := startOneEngine(t, root, "ordinary") + next.hello(t) + next.leave(t) + }) + t.Run("only one home's app is held", func(t *testing.T) { + root := t.TempDir() + closing(t, root) + other := startOneEngine(t, t.TempDir(), "ordinary") + other.hello(t) + other.leave(t) + for _, mode := range []string{"schema", "version"} { + p := startOneEngine(t, root, mode) + select { + case <-p.done: + case <-time.After(10 * time.Second): + t.Fatalf("%s waited on the app's lock", mode) + } + } + }) +} + +type oneEngineProcess struct { + cmd *exec.Cmd + stdin io.WriteCloser + lines chan string + done chan struct{} + stderr bytes.Buffer +} + +func startOneEngine(t *testing.T, root, mode string) *oneEngineProcess { + t.Helper() + p := &oneEngineProcess{lines: make(chan string, 4096), done: make(chan struct{})} + p.cmd = exec.Command(os.Args[0], "-test.run=^TestAppOneEngine$") + p.cmd.Env = append(os.Environ(), "CODEAF_TEST_ONE_ENGINE="+mode, home.EnvVar+"="+filepath.Join(root, "inherited"), "CODEAF_TEST_ONE_HOME="+root) + var err error + p.stdin, err = p.cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := p.cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + p.cmd.Stderr = &p.stderr + if err := p.cmd.Start(); err != nil { + t.Fatal(err) + } + scanned := make(chan struct{}) + go func() { + defer close(scanned) + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 4096), 16<<20) + for scanner.Scan() { + select { + case p.lines <- scanner.Text(): + default: + } + } + }() + go func() { <-scanned; _ = p.cmd.Wait(); close(p.done) }() + t.Cleanup(func() { p.kill(t); _ = p.stdin.Close() }) + return p +} + +func (p *oneEngineProcess) hello(t *testing.T) { + t.Helper() + select { + case line := <-p.lines: + if !strings.Contains(line, `"event":"hello"`) { + t.Fatalf("not hello: %s", line) + } + case <-p.done: + t.Fatalf("engine ended before hello: %s", p.stderr.String()) + case <-time.After(10 * time.Second): + t.Fatal("engine did not say hello") + } +} + +func (p *oneEngineProcess) leave(t *testing.T) { + t.Helper() + _ = p.stdin.Close() + select { + case <-p.done: + case <-time.After(10 * time.Second): + t.Fatal("engine did not close") + } +} + +func (p *oneEngineProcess) kill(t *testing.T) { + t.Helper() + select { + case <-p.done: + return + default: + } + // Check the exact child before killing it; never select processes by name. + if raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", p.cmd.Process.Pid)); err == nil && !bytes.Contains(raw, []byte("TestAppOneEngine")) { + t.Fatalf("unexpected child command: %q", raw) + } + _ = p.cmd.Process.Kill() + <-p.done +} + +func waitOneEngineFile(t *testing.T, path string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(path); err == nil { + return + } + if time.Now().After(deadline) { + t.Fatal("signal did not reach final saving") + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/docs/changes/unreleased/2274-one-engine.md b/docs/changes/unreleased/2274-one-engine.md new file mode 100644 index 0000000000..b34a911afa --- /dev/null +++ b/docs/changes/unreleased/2274-one-engine.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: One desktop engine runs on a home at a time +pr: 2274 +surface: [engine] +invalidates: + - "A second desktop engine could start on a home while the first was still closing, and both could pay for the same Factory read." +--- + +`codeaf app` holds an OS lock on its home until it exits. A second engine waits +before saying hello and starts when the first ends. If the wait runs out it exits +with a clear log line and has started nothing. A crash releases the lock. Terminal +`codeaf`, headless runs and other homes don't take this lock. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index e0963ede6c..ab9d63fa77 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -1172,6 +1172,11 @@ role pins and checked as the other pins are. |---|---|---| | `slash.catalog` | `{}` | `SlashCatalogResult` | + +## One engine on a home + +`codeaf app` holds an OS lock on its home until it exits. A second `codeaf app` on that home waits before `hello`, starting no recovery, Factory work or clock, and starts as soon as the first ends. If the first is still there when the wait runs out, it exits with status 1 and its stderr says `another codeaf app is still closing this home`. A crash releases the lock. No method or feature changes. + ## spend methods | method | params | result | diff --git a/internal/identity/l4fix2_startup_test.go b/internal/identity/l4fix2_startup_test.go index 61d058a74a..8b0399bb9b 100644 --- a/internal/identity/l4fix2_startup_test.go +++ b/internal/identity/l4fix2_startup_test.go @@ -53,7 +53,7 @@ func firstStart(t *testing.T) string { if err := autos.Close(); err != nil { t.Fatal(err) } - for _, name := range []string{"teams.json.lock", "v3/automations/clock.lock", "v3/automations/clock.log", "v3/automations/environment.salt", "v3/automations/windows/window", "v3/projects/project/chat/meta.lock", startupChat + "/.furrowpolicy"} { + for _, name := range []string{"app.lock", "teams.json.lock", "v3/automations/clock.lock", "v3/automations/clock.log", "v3/automations/environment.salt", "v3/automations/windows/window", "v3/projects/project/chat/meta.lock", startupChat + "/.furrowpolicy"} { startupFile(t, home, name, "") } startupFile(t, home, "model-catalog-a88839f814a18779.json", "{}") diff --git a/internal/identity/pristine.go b/internal/identity/pristine.go index a5a934ba74..6bd685e107 100644 --- a/internal/identity/pristine.go +++ b/internal/identity/pristine.go @@ -35,6 +35,7 @@ var startupRules = rules{ // The pairing name prompt writes this before checking whether the home can join. "device-name": never, "teams.json.lock": never, + "app.lock": never, "model-catalog-*.json": catalogCache, // Provider catalog caches are fetched again on demand. "knowledge": within(rules{ ".worker.lock": never, diff --git a/internal/manual/chat/desktop-sync.md b/internal/manual/chat/desktop-sync.md index 030bdd0a33..fe84d06451 100644 --- a/internal/manual/chat/desktop-sync.md +++ b/internal/manual/chat/desktop-sync.md @@ -229,6 +229,12 @@ A model choice is applied here only when its provider is set up on this computer otherwise this computer keeps its own model. Setting up that provider later does not apply an older skipped choice; only a newer choice on the other computer travels. +## Why the desktop takes a moment to start after an update or a quick reopen + +One desktop engine runs on your codeaf home at a time, so two can't spend on the same work. If the last one is still saving as it closes, the new one waits, starts nothing, and carries on as soon as the last one ends. A crash doesn't make you wait. + +If the last one is still there after about two minutes, the new one stops and the log says `another codeaf app is still closing this home`. Quit it, then open the app again. Terminal `codeaf`, headless runs and other `--home` folders don't wait. + ## Saved-key sync when the local vault is unreadable A removal check returns a local vault read error rather than stopping the From 6405b0626702aa31a9c3a2bca92bd91058a8169b Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:36:57 -0500 Subject: [PATCH 19/28] spend: a failed or cut-off call still counts (#2267) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: reproduce missing factory charge settlement Co-Authored-By: Claude Opus 5.5 * fix: settle factory charges from final provider receipts Interim stream usage suppressed reconciliation even when no session billing sink could consume it. Factory calls now recover the final receipt and retain unknown-price ownership. Write through ledger overflow without holding a turn indefinitely. Memory keeps its existing billing path. Co-Authored-By: Claude Opus 5.5 * fix: snapshot pending overflow writes across ledger flushes Capture completion barriers without racing WaitGroup reuse. A full queue cannot certify a successful flush; it reports false under the existing deadline instead. Co-Authored-By: Claude Opus 5.5 * test: keep unpriced factory receipts visibly unknown Co-Authored-By: Claude Opus 5.5 * fix: keep factory token-only receipts unbilled A receipt with token counts but no provider price cannot establish a dollar charge. Final-price consumers now retain the existing unknown-price marker instead of treating those counts as a settled bill. Other receipt consumers remain unchanged. Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * spend: a full usage queue never makes a turn wait The 100 ms per overflow row added up on a stalled disk and broke the ledger's turn law. One background writer takes overflow rows instead. Co-Authored-By: Claude Opus 5.5 * spend: the ledger's one writer takes overflow rows, in order Co-Authored-By: Claude Opus 5.5 * spend: format the ledger writer Co-Authored-By: Claude Opus 5.5 * spend: a full queue's flush waits for the kept rows instead of reporting false Co-Authored-By: Claude Opus 5.5 * spend: wake the ledger writer when a row is kept after it emptied the queue Co-Authored-By: Claude Opus 5.5 * spend: an answer with no usage doesn't hold the job's next step for its receipt Co-Authored-By: Claude Opus 5.5 * Take the gated copy of the union merged docs --------- Co-authored-by: Claude Opus 5.5 --- .../appfactory_charge_settlement_test.go | 148 +++++++++++++++ .../2267-factory-charge-settlement.md | 16 ++ docs/design/app-v3/PROTOCOL.md | 9 + .../chat/factory-stage-conversations.md | 3 +- internal/manual/chat/factory.md | 17 +- internal/manual/chat/models-and-cost.md | 9 + internal/provider/billing.go | 15 +- internal/provider/receipt.go | 30 +++- internal/session/clientdoor.go | 8 + internal/session/usage_ledger.go | 168 ++++++++++++------ 10 files changed, 366 insertions(+), 57 deletions(-) create mode 100644 cmd/codeaf/appfactory_charge_settlement_test.go create mode 100644 docs/changes/unreleased/2267-factory-charge-settlement.md diff --git a/cmd/codeaf/appfactory_charge_settlement_test.go b/cmd/codeaf/appfactory_charge_settlement_test.go new file mode 100644 index 0000000000..27b0bc3b4b --- /dev/null +++ b/cmd/codeaf/appfactory_charge_settlement_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestFactoryChargedCallsReachLedger(t *testing.T) { + for _, mode := range []string{"failed turn", "failed errand", "unmetered turn", "unmetered errand", "unknown turn", "unknown errand", "unpriced receipt"} { + t.Run(mode, func(t *testing.T) { + s := newLocalSetup(t) + st := s.start() + floor := s.backend.local.host.floor + it, err := floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Repair accounting", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: 10}) + if err != nil { + t.Fatal(err) + } + failed := mode == "failed turn" || mode == "failed errand" + unknown := mode == "unknown turn" || mode == "unknown errand" || mode == "unpriced receipt" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + if r.URL.Path == "/generation" && mode == "unpriced receipt" { + fmt.Fprint(w, `{"data":{"tokens_prompt":20,"tokens_completion":5}}`) + } else if r.URL.Path == "/generation" { + fmt.Fprint(w, `{"data":{"total_cost":0.12,"tokens_prompt":20,"tokens_completion":5}}`) + } else { + fmt.Fprint(w, `{"data":[]}`) + } + return + } + var request struct { + Stream bool `json:"stream"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + id := "gen-factory-charge" + if unknown && mode != "unpriced receipt" { + id = "" + } + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"id\":%q,\"model\":\"deepseek/deepseek-v4.1-flash\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Done.\"}}]}\n\n", id) + if failed { + fmt.Fprint(w, "data: {\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":2,\"cost\":0.10},\"choices\":[]}\n\n") + fmt.Fprint(w, "data: {\"error\":{\"code\":400,\"message\":\"fixture failure\"}}\n\n") + } else { + fmt.Fprint(w, "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n") + } + })) + t.Cleanup(server.Close) + agent, err := session.New(session.Config{Workspace: t.TempDir(), SessionFile: filepath.Join(t.TempDir(), "stage.jsonl"), ProfileDir: filepath.Join(s.home, "stage-profile"), Model: "deepseek/deepseek-v4.1-flash", APIKey: "fixture", BaseURL: server.URL, JobBudget: &stageBudget{floor: floor, item: it}, Unattended: true, DeterministicNames: true, OneModel: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = agent.Close() }) + if mode == "failed errand" || mode == "unmetered errand" || mode == "unknown errand" { + _, err = agent.RunErrand(context.Background(), session.ErrandAsk{Purpose: "charge_fixture", Role: "brief", System: "Answer briefly.", Messages: []session.ErrandMessage{{Role: "user", Text: "Finish."}}, MaxTokens: 100, Deadline: time.Second}) + if err != nil { + t.Fatal(err) + } + } else { + events, err := agent.Submit(context.Background(), "Finish.") + if err != nil { + t.Fatal(err) + } + for range events { + } + } + deadline := time.Now().Add(4 * time.Second) + for { + if !session.FlushUsage() { + t.Fatal("ledger flush timed out") + } + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + var cost float64 + rows, missing := 0, 0 + for _, line := range lines { + if line.FactoryJob == factory.LedgerName(it) { + cost += line.USD + rows++ + if line.Unbilled { + missing++ + } + } + } + var header api.Job + s.must("factory.job", map[string]any{"factory": st.Factory, "job": it.ID}, &header) + var usage appwire.FactoryUsage + day := time.Now().Format(time.DateOnly) + s.must("factory.usage_by_model", map[string]any{"factory": st.Factory, "job": it.ID, "from": day, "to": day}, &usage) + valid := rows == 1 && math.Abs(cost-.12) < 1e-9 && math.Abs(header.Cost-.12) < 1e-9 + if unknown { + valid = rows == 1 && missing == 1 && cost == 0 && len(usage.Models) == 1 && usage.Models[0].CostUSD == nil && usage.Models[0].UnbilledCalls == 1 + } + + if valid { + var spend appwire.SpendReportResult + s.must("spend.report", map[string]any{"from": day, "to": day}, &spend) + if unknown && (spend.Totals.UnbilledRecords != 1 || spend.Totals.CostUSD != nil) { + t.Fatalf("Spending unknown price: %+v", spend.Totals) + } + if !unknown && (spend.Totals.CostUSD == nil || math.Abs(*spend.Totals.CostUSD-.12) > 1e-9) { + t.Fatalf("Spending charge: %+v", spend.Totals) + } + break + } + if time.Now().After(deadline) { + t.Fatalf("charge settlement: ledger=%.2f header=%.2f rows=%d unknown=%d usage=%+v", cost, header.Cost, rows, missing, usage.Models) + } + time.Sleep(10 * time.Millisecond) + } + }) + } + t.Run("queue overflow", func(t *testing.T) { + path := filepath.Join(t.TempDir(), session.UsageLedgerName) + t.Cleanup(func() { session.StopUsageWriter(path) }) + const count = 20000 + for i := 0; i < count; i++ { + session.RecordUsage(path, session.UsageLine{Model: "fixture", Calls: 1, USD: .01}) + } + if !session.FlushUsage() { + t.Fatal("ledger flush timed out") + } + rows, err := session.ReadUsage(path, time.Time{}) + if err != nil { + t.Fatal(err) + } + if len(rows) != count { + t.Fatalf("charge rows on disk=%d, want %d", len(rows), count) + } + }) +} diff --git a/docs/changes/unreleased/2267-factory-charge-settlement.md b/docs/changes/unreleased/2267-factory-charge-settlement.md new file mode 100644 index 0000000000..0f197a59f0 --- /dev/null +++ b/docs/changes/unreleased/2267-factory-charge-settlement.md @@ -0,0 +1,16 @@ +--- +kind: fixed +title: codeaf collects Factory charges after failed and unmetered calls +pr: 2267 +surface: [engine] +invalidates: + - "A Factory stream's interim usage could prevent its final receipt from reaching the ledger and job total." + - "Successful Factory calls without usage could disappear from Spending, and queue overflow could discard charge rows." +--- + +Factory jobs and their page helpers recover the final provider receipt after a +failed call or a successful answer without usage. Each recovered charge reaches +the job total and Spending once. Unknown prices remain unbilled markers, attributed +to their job, with no invented zero price. A full ledger queue writes through +instead of dropping charges; the caller never waits. +Memory billing keeps its existing path. Historical gaps are not reconstructed. diff --git a/docs/design/app-v3/PROTOCOL.md b/docs/design/app-v3/PROTOCOL.md index ab9d63fa77..fc771b5a8e 100644 --- a/docs/design/app-v3/PROTOCOL.md +++ b/docs/design/app-v3/PROTOCOL.md @@ -513,6 +513,13 @@ the protected-path gate then pauses safely. The stored unknown-list hold suffix remains `the plan names no files`; visible words describe the missing checkable list rather than claiming the plan names no files in prose. +`factory.job.approval` keeps the latest settled decision, including its kind, +actor and times, while later work or gates wait and after restarting the app. +A later gate retains its own waiting step, `needs_you` and `can.approve`; the +earlier decision never passes it. Beginning another approval replaces the single +summary with that decision and its own undo window. Activity keeps the receipts. +No method, parameter, reply field, error code or hello feature changes. + `factory.activity` records automatic approvals and their publications with `actor: {kind: "factory"}`, an empty `handle`, and no actor name. A person's approval keeps `actor: {kind: "person", name: "@handle"}` and their handle. @@ -706,6 +713,8 @@ failed}`: jobs read now; jobs skipped (already read, not an issue job, not visible to the caller, or past the first 20 read); jobs whose model call failed, left unread to try again. A factory with no model key fails every job. `queue.desk.last_read_cost` is what the last read cost, for "about $X". +Both read and scout bank each reported provider charge once on Spending, +including failed generations. Their methods, replies and hello features are unchanged. An engine whose hello carries the feature `factory.repo_checkout` answers `factory.queue` with `checkout_state` on each of `repos[]`: how a run of that diff --git a/internal/manual/chat/factory-stage-conversations.md b/internal/manual/chat/factory-stage-conversations.md index 50fd3b0e17..86638d0fb7 100644 --- a/internal/manual/chat/factory-stage-conversations.md +++ b/internal/manual/chat/factory-stage-conversations.md @@ -351,7 +351,8 @@ from the job's budget like the job's steps: everything done for the job spends f - **During a run** the manager, stages and their delegated workers record priced calls immediately against the same job. Child totals folded back into a parent are not charged again. - **Counted once.** The spend page still lists every call once. The job's figure reads the - same dollars; it is not an additional charge. + same dollars; it is not an additional charge. If a slow call races another provider, + both completed calls count toward the job, including the answer that was discarded. - **At the budget** new model calls and tools are refused. The job's owned agent families stop promptly, including delegated tasks, background commands and adaptive runs. This applies while the manager is idle too. Calls already sent can still report charges afterward; diff --git a/internal/manual/chat/factory.md b/internal/manual/chat/factory.md index 39257a2633..dd31b55bee 100644 --- a/internal/manual/chat/factory.md +++ b/internal/manual/chat/factory.md @@ -177,6 +177,14 @@ A repository without `.codeaf/factory.yaml` runs the shipped steps, and so does A step that falls short goes round again until its rounds run out, and only then asks you what to do. New jobs default to Auto, which passes routine optional approvals and publishes passing proof as a draft PR within the job budget. Manual asks before enabled approval gates and publication; the job header switch saves that choice. Activity credits automatic approvals and their publications to the factory, without naming the job owner or starter. A stale action says "approved automatically at" and the approval time in UTC. Personal approvals still name the person; saved activity is unchanged. Fixed gates and critical unresolved decisions still ask. Merge remains your decision. proof is fixed: an edit cannot reword or skip it. A repository can explicitly fix an approval too. security is shipped but in no list: a job typed with the word `security` (`fix the login, security`) gains it after review, and a pipeline file names it to run it. A pipeline file's `max:` replaces a shipped step's rounds; a step of the file's own, under a name codeaf does not ship, runs once unless it says `max:`. +## approval decisions — you approved the plan, and later gates still ask + +The job keeps your latest settled approval and who approved it while later +steps or gates wait, including when you open the app again. Approving +a plan does not approve a later gate: that gate still waits for its own answer. +When another approval begins, the job shows that decision with its own undo +window. Activity keeps the earlier approval receipts. + ## the factory's own rules — rules for every repository, instructions, protected paths, ask first, never, "don't touch auth without checking with me" A repository's rules live in its `.codeaf/factory.yaml` (`rules:` and `start: {ask_first, never}`); in the app, the factory's Steps tab, in its Repository view, edits them with its pipeline, as one pull request, and a repository with no file yet gets one. A change to a file the repository already has reaches the next start; a file added where there was none (the first `factory.yaml`, or a `CODEOWNERS`) is read within five minutes of its merge. **Use on this factory** runs the edit at once instead (see the next section). The factory can also keep **its own rules**, for every repository it watches, so a rule like "don't touch auth without checking with me" is said once instead of copied into each repository. They are kept by the factory itself (on a team, the shared server, not one person's machine), and only its operator can change them; everyone signed in can read them. In the app they are the Global view of the factory's **Steps** tab. @@ -494,7 +502,7 @@ Start rules: jobs like this one that cost more than `max_cost` ask first (`jobs Look closer on a job asks a model to read the issue again with read-only tools on the factory's own checkout: list a folder, search, read lines of a file. It answers the files it expects to change (each with a short why), a three-step plan, what the issue does not say (as questions for its author), the tests that would prove it, and the complexity again, each factor citing a file and line. Every path shown exists in the repository; a path the model named that does not exist is dropped and counted. A look never writes a price. -A look stops asking for files at 12 tool calls, one minute or $0.10, whichever comes first, and then answers once. The button shows what a typical look costs on this factory (`Look closer ≈ $0.02`, never under `≈ $0.01`) once one has run; its cost goes on Spending as `factory scout`. The same issue at the same commit answers the kept look for free; an edited issue shows the look as older and offers a refresh. +A look stops asking for files at 12 tool calls, one minute or $0.10, whichever comes first, and then answers once. The button shows what a typical look costs on this factory (`Look closer ≈ $0.02`, never under `≈ $0.01`) once one has run; its cost goes on Spending as `factory scout`. AI read and Look closer charges appear on Spending even when the model call fails; each charge is counted once. The same issue at the same commit answers the kept look for free; an edited issue shows the look as older and offers a refresh. Its complexity replaces the read's, and its questions become the job's when the job has none. Credential-looking files (`.env`, keys, `.ssh/`) are never listed or read, and the repository's text is treated as data, so an instruction inside a file or the issue is not followed. @@ -685,3 +693,10 @@ uses this same rule. The raise button accepts $5, $10, $20, $30 or exactly the job's next budget. Other amounts are refused. Stop keeps the recorded spend and work. + +## Why does Spending show Factory AI — the third part of today's money + +Today's figure on Spending counts every model call on this computer's ledger, and it moves as each one lands, not only when a chat turn ends. + +The line under it splits today into factory jobs, `Factory AI` and your chats. `Factory AI` is what Factory spent outside any job: triage, scouts, and the AI on Factory's pages (asking about a repo, Build with AI, AI views). A call made for a job counts under factory jobs, whatever made it. These calls count against the one daily limit too. + diff --git a/internal/manual/chat/models-and-cost.md b/internal/manual/chat/models-and-cost.md index 96d224c534..4ec46c0e85 100644 --- a/internal/manual/chat/models-and-cost.md +++ b/internal/manual/chat/models-and-cost.md @@ -1845,6 +1845,15 @@ say, for example, `2 calls the provider charged for and could not be priced`. At say nothing. Settings→Spending also shows missing receipts learned during this process. Receipt workers exit when their queue is empty and start again when another receipt arrives. +Factory jobs and their page helpers also request receipts for successful answers +without usage. After a failed stream, the final receipt replaces interim usage +for billing: a $0.10 interim figure followed by a $0.12 receipt charges $0.12 once. +Missing receipts keep their job attribution. Job usage and Spending expose the +missing price as unbilled, never as a measured $0. A saturated ledger queue writes +through instead of discarding charges; a caller never waits on the disk. +Filesystem write failures still appear as missing records. + + ## A reply that never finished — the turn ran for half an hour, codeaf looked frozen, nothing happened for ages, the model kept writing and never stopped The two clocks above are both about **silence**. A reply that keeps producing a token every diff --git a/internal/provider/billing.go b/internal/provider/billing.go index 222aed5191..9345ed14ab 100644 --- a/internal/provider/billing.go +++ b/internal/provider/billing.go @@ -118,6 +118,7 @@ type billingContextKey struct{} type reconcileContextKey struct{} type receiptPendingContextKey struct{} type unmeteredReceiptsContextKey struct{} +type finalReceiptsContextKey struct{} // WithBilling arms one piece of work's banking. Like the transcript sink it // belongs to the work rather than to the client, because one client serves @@ -166,6 +167,18 @@ func WithUnmeteredReceipts(ctx context.Context) context.Context { return context.WithValue(ctx, unmeteredReceiptsContextKey{}, true) } +// WithFinalReceipts asks for a failed call's final receipt even when its stream +// carried interim usage. Successful unmetered answers also seek a receipt. +// Callers bank successful usage themselves and must not also install a billing sink. +func WithFinalReceipts(ctx context.Context) context.Context { + return context.WithValue(WithUnmeteredReceipts(ctx), finalReceiptsContextKey{}, true) +} + +func finalReceiptsFrom(ctx context.Context) bool { + armed, _ := ctx.Value(finalReceiptsContextKey{}).(bool) + return armed +} + // unmeteredReceiptsFrom reports whether [WithUnmeteredReceipts] armed ctx. func unmeteredReceiptsFrom(ctx context.Context) bool { if ctx == nil { @@ -268,7 +281,7 @@ func (c *Client) billAnswered(ctx context.Context, model string, response *ai.Re // A direct client's receipt bearer belongs to its original service. // Preserve the whole-answer boundary until recovery can carry the // overflow attempt's own credential to its different host. - if c.config.Direct { + if c.config.Direct && !finalReceiptsFrom(ctx) { return } c.settle(ctx, model, response, receiptUnmeteredReason, answerBytes, traces...) diff --git a/internal/provider/receipt.go b/internal/provider/receipt.go index 09623c2fa1..534fdb77f9 100644 --- a/internal/provider/receipt.go +++ b/internal/provider/receipt.go @@ -104,6 +104,8 @@ type receiptWork struct { queued time.Time usage func(int, int, bool, bool) base string + // requirePrice keeps token-only receipts unknown for final-price consumers. + requirePrice bool // attempt is the send this receipt prices, captured when the receipt was // owed (attemptobserver.go). It is NOT read from the trace at arrival: by // then the call may be on a later send, or over. @@ -175,7 +177,7 @@ func (c *Client) settle(ctx context.Context, model string, response *ai.Response // bank exactly once, but it cannot prove the completed price of an observed // send. Only that send's final generation receipt can settle its token. handle := attemptOf(traces...) - bankedUsage := response != nil && response.Usage != nil + bankedUsage := response != nil && response.Usage != nil && !finalReceiptsFrom(ctx) if bankedUsage { if handle == nil { c.bill(ctx, model, response, traces...) @@ -190,6 +192,20 @@ func (c *Client) settle(ctx context.Context, model string, response *ai.Response // calling the router's route on the vendor would send that vendor a // request and bearer for a fact this client already knows cannot be there. if c.config.Direct { + if finalReceiptsFrom(ctx) { + if sink := reconcileFrom(ctx); sink != nil { + result := Reconciled{Billed: Billed{Model: model, Billing: c.answeredBilling(traces...)}, Reason: reason} + if response != nil && response.Usage != nil && response.Usage.Cost != nil { + result.Found = true + result.Cost = *response.Usage.Cost + result.PromptTokens = response.Usage.PromptTokens + result.CompletionTokens = response.Usage.CompletionTokens + sink(result) + } else if answerBytes > 0 { + sink(result) + } + } + } handle.unknown() return } @@ -231,7 +247,7 @@ func (c *Client) settle(ctx context.Context, model string, response *ai.Response if len(traces) > 0 && traces[0] != nil { attemptID = traces[0].attemptID } - work := receiptWork{result: result, sink: sink, base: base, attempt: handle, usage: telemetry.CaptureUsageRecorderWithPresence(telemetry.UsageDimensions{ + work := receiptWork{result: result, sink: sink, base: base, attempt: handle, requirePrice: finalReceiptsFrom(ctx), usage: telemetry.CaptureUsageRecorderWithPresence(telemetry.UsageDimensions{ RoutingProvider: c.usageRoutingProvider(base), ModelFamily: telemetry.ModelFamily(result.Model), UsageStatus: "reported", AccountingSource: "provider", ReceiptID: usageReceiptID(base, result.Ref, attemptID), @@ -245,7 +261,10 @@ func (c *Client) settle(ctx context.Context, model string, response *ai.Response // answered only after the sink has banked it, so a caller waiting for its // receipts cannot see zero owed while money is between the two // ([ReceiptPending]). - if pending := receiptPendingFrom(ctx); !bankedUsage && pending != nil { + // A finished answer that only lacks usage doesn't hold up the job's next step + // for its receipt: the job has moved on, and the row lands when it is answered. + moved := finalReceiptsFrom(ctx) && result.Reason == receiptUnmeteredReason + if pending := receiptPendingFrom(ctx); !bankedUsage && !moved && pending != nil { done := pending() work.sink = func(answer Reconciled) { defer done() @@ -348,6 +367,11 @@ func (c *Client) reconcile(work receiptWork) { return } if found { + if work.requirePrice && read.financial.State != FinancialKnown { + work.attempt.unknown() + work.sink(result) + return + } work.attempt.financial(last) billed.Node = result.Node billed.Model = result.Model diff --git a/internal/session/clientdoor.go b/internal/session/clientdoor.go index 7824d728d9..08b56dcc05 100644 --- a/internal/session/clientdoor.go +++ b/internal/session/clientdoor.go @@ -795,6 +795,14 @@ func (a *Agent) completeWithNamedModel(ctx context.Context, purpose callPurpose, // internal/session that spells it, here and for the three roads that carry a // client of their own. ctx = withPurpose(ctx, purpose) + // Factory calls bank returned answers at their callers. Failed attempts and + // unmetered answers need the final receipt instead, including after Stop. + if a.factoryJobName() != "" && provider.BillingSinkFrom(ctx) == nil { + ctx = provider.WithFinalReceipts(provider.WithReconcile(ctx, a.reconciled)) + if pending := a.config.JobPendingSpend; pending != nil { + ctx = provider.WithReceiptPending(ctx, pending) + } + } ctx = provider.WithDiscardedUsage(ctx, func(served string, tag string, response *ai.Response) { // Full billing owners (program model APIs) already bank every transport // attempt. Ordinary session callers bank only the returned answer. diff --git a/internal/session/usage_ledger.go b/internal/session/usage_ledger.go index 2bf04d9883..abd3f9ee10 100644 --- a/internal/session/usage_ledger.go +++ b/internal/session/usage_ledger.go @@ -44,16 +44,10 @@ package session // (orchestrate.go's [orchestrateExec.spend]) journals its own calls here and // folds home silently. It folded through the auxiliary door until issue // #168, and the day this file's rail reads was double on every one of them. -// - IT NEVER BLOCKS A TURN, AND THE TURN'S OWN GOROUTINE NEVER TOUCHES THE -// DISK. [RecordUsage] serializes the row and hands it to a bounded queue -// that one background writer per ledger drains ([usageWriter]); a full -// queue drops the row rather than waiting on it, and so does a write that -// fails. A home directory on a stalled mount therefore costs a spending -// record and never a person's turn — which the older shape, an -// open-write-close under a process-wide mutex, could not promise. AND THE -// DROP IS COUNTED WHERE IT HAPPENS ([UsageDrops]), because a gap nobody can -// see is a bill that reads as smaller than it was: the spend surfaces say -// how many records went missing rather than quietly under-reporting. +// - A FULL QUEUE DOES NOT DISCARD A CHARGE. The turn's goroutine never +// touches disk and never waits. Overflow is kept for the ledger's one +// writer. Flush and shutdown include those writes under their existing +// ceiling. Filesystem failures remain counted by [UsageDrops]. // - A LINE THAT SPENT NOTHING IS NOT WRITTEN. The emptiness law applied to a // file: a call whose usage block or later receipt carries no cost and no // tokens leaves no priced row — an explicit unbilled marker records a @@ -526,11 +520,8 @@ func (w *laneWitness) answered(seen hedgeSeen, output int) laneFacts { // disk, and a ledger row is written at most once per model call, so this writer // wakes on the row itself and there is nothing to batch. -// usageQueueDepth is how many serialized rows one ledger's queue holds before a -// row is DROPPED rather than waited on. Two hundred and fifty-six is far more -// than the handful of calls that can be in flight at once in this process, so -// the queue only fills when the disk behind it has stopped answering — which is -// exactly the case the drop exists for. +// usageQueueDepth bounds the ordinary background queue. Overflow writes through +// independently rather than dropping a charge. const usageQueueDepth = 256 // usageFlushLimit is the longest [FlushUsage] may wait for the rows in front of @@ -553,25 +544,16 @@ type usageWrite struct { // usageWriter is one ledger file's background writer. type usageWriter struct { - queue chan usageWrite - stopped chan struct{} - beforeWrite func() + queue chan usageWrite + stopped chan struct{} + beforeWrite func() + overflowMu sync.Mutex + overflow map[chan struct{}]struct{} + overflowRows []usageOverflowRow } -// usageDropped is how many rows this process could not get onto a ledger — a -// queue full because the disk stopped answering, a directory that would not be -// made, a write that failed. It is process-wide rather than per-path because -// every reader of it asks one question — "is what I am looking at short, and by -// how much" — and a person reading a spending page does not hold a mental map -// of which ledger a row was bound for. -// -// IT EXISTS BECAUSE THE DROP ITSELF IS RIGHT AND THE SILENCE WAS NOT. The -// bargain above stands: a spending record is worth less than the turn that -// earned it, so the row gives way. But a ledger that quietly loses rows reads -// as a machine that spent less, which is the flattering direction and the one -// direction a bill must never be wrong in. So the drop is counted and the spend -// surfaces say so ([UsageDrops]). [UnbilledCalls] makes the same sentence about -// calls whose provider receipt could not be had. +// usageDropped counts serialization and filesystem failures across all ledgers. +// A full queue no longer contributes: overflow writes through instead. var usageDropped atomic.Int64 // unbilledCalls is process-wide for the same reason [usageDropped] is: every @@ -626,14 +608,34 @@ func usageWriterForLocked(path string) *usageWriter { return writer } writer := &usageWriter{ - queue: make(chan usageWrite, usageQueueDepth), - stopped: make(chan struct{}), + queue: make(chan usageWrite, usageQueueDepth), + stopped: make(chan struct{}), + overflow: make(map[chan struct{}]struct{}), } usageWriters[path] = writer go writer.run(path) return writer } +// usageOverflowRow is a row waiting for the overflow writer; done closes once +// it was written or counted as dropped. +type usageOverflowRow struct { + line []byte + done chan struct{} +} + +// pendingOverflow captures the writes already admitted without waiting under a +// lock. Later writes belong to a later flush; shutdown admits no later writes. +func (w *usageWriter) pendingOverflow() []chan struct{} { + w.overflowMu.Lock() + defer w.overflowMu.Unlock() + pending := make([]chan struct{}, 0, len(w.overflow)) + for done := range w.overflow { + pending = append(pending, done) + } + return pending +} + // run drains the queue forever, holding ONE descriptor open across rows. // // The descriptor is opened lazily and dropped on the first write that fails, so @@ -649,11 +651,7 @@ func (w *usageWriter) run(path string) { _ = file.Close() } }() - for work := range w.queue { - if work.done != nil { - close(work.done) - continue - } + write := func(line []byte) { if w.beforeWrite != nil { w.beforeWrite() } @@ -661,19 +659,33 @@ func (w *usageWriter) run(path string) { file = openUsageLedger(path) if file == nil { dropUsageRow() - continue + return } } // ONE write per complete line, so O_APPEND's atomic offset covers the // whole row — which is also what lets [UsageCache] trust that the bytes // before the last newline are whole lines, and what keeps two processes // appending to one ledger from interleaving halves of two rows. - if _, err := file.Write(work.line); err != nil { + if _, err := file.Write(line); err != nil { dropUsageRow() _ = file.Close() file = nil } } + for work := range w.queue { + if work.done != nil { + w.writeOverflow(write) + close(work.done) + continue + } + if work.line != nil { + write(work.line) + } + if len(w.queue) == 0 { + w.writeOverflow(write) + } + } + w.writeOverflow(write) } // openUsageLedger opens one ledger for appending, creating its directory, and @@ -699,7 +711,7 @@ func openUsageLedger(path string) *os.File { // a test uses instead of sleeping — internal/history's Close is the same door // for the same reason. // -// IT IS THE ONE PLACE IN THIS FILE THAT WAITS, deliberately: a caller asking for +// IT WAITS FOR THE WRITES, deliberately: a caller asking for // a flush is asking to be told when the writing is done, and a flush that gave // up early would be an answer about nothing. Nothing on a turn path may call it. // @@ -781,18 +793,22 @@ func detachUsageWriter(path string) *usageWriter { // would panic the process). The send is non-blocking for [RecordUsage]'s // reason: a full queue is a writer stalled on a disk that is not answering, and // a flush will not wait on a marker that cannot land, no more than on a row that -// cannot. Only the markers that were accepted are waited on. +// cannot. An unaccepted marker makes the bounded flush report false. func enqueueUsageFlush() []chan struct{} { usageWritersMu.Lock() defer usageWritersMu.Unlock() dones := make([]chan struct{}, 0, len(usageWriters)) for _, writer := range usageWriters { + dones = append(dones, writer.pendingOverflow()...) done := make(chan struct{}) select { case writer.queue <- usageWrite{done: done}: - dones = append(dones, done) default: + // The queue is full, so its writer will reach the kept rows, and + // it takes them only once everything queued has been written. + writer.addBarrier(done) } + dones = append(dones, done) } return dones } @@ -867,17 +883,66 @@ func RecordUsage(path string, line UsageLine) { dropUsageRow() return } - // THE ENQUEUE IS NON-BLOCKING AND THE ROW IS THE THING THAT GIVES WAY. A - // full queue means the writer is stuck on a disk that is not answering, and - // a turn made to wait behind it would be this file's fourth rule broken to - // save a record of what the turn cost. + // Overflow is kept for the writer, so a saturated queue never discards a + // charge and the caller still never waits on the disk. usageWritersMu.Lock() - defer usageWritersMu.Unlock() writer := usageWriterForLocked(path) + work := usageWrite{line: append(payload, '\n')} select { - case writer.queue <- usageWrite{line: append(payload, '\n')}: + case writer.queue <- work: + usageWritersMu.Unlock() + return default: - dropUsageRow() + } + writer.addOverflow(work.line) + usageWritersMu.Unlock() +} + +// addOverflow keeps a row the full queue couldn't take. The queue's own writer +// takes it once the queue is empty, so rows stay in order and a stalled mount +// holds one thread. A full queue means that writer has work, so it will reach +// this row. +func (w *usageWriter) addOverflow(line []byte) { + done := make(chan struct{}) + w.overflowMu.Lock() + defer w.overflowMu.Unlock() + w.overflow[done] = struct{}{} + w.overflowRows = append(w.overflowRows, usageOverflowRow{line: line, done: done}) + w.wakeForOverflow() +} + +// wakeForOverflow covers a writer that emptied the queue between the caller +// seeing it full and keeping the row: an empty item sends it back to the kept +// rows. If the queue is full again, the writer has work and will reach them. +func (w *usageWriter) wakeForOverflow() { + select { + case w.queue <- usageWrite{}: + default: + } +} + +// addBarrier closes done once the rows kept before it have been written. +func (w *usageWriter) addBarrier(done chan struct{}) { + w.overflowMu.Lock() + defer w.overflowMu.Unlock() + w.overflowRows = append(w.overflowRows, usageOverflowRow{done: done}) + w.wakeForOverflow() +} + +// writeOverflow writes the rows kept so far. +func (w *usageWriter) writeOverflow(write func([]byte)) { + w.overflowMu.Lock() + rows := w.overflowRows + w.overflowRows = nil + w.overflowMu.Unlock() + for _, row := range rows { + if row.line != nil { + write(row.line) + } + w.overflowMu.Lock() + delete(w.overflow, row.done) + w.overflowMu.Unlock() + close(row.done) } } @@ -1273,6 +1338,7 @@ func (a *Agent) recordUnbilledReceipt(model string) { RecordUnbilledCall(path, UsageLine{ Session: owner, Model: model, Task: usageTaskID(a.config.taskID), Root: a.config.rootSession, Automation: a.config.automationID, Workspace: a.config.Workspace, + FactoryJob: a.factoryJobName(), }) } From 909c9dbedf180fe3db358de713f760e081955898 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:37:10 -0500 Subject: [PATCH 20/28] spend: today's figure moves with every ledger row and names Factory AI (#2275) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: reproduce missing factory charge settlement Co-Authored-By: Claude Opus 5.5 * fix: settle factory charges from final provider receipts Interim stream usage suppressed reconciliation even when no session billing sink could consume it. Factory calls now recover the final receipt and retain unknown-price ownership. Write through ledger overflow without holding a turn indefinitely. Memory keeps its existing billing path. Co-Authored-By: Claude Opus 5.5 * fix: snapshot pending overflow writes across ledger flushes Capture completion barriers without racing WaitGroup reuse. A full queue cannot certify a successful flush; it reports false under the existing deadline instead. Co-Authored-By: Claude Opus 5.5 * test: keep unpriced factory receipts visibly unknown Co-Authored-By: Claude Opus 5.5 * fix: keep factory token-only receipts unbilled A receipt with token counts but no provider price cannot establish a dollar charge. Final-price consumers now retain the existing unknown-price marker instead of treating those counts as a settled bill. Other receipt consumers remain unchanged. Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: a ledger row with no turn must move today's figure Red: spend.changed fires only when a chat turn ends, so Factory triage, scouts and page AI never reach the page. Co-Authored-By: Claude Opus 5.5 * spend: today's figure moves when any ledger row lands, not only when a turn ends Co-Authored-By: Claude Opus 5.5 * spend: a full usage queue never makes a turn wait The 100 ms per overflow row added up on a stalled disk and broke the ledger's turn law. One background writer takes overflow rows instead. Co-Authored-By: Claude Opus 5.5 * spend: overflow rows move today's figure too Co-Authored-By: Claude Opus 5.5 * spend: the ledger's one writer takes overflow rows, in order Co-Authored-By: Claude Opus 5.5 * spend: finish the merge of the ledger writer Co-Authored-By: Claude Opus 5.5 * spend: format the ledger writer Co-Authored-By: Claude Opus 5.5 * spend: format the ledger writer Co-Authored-By: Claude Opus 5.5 * spend: drop the merge's duplicated writer loop Co-Authored-By: Claude Opus 5.5 * spend: a full queue's flush waits for the kept rows instead of reporting false Co-Authored-By: Claude Opus 5.5 * spend: spend.today names Factory AI, so triage, scouts and page AI aren't called chats Co-Authored-By: Claude Opus 5.5 * changelog: name what changed Co-Authored-By: Claude Opus 5.5 * spend: wake the ledger writer when a row is kept after it emptied the queue Co-Authored-By: Claude Opus 5.5 * spend: an answer with no usage doesn't hold the job's next step for its receipt Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/app.go | 27 ++++- cmd/codeaf/appspendreport_test.go | 98 +++++++++++++++++++ .../unreleased/2275-spend-today-moves.md | 15 +++ internal/appwire/wire.go | 7 +- internal/session/daily_spend.go | 19 +++- internal/session/usage_ledger.go | 51 ++++++++++ 6 files changed, 211 insertions(+), 6 deletions(-) create mode 100644 docs/changes/unreleased/2275-spend-today-moves.md diff --git a/cmd/codeaf/app.go b/cmd/codeaf/app.go index 90a8596e1e..6c638632a7 100644 --- a/cmd/codeaf/app.go +++ b/cmd/codeaf/app.go @@ -178,6 +178,7 @@ func runApp(args []string) error { if *turnLong > 0 { backend.turnLong = *turnLong } + defer backend.watchSpendRows()() // The crews are watched from the start: a manager's start or a member // woken by its manager must not wait for the window to ask for crews. backend.memoryStore = func() (*store.Store, error) { @@ -716,6 +717,9 @@ type appBackend struct { dayAuth appDayAuthority // today replaces the usage ledger's figure for today in a test. today func(time.Time) float64 + // spendRowsPending holds one read of today's figure for rows written + // together (watchSpendRows). + spendRowsPending atomic.Bool // turnLong is how long a reply runs before turn.long ([appTurnLong]). turnLong time.Duration // inbox is every open ask as one list (appinbox.go); emit is its tap. @@ -4250,7 +4254,15 @@ func (b *appBackend) SpendToday() (appwire.Spend, error) { if err != nil { return appwire.Spend{}, appEngineError(err) } - return appwire.Spend{TodayUSD: spent, DayCapUSD: appFigure(limit)}, nil + spend := appwire.Spend{TodayUSD: spent, DayCapUSD: appFigure(limit)} + if b.today == nil { + reading, err := session.ReadDailySpend(session.UsageLedgerPath(), now) + if err != nil { + return appwire.Spend{}, appEngineError(err) + } + spend.FactoryAIUSD = appFigure(reading.FactoryAIUSD) + } + return spend, nil } // appSpendSettle is how long after a turn ends its spending is read again: the @@ -4283,6 +4295,19 @@ func (b *appBackend) spendMoved(after func(appwire.Spend)) { }) } +// watchSpendRows says spend.changed for ledger rows no turn ended with. Rows +// that arrive together cost one read. The answer stops the watching. +func (b *appBackend) watchSpendRows() (stop func()) { + return session.OnUsageWritten(func() { + if b.spendRowsPending.CompareAndSwap(false, true) { + time.AfterFunc(appSpendSettle, func() { + b.spendRowsPending.Store(false) + b.spendMoved(nil) + }) + } + }) +} + // ListTasks reads the agent's tasks off the disk whether or not its // conversation is held — its own checkpoint first, which holds every task it // ever had, finished ones included, then its project's task index for any the diff --git a/cmd/codeaf/appspendreport_test.go b/cmd/codeaf/appspendreport_test.go index ccbd96d24c..beab85a36c 100644 --- a/cmd/codeaf/appspendreport_test.go +++ b/cmd/codeaf/appspendreport_test.go @@ -1,10 +1,13 @@ package main import ( + "math" + "path/filepath" "testing" "time" "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/session" ) @@ -109,3 +112,98 @@ func TestAppSpendReportOmitsWhatNoLineCarried(t *testing.T) { t.Fatalf("an unpriced, unnamed, unattributed line must stay absent: %+v / %+v", c, report.Totals) } } + +// A row made outside any job and any chat turn still reaches the page: the +// report has it, and spend.today moves without a turn ending. +func TestLedgerRowWithoutATurnMovesSpendToday(t *testing.T) { + path := filepath.Join(t.TempDir(), "usage.jsonl") + previous := appLedgerPath + appLedgerPath = func() string { return path } + t.Cleanup(func() { appLedgerPath = previous }) + t.Cleanup(func() { session.StopUsageWriter(path) }) + now := time.Now() + day := now.Format(time.DateOnly) + got := make(chan appwire.SpendChanged, 8) + b := &appBackend{now: time.Now, today: func(at time.Time) float64 { + usd, err := session.DailySpendUSD(path, at) + if err != nil { + t.Error(err) + } + return usd + }, emit: appEmitFunc(func(event string, payload any) { + if changed, ok := payload.(appwire.SpendChanged); ok && event == appwire.EventSpendChanged { + got <- changed + } + })} + t.Setenv(config.ProfileDirEnv, t.TempDir()) + b.dayAuth.profile, b.dayAuth.settled = config.ProfileDir(), true + t.Cleanup(b.watchSpendRows()) + rows := []session.UsageLine{ + {Role: "repo_ask", USD: .0061, Calls: 1, Session: "page", Input: 9}, + {Role: "pipeline_ai", USD: .0041, Calls: 1, Session: "page", Input: 9}, + {Role: "view_topic", USD: .0014, Calls: 1, Session: "page", Input: 9}, + {Role: "factory triage", USD: .1103, Calls: 1, Input: 9}, + {Role: "factory scout", USD: .0015, Calls: 1, Input: 9}, + } + want := 0.0 + for _, row := range rows { + row.At, row.Day, row.Model = now, day, "m/one" + want += row.USD + session.RecordUsage(path, row) + } + deadline := time.After(5 * time.Second) + last := 0.0 + for math.Abs(last-want) > 1e-9 { + select { + case changed := <-got: + last = changed.Spend.TodayUSD + case <-deadline: + t.Fatalf("spend.changed last read %.4f, ledger has %.4f", last, want) + } + } + report, err := b.SpendReport(appwire.SpendReportAsk{Filter: appwire.SpendLedgerFilter{From: day, To: day}}) + if err != nil { + t.Fatal(err) + } + if report.Totals.CostUSD == nil || math.Abs(*report.Totals.CostUSD-want) > 1e-9 || report.Totals.Records != len(rows) { + t.Fatalf("spend.report totals %+v, ledger has %.4f in %d rows", report.Totals, want, len(rows)) + } +} + +type appEmitFunc func(event string, payload any) + +func (f appEmitFunc) Emit(event string, payload any) { f(event, payload) } + +// Factory's own AI has its own figure on spend.today, so it isn't called a chat. +// A job's rows stay the job's, whatever their role. +func TestSpendTodayNamesFactoryAI(t *testing.T) { + t.Setenv(config.ProfileDirEnv, t.TempDir()) + path := session.UsageLedgerPath() + t.Cleanup(func() { session.StopUsageWriter(path) }) + b := &appBackend{now: time.Now} + b.dayAuth.profile, b.dayAuth.settled = config.ProfileDir(), true + now := time.Now() + for _, row := range []session.UsageLine{ + {Role: factoryTriageName, USD: .1103, Input: 9}, + {Role: factoryScoutName, USD: .0015, Input: 9}, + {Role: "repo_ask", USD: .0061, Input: 9}, + {Role: "pipeline_ai", USD: .0041, Input: 9}, + {Role: "view_topic", USD: .0014, Input: 9}, + {Role: "markreader", USD: .0109, Input: 9}, + {USD: .02, Input: 9}, + {Role: "factory triage", USD: .03, Input: 9, FactoryJob: "acme/api#1"}, + } { + row.At, row.Model = now, "m/one" + session.RecordUsage(path, row) + } + if !session.FlushUsage() { + t.Fatal("flush") + } + spend, err := b.SpendToday() + if err != nil { + t.Fatal(err) + } + if spend.FactoryAIUSD == nil || math.Abs(*spend.FactoryAIUSD-.1234) > 1e-9 || math.Abs(spend.TodayUSD-.1843) > 1e-9 { + t.Fatalf("spend.today = %+v", spend) + } +} diff --git a/docs/changes/unreleased/2275-spend-today-moves.md b/docs/changes/unreleased/2275-spend-today-moves.md new file mode 100644 index 0000000000..d7d5e3930e --- /dev/null +++ b/docs/changes/unreleased/2275-spend-today-moves.md @@ -0,0 +1,15 @@ +--- +kind: fixed +title: Today's spending moves with every ledger row and names Factory AI +pr: 2275 +surface: [engine] +invalidates: + - "Today's figure could stay stale until a chat turn ended, because spend.changed fired only then." + - "Factory triage, scouts and page AI read as your chats in today's split." +--- + +Today's figure on the Spending page moves when any ledger row lands. It used to +move only when a chat turn ended, so Factory triage, scouts and page AI left it +stale until the next chat. `spend.today` and `spend.changed` gain `factory_ai_usd`, +the part of today that triage, scouts and page AI spent, absent when zero. A row +tagged with a job stays with the job. diff --git a/internal/appwire/wire.go b/internal/appwire/wire.go index 9ae38ace99..eeefd44b65 100644 --- a/internal/appwire/wire.go +++ b/internal/appwire/wire.go @@ -911,8 +911,11 @@ const ( // Spend is today's spending on this machine against the daily cap, which is // null when there is none. type Spend struct { - TodayUSD float64 `json:"today_usd"` - DayCapUSD *float64 `json:"day_cap_usd"` + TodayUSD float64 `json:"today_usd"` + // FactoryAIUSD is the part of today that Factory's triage, scouts and page + // AI spent. Absent when there is none. It is inside TodayUSD. + FactoryAIUSD *float64 `json:"factory_ai_usd,omitempty"` + DayCapUSD *float64 `json:"day_cap_usd"` } // AskOption is one answer an ask offers. diff --git a/internal/session/daily_spend.go b/internal/session/daily_spend.go index 276a6ea76a..35b69fcdc0 100644 --- a/internal/session/daily_spend.go +++ b/internal/session/daily_spend.go @@ -15,7 +15,10 @@ import ( // DailySpendReading retains coverage gaps separately from the recorded figure. // Legacy malformed rows have no trustworthy date and are never assigned one. type DailySpendReading struct { - USD float64 + USD float64 + // FactoryAIUSD is the part of USD that Factory's own AI spent: rows of the + // factoryAIRoles with no job on them. A job's rows are that job's. + FactoryAIUSD float64 Malformed, Oversized int } @@ -70,6 +73,13 @@ func ReadDailySpend(path string, now time.Time) (DailySpendReading, error) { // spendTally is one ledger's day as far as it has been read: the day summed, // the file read, how many bytes of whole lines, and the last of those bytes, // which prove the file still holds what was read. +// factoryAIRoles are the ledger roles of Factory's own AI: triage, scouts and the +// page AI. They aren't chats, so they don't count under "your chats". +var factoryAIRoles = map[string]bool{ + "factory triage": true, "factory scout": true, + "repo_ask": true, "pipeline_ai": true, "view_topic": true, +} + type spendTally struct { day string file os.FileInfo @@ -100,7 +110,7 @@ func (t *spendTally) read(file *os.File, info os.FileInfo) error { if _, err := file.Seek(t.offset, io.SeekStart); err != nil { return err } - usd := t.reading.USD + usd, factoryAI := t.reading.USD, t.reading.FactoryAIUSD stats, err := scanUsageLines(file, info.Size()-t.offset, func(_ int64, line UsageLine) error { if UsageLineDay(line).Format(usageDayLayout) != t.day { return nil @@ -109,6 +119,9 @@ func (t *spendTally) read(file *os.File, info os.FileInfo) error { return fmt.Errorf("spending ledger contains an invalid charge") } usd += line.USD + if factoryAIRoles[line.Role] && line.FactoryJob == "" { + factoryAI += line.USD + } if math.IsInf(usd, 0) { return fmt.Errorf("spending ledger total is not finite") } @@ -119,7 +132,7 @@ func (t *spendTally) read(file *os.File, info os.FileInfo) error { } t.file, t.offset = info, t.offset+stats.Bytes t.tail = ledgerTail(file, t.offset) - t.reading = DailySpendReading{USD: usd, Malformed: t.reading.Malformed + stats.Malformed, Oversized: t.reading.Oversized + stats.Oversized} + t.reading = DailySpendReading{USD: usd, FactoryAIUSD: factoryAI, Malformed: t.reading.Malformed + stats.Malformed, Oversized: t.reading.Oversized + stats.Oversized} return nil } diff --git a/internal/session/usage_ledger.go b/internal/session/usage_ledger.go index abd3f9ee10..10c218da24 100644 --- a/internal/session/usage_ledger.go +++ b/internal/session/usage_ledger.go @@ -670,8 +670,59 @@ func (w *usageWriter) run(path string) { dropUsageRow() _ = file.Close() file = nil + return + } + announceUsageWritten() + } + for work := range w.queue { + if work.done != nil { + w.writeOverflow(write) + close(work.done) + continue + } + if work.line != nil { + write(work.line) + } + if len(w.queue) == 0 { + w.writeOverflow(write) } } + w.writeOverflow(write) +} + +// usageWatchers are told after a row reaches the file. Spending happens outside +// chat turns (Factory triage, scouts, page AI), so a reader of today's figure +// can't wait for a turn's ending to know the ledger moved. +var usageWatchers struct { + sync.Mutex + next int + of map[int]func() +} + +// OnUsageWritten calls watch after each row reaches the ledger, on the writer's +// goroutine, so watch must not block. The answer stops the watching. +func OnUsageWritten(watch func()) (stop func()) { + usageWatchers.Lock() + defer usageWatchers.Unlock() + if usageWatchers.of == nil { + usageWatchers.of = map[int]func(){} + } + id := usageWatchers.next + usageWatchers.next++ + usageWatchers.of[id] = watch + return func() { + usageWatchers.Lock() + defer usageWatchers.Unlock() + delete(usageWatchers.of, id) + } +} + +func announceUsageWritten() { + usageWatchers.Lock() + defer usageWatchers.Unlock() + for _, watch := range usageWatchers.of { + watch() + } for work := range w.queue { if work.done != nil { w.writeOverflow(write) From af0fe9a9cb8e80509b90d33b99329ac890e8f393 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:37:23 -0500 Subject: [PATCH 21/28] spend: Factory helper calls count on the job and the page (#2268) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: expose lost Factory page charges on failed generations Co-Authored-By: Claude Opus 5.5 * test: give Factory scout a real checkout in spending fixture Co-Authored-By: Claude Opus 5.5 * fix: bank Factory page charges before handling failed generations Record provider billing events because failed generations may return no response. Use the returned response only when no event was banked, so success counts once. Co-Authored-By: Claude Opus 5.5 * test: require an exact ledger-owner inventory for production model sends Co-Authored-By: Claude Opus 5.5 * test: inventory current ledger owners and explicit model-spend audit debt Co-Authored-By: Claude Opus 5.5 * fix: preserve returned usage and include model sends in initializers Keep the returned response price-presence fact instead of synthesizing an explicit zero. Enumerate declaration initializers as well as named functions and prove package-scope mutations are rejected. Co-Authored-By: Claude Opus 5.5 * fix: a failed scout reply counts in its limit, and charges may arrive concurrently Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_page_spend_test.go | 88 ++++++++ cmd/codeaf/appfactory_spend_law_test.go | 206 ++++++++++++++++++ cmd/codeaf/factory_scout.go | 5 +- cmd/codeaf/factory_triage.go | 60 +++-- cmd/codeaf/testdata/model_spend_owners.tsv | 83 +++++++ .../unreleased/2268-factory-page-charges.md | 13 ++ internal/factory/scout/scout.go | 2 +- 7 files changed, 440 insertions(+), 17 deletions(-) create mode 100644 cmd/codeaf/appfactory_page_spend_test.go create mode 100644 cmd/codeaf/appfactory_spend_law_test.go create mode 100644 cmd/codeaf/testdata/model_spend_owners.tsv create mode 100644 docs/changes/unreleased/2268-factory-page-charges.md diff --git a/cmd/codeaf/appfactory_page_spend_test.go b/cmd/codeaf/appfactory_page_spend_test.go new file mode 100644 index 0000000000..d8d3e49cc3 --- /dev/null +++ b/cmd/codeaf/appfactory_page_spend_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "os/exec" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/session" +) + +// A page's failed generation costs the same as a successful one. Spending +// must show each provider charge exactly once through the real app wire. +func TestAppFactoryPageChargesReachSpendingOnce(t *testing.T) { + for _, method := range []string{"factory.read", "factory.scout"} { + for _, failed := range []bool{true, false} { + t.Run(fmt.Sprintf("%s/failed=%v", method, failed), func(t *testing.T) { + s := newLocalSetup(t) + var calls atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + fmt.Fprint(w, `{"data":[]}`) + return + } + calls.Add(1) + text := `{"read":"A read","rubric":{"scope":{"score":0},"clarity":{"score":0},"coupling":{"score":0},"proof":{"score":1},"novelty":{"score":0}}}` + if method == "factory.scout" { + text = `{"answer":{"plan":["Check retries"]}}` + } + finish := "stop" + if failed { + finish = "error" + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"model": readLoopsPin, "choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": text}, "finish_reason": finish}}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": 0.01}}) + })) + t.Cleanup(server.Close) + readLoopsProfile(t, &readLoopsWorld{model: server}, readLoopsPin) + st := s.start() + dir := t.TempDir() + gitCheckout(t, dir, "acme/api") + if out, err := exec.Command("git", "-C", dir, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.com", "commit", "--allow-empty", "-qm", "fixture").CombinedOutput(); err != nil { + t.Fatalf("checkout: %v: %s", err, out) + } + if err := s.backend.local.host.floor.SetCheckout("acme/api", dir); err != nil { + t.Fatal(err) + } + item, err := s.backend.local.host.floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Check retries", Kind: factory.KindIssue, State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + params := map[string]any{"factory": st.Factory, "job": item.ID} + if method == "factory.read" { + delete(params, "job") + params["jobs"] = []int{item.ID} + var got api.ReadResult + s.must(method, params, &got) + if failed && len(got.Failed) != 1 || !failed && len(got.Read) != 1 { + t.Fatalf("read outcome: %+v", got) + } + } else { + _, code, _ := s.call(method, params) + if (code != "") != failed { + t.Fatalf("scout failure=%v, code=%s", failed, code) + } + } + if !session.FlushUsage() { + t.Fatal("ledger did not flush") + } + var got appwire.SpendLedgerResult + today := time.Now().Format("2006-01-02") + s.must("spend.ledger", map[string]any{"from": today, "to": today}, &got) + want := float64(calls.Load()) * .01 + if calls.Load() == 0 || got.Totals.Calls != int(calls.Load()) || got.Totals.CostUSD == nil || math.Abs(*got.Totals.CostUSD-want) > 1e-9 { + t.Fatalf("charged page calls=%d cost=%.2f; Spending calls=%d cost=%v: every charge must appear once", calls.Load(), want, got.Totals.Calls, got.Totals.CostUSD) + } + }) + } + } +} diff --git a/cmd/codeaf/appfactory_spend_law_test.go b/cmd/codeaf/appfactory_spend_law_test.go new file mode 100644 index 0000000000..94dd64769a --- /dev/null +++ b/cmd/codeaf/appfactory_spend_law_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// The inventory is exact in both directions: a new caller needs a ledger +// owner or an explicit debt entry; a removed caller must leave the debt list. +// Like the provider funnel and session purpose laws, this parses production +// sources rather than relying on which branches a fixture happens to run. +func TestFactoryModelSendsHaveLedgerOwners(t *testing.T) { + root := filepath.Join("..", "..") + calls, err := modelSpendCallers(root) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile("testdata/model_spend_owners.tsv") + if err != nil { + t.Fatal(err) + } + entries := map[string]string{} + for _, row := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if row == "" || strings.HasPrefix(row, "#") { + continue + } + key, owner, ok := strings.Cut(row, "\t") + if !ok || !(strings.HasPrefix(owner, "owner: ") || strings.HasPrefix(owner, "exception: ")) { + t.Fatalf("invalid ledger owner: %s", row) + } + if _, exists := entries[key]; exists { + t.Fatalf("duplicate ledger owner: %s", key) + } + entries[key] = owner + } + for _, violation := range modelSpendViolations(calls, entries) { + t.Error(violation) + } + if len(calls) < 50 { + t.Fatalf("only %d send callers found; source walk is incomplete", len(calls)) + } + // Plant a production helper in a copy. A new unowned send must be rejected. + dir := t.TempDir() + source := `package fixture + func unowned(c Client) { c.CompleteWithMessages(ctx, nil) } + var unownedClosure = func(c Client) { c.CompleteWithMessages(ctx, nil) } + var unownedInitializer = c.CompleteWithMessages(ctx, nil) + ` + if err := os.WriteFile(filepath.Join(dir, "unowned.go"), []byte(source), 0600); err != nil { + t.Fatal(err) + } + mutant, err := modelSpendCallers(dir) + if err != nil { + t.Fatal(err) + } + for key, at := range calls { + mutant[key] = at + } + violations := modelSpendViolations(mutant, entries) + found := false + for _, v := range violations { + if strings.Contains(v, "unowned.go:unowned:CompleteWithMessages") { + found = true + t.Log("mutation rejected: " + v) + } + } + if !found { + t.Fatal("ledger law accepted an unowned production helper") + } + if len(mutant) != len(calls)+3 { + t.Fatal("ledger law accepted package-scope model sends") + } + // Removing an exception without retiring its entry must also fail. + for key := range entries { + if !strings.HasPrefix(entries[key], "exception: ") { + continue + } + delete(mutant, key) + stale := false + for _, v := range modelSpendViolations(mutant, entries) { + if v == "stale ledger entry: "+key { + stale = true + } + } + if !stale { + t.Fatalf("ledger law accepted stale exception %s", key) + } + break + } +} + +func modelSpendViolations(calls, entries map[string]string) []string { + var out []string + for key, at := range calls { + if entries[key] == "" { + out = append(out, fmt.Sprintf("unowned model send: %s (%s)", key, at)) + } + } + for key := range entries { + if calls[key] == "" { + out = append(out, "stale ledger entry: "+key) + } + } + sort.Strings(out) + return out +} + +func modelSpendCallers(root string) (map[string]string, error) { + calls := map[string]string{} + verbs := map[string]bool{"CompleteWithMessages": true, "StreamComplete": true, "ExecuteToolCallLoop": true, "ParseDocument": true, "GenerateImage": true, "GenerateMusic": true, "GenerateVideo": true, "Speak": true, "Transcribe": true} + // The provider funnel's request functions are counted as well, including + // media requests and lane probes that do not call a completion method. + requests := map[string]bool{"send": true, "sendShaped": true, "sendRecovered": true, "sendRepaired": true, "resend": true, "do": true, "doEndpoint": true, "postJSON": true, "newHTTPRequest": true, "newHTTPRequestAt": true} + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if entry.Name() == ".git" || entry.Name() == "vendor" || entry.Name() == "node_modules" || entry.Name() == "testdata" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + set := token.NewFileSet() + file, err := parser.ParseFile(set, path, nil, 0) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + var bodies []struct { + name string + node ast.Node + } + for _, decl := range file.Decls { + switch decl := decl.(type) { + case *ast.FuncDecl: + if decl.Body == nil { + continue + } + name := decl.Name.Name + if decl.Recv != nil { + recv := decl.Recv.List[0].Type + if p, ok := recv.(*ast.StarExpr); ok { + recv = p.X + } + if id, ok := recv.(*ast.Ident); ok { + name = id.Name + "." + name + } + } + bodies = append(bodies, struct { + name string + node ast.Node + }{name, decl.Body}) + case *ast.GenDecl: + for _, spec := range decl.Specs { + if value, ok := spec.(*ast.ValueSpec); ok { + var names []string + for _, name := range value.Names { + names = append(names, name.Name) + } + bodies = append(bodies, struct { + name string + node ast.Node + }{"initializer." + strings.Join(names, ","), value}) + } + } + } + } + for _, body := range bodies { + ast.Inspect(body.node, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + verb := selector.Sel.Name + if !verbs[verb] && !(strings.HasPrefix(rel, "internal/provider/") && requests[verb]) { + return true + } + key := rel + ":" + body.name + ":" + verb + calls[key] = fmt.Sprintf("%s:%d", rel, set.Position(call.Pos()).Line) + return true + }) + } + return nil + }) + return calls, err +} diff --git a/cmd/codeaf/factory_scout.go b/cmd/codeaf/factory_scout.go index 4fff9532d4..991a2a113d 100644 --- a/cmd/codeaf/factory_scout.go +++ b/cmd/codeaf/factory_scout.go @@ -71,11 +71,12 @@ func factoryScoutChat(load func() (config.Config, error)) scout.Chat { sent[i] = judgeMessage(m.Role, m.Text) } ctx = provider.WithoutStream(provider.WithCallTag(ctx, factoryScoutName)) + ctx, recorded := factoryAIAccounting(ctx, settings, model, factoryScoutName, nil) response, err := client.CompleteWithMessages(ctx, sent) + usd := recorded(response) if err != nil { - return scout.Reply{}, err + return scout.Reply{USD: usd, Model: model}, err } - usd := recordFactoryAIUsage(settings, model, factoryScoutName, response) if response == nil || strings.TrimSpace(response.Text()) == "" { return scout.Reply{USD: usd, Model: model}, errors.New("the model answered nothing") } diff --git a/cmd/codeaf/factory_triage.go b/cmd/codeaf/factory_triage.go index 2fd5da7fa2..158226f761 100644 --- a/cmd/codeaf/factory_triage.go +++ b/cmd/codeaf/factory_triage.go @@ -158,13 +158,12 @@ func factoryTriageCallNoting(load func() (config.Config, error), noted func(usd ctx = provider.WithoutStream(provider.WithCallTag(ctx, factoryTriageName)) // The read asks at temperature 0, so the same issue read twice lands // in the same band. + ctx, recorded := factoryAIAccounting(ctx, settings, model, factoryTriageName, noted) response, err := client.CompleteWithMessages(ctx, []ai.Message{judgeMessage("user", prompt)}, ai.WithTemperature(0)) + recorded(response) if err != nil { return "", err } - if usd := recordFactoryTriageUsage(settings, model, response); usd > 0 && noted != nil { - noted(usd) - } if response == nil || strings.TrimSpace(response.Text()) == "" { return "", errors.New("the model answered nothing") } @@ -207,17 +206,6 @@ func factoryRoleModel(settings config.Config, role roles.Role) (string, error) { return strings.TrimSpace(settings.Model), nil } -// recordFactoryTriageUsage puts one read's cost on this machine's spend -// ledger: the role column says `factory triage`, the seat is the low one it -// ran on. The dollars are the provider's own receipt when one arrived, -// otherwise the model's published price over the tokens, and a call nobody -// priced writes no row ([session.RecordUsage] refuses an all-zero line). -// -// It answers the dollars it wrote, 0 when none were known. -func recordFactoryTriageUsage(settings config.Config, model string, response *ai.Response) float64 { - return recordFactoryAIUsage(settings, model, factoryTriageName, response) -} - // recordFactoryAIUsage puts one factory AI call's cost on the spend ledger // under name in the role column, on the low seat, and answers its dollars. func recordFactoryAIUsage(settings config.Config, model, name string, response *ai.Response) float64 { @@ -234,3 +222,47 @@ func recordFactoryAIUsage(settings config.Config, model, name string, response * session.RecordUsage(session.UsageLedgerPath(), session.TagUsage(line, roles.Role(name), session.SeatLow)) return line.USD } + +// factoryAIAccounting collects provider charges even when a failed generation +// returns no response. Its finisher banks them before the caller judges the +// error. The returned usage keeps its price-presence fact on successful calls. +func factoryAIAccounting(ctx context.Context, settings config.Config, model, name string, noted func(float64)) (context.Context, func(*ai.Response) float64) { + var mu sync.Mutex + var charges []provider.Billed + ctx = provider.WithBilling(ctx, func(charge provider.Billed) { + mu.Lock() + defer mu.Unlock() + charges = append(charges, charge) + }) + return ctx, func(response *ai.Response) float64 { + mu.Lock() + defer mu.Unlock() + total := 0.0 + bank := func(served string, response *ai.Response) { + usd := recordFactoryAIUsage(settings, served, name, response) + total += usd + if usd > 0 && noted != nil { + noted(usd) + } + } + for i, charge := range charges { + served := charge.Model + if served == "" { + served = model + } + if i == len(charges)-1 && response != nil { + bank(served, response) + continue + } + used := &ai.Usage{PromptTokens: charge.PromptTokens, CompletionTokens: charge.CompletionTokens} + if charge.Cost != 0 { + used.Cost = &charge.Cost + } + bank(served, &ai.Response{Usage: used}) + } + if len(charges) == 0 && response != nil { + bank(model, response) + } + return total + } +} diff --git a/cmd/codeaf/testdata/model_spend_owners.tsv b/cmd/codeaf/testdata/model_spend_owners.tsv new file mode 100644 index 0000000000..0d4c27a918 --- /dev/null +++ b/cmd/codeaf/testdata/model_spend_owners.tsv @@ -0,0 +1,83 @@ +# Exact production caller inventory. Exceptions are audit debt, not proof of missing charges. +cmd/codeaf/carried.go:wireCompleter.CompleteWithMessages:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:consolidateFacts:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:digestTerritory:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:distillFacts:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:fillCraftParams:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:quorumVerify:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:reflectAcrossJobs:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:repairCraft:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:titleGoal:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/chat.go:unlockedWhileThinking.CompleteWithMessages:CompleteWithMessages exception: untraced terminal helpers +cmd/codeaf/factory_scout.go:factoryScoutChat:CompleteWithMessages owner: Factory page accounting (factoryAIAccounting -> recordFactoryAIUsage -> session.RecordUsage) +cmd/codeaf/factory_triage.go:factoryTriageCallNoting:CompleteWithMessages owner: Factory page accounting (factoryAIAccounting -> recordFactoryAIUsage -> session.RecordUsage) +cmd/codeaf/image.go:runImage:GenerateImage exception: untraced image and media transport +cmd/codeaf/poolrecord.go:poolJudgeAsk:CompleteWithMessages exception: pool judge +cmd/codeaf/subharness_env.go:headlessEnv.AI:CompleteWithMessages exception: untraced execution helpers +internal/exec/document.go:Toolbox.parseDocument:ParseDocument exception: untraced execution helpers +internal/exec/linear.go:Linear.complete:CompleteWithMessages exception: untraced execution helpers +internal/exec/media.go:Toolbox.generateImage:GenerateImage exception: untraced image and media transport +internal/exec/media.go:Toolbox.generateMusic:GenerateMusic exception: music +internal/exec/media.go:Toolbox.generateVideo:GenerateVideo exception: video +internal/exec/media.go:Toolbox.speak:Speak exception: speech +internal/exec/media.go:Toolbox.viewImage:CompleteWithMessages exception: untraced image and media transport +internal/head/absorb.go:Head.deliveryAnswer:CompleteWithMessages exception: untraced resident assistants +internal/head/aside.go:Head.Ask:CompleteWithMessages exception: untraced resident assistants +internal/head/loop.go:Head.runTurn:CompleteWithMessages exception: untraced resident assistants +internal/head/revisionvoice.go:Head.composeRevision:CompleteWithMessages exception: untraced resident assistants +internal/head/scribe.go:Head.nameRoom:CompleteWithMessages exception: untraced resident assistants +internal/head/wake.go:Head.wakeReceipt:CompleteWithMessages exception: untraced resident assistants +internal/knowledge/compiler_provider.go:compilerModel.call:CompleteWithMessages owner: Memory compiler (memoryBilling -> session knowledge usage recorder -> RecordUsage) +internal/plan/brief.go:writeBrief:CompleteWithMessages exception: untraced plan helpers +internal/provider/client.go:Client.StreamComplete:sendShaped exception: untraced provider routing and transport +internal/provider/client.go:Client.completeWithMessagesStreaming:sendShaped exception: untraced provider routing and transport +internal/provider/client.go:Client.newHTTPRequest:newHTTPRequestAt exception: untraced provider routing and transport +internal/provider/client.go:Client.resend:send exception: untraced provider routing and transport +internal/provider/client.go:Client.sendRecovered:sendRepaired exception: untraced provider routing and transport +internal/provider/client.go:Client.sendRepaired:resend exception: untraced provider routing and transport +internal/provider/client.go:Client.sendRepaired:send exception: untraced provider routing and transport +internal/provider/client.go:Client.sendShaped:sendRecovered exception: untraced provider routing and transport +internal/provider/dispatch.go:Client.send:newHTTPRequestAt exception: untraced provider routing and transport +internal/provider/document.go:Client.ParseDocument:send exception: untraced provider routing and transport +internal/provider/endpoints.go:Client.attemptShaped:send exception: untraced provider routing and transport +internal/provider/endpoints.go:Client.widenPastTheUncarriedPreference:sendRepaired exception: untraced provider routing and transport +internal/provider/media.go:MediaClient.GenerateImage:postJSON exception: untraced image and media transport +internal/provider/media.go:MediaClient.Speak:do exception: speech +internal/provider/media.go:MediaClient.do:doEndpoint exception: untraced image and media transport +internal/provider/media.go:MediaClient.downloadVideo:doEndpoint exception: video +internal/provider/media.go:MediaClient.generateVideo:postJSON exception: video +internal/provider/media.go:MediaClient.getJSON:doEndpoint exception: untraced image and media transport +internal/provider/media.go:MediaClient.postJSON:do exception: untraced image and media transport +internal/provider/modelapi/server.go:Server.complete:CompleteWithMessages exception: untraced provider routing and transport +internal/provider/music.go:MediaClient.GenerateMusic:do exception: music +internal/provider/openrouter_client.go:Client.executeOwnToolCallLoop:CompleteWithMessages exception: untraced provider routing and transport +internal/provider/openrouter_client.go:Client.finalToolTurn:CompleteWithMessages exception: untraced provider routing and transport +internal/provider/pool/pool.go:Client.CompleteWithMessages:CompleteWithMessages exception: untraced provider routing and transport +internal/provider/pool/wall.go:walled.completion:CompleteWithMessages exception: untraced provider routing and transport +internal/provider/probe.go:Client.probeLane:newHTTPRequest exception: lane probes +internal/provider/transcribe.go:MediaClient.Transcribe:postJSON exception: speech +internal/reflex/reflex.go:bound.CompleteWithMessages:CompleteWithMessages exception: untraced reflex helpers +internal/reflex/reflex.go:bound.completeAttempt:CompleteWithMessages exception: untraced reflex helpers +internal/reflex/reflex.go:completeWithRecovery:CompleteWithMessages exception: untraced reflex helpers +internal/revision/compose.go:Compose:CompleteWithMessages exception: untraced revision helpers +internal/router/router.go:Router.StreamComplete:StreamComplete exception: untraced router helpers +internal/router/router.go:Router.cascade:CompleteWithMessages exception: untraced router helpers +internal/router/router.go:Router.leaf:CompleteWithMessages exception: untraced router helpers +internal/run/delegateworker.go:ownLineage.CompleteWithMessages:CompleteWithMessages exception: untraced run helpers +internal/session/agent.go:sessionCompleter.CompleteWithMessages:CompleteWithMessages exception: untraced session helpers +internal/session/automation_run.go:automationRunner.Judge:CompleteWithMessages exception: automation judge +internal/session/clientdoor.go:Agent.completeWithNamedModel:CompleteWithMessages exception: untraced session helpers +internal/session/memory.go:billedCompleter.CompleteWithMessages:CompleteWithMessages exception: untraced session helpers +internal/session/seatcompleter.go:seatCompleter.CompleteWithMessages:CompleteWithMessages exception: untraced session helpers +internal/session/spendguard.go:guardedCompleter.CompleteWithMessages:CompleteWithMessages exception: untraced session helpers +internal/session/tools_doc.go:ReadDocument:ParseDocument exception: untraced session helpers +internal/session/tools_image.go:GenerateImage:GenerateImage exception: untraced image and media transport +internal/session/tools_music.go:Agent.composeMusic:GenerateMusic exception: music +internal/session/tools_sense.go:Agent.audioSense:Transcribe exception: speech +internal/session/tools_speak.go:Agent.speakTool:Speak exception: speech +internal/session/tools_video.go:Agent.renderVideo:GenerateVideo exception: video +internal/shaped/prose.go:Prose:CompleteWithMessages exception: untraced shaped helpers +internal/shaped/shaped.go:Answer:CompleteWithMessages exception: untraced shaped helpers +internal/shaped/shaped.go:repair:CompleteWithMessages exception: untraced shaped helpers +internal/subharness/exec_model.go:modelEnv.say:CompleteWithMessages exception: untraced execution helpers +internal/subharness/exec_model.go:modelEnv.work:ExecuteToolCallLoop exception: untraced execution helpers diff --git a/docs/changes/unreleased/2268-factory-page-charges.md b/docs/changes/unreleased/2268-factory-page-charges.md new file mode 100644 index 0000000000..67ffbea4cc --- /dev/null +++ b/docs/changes/unreleased/2268-factory-page-charges.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: codeaf keeps Factory page charges when a model call fails +pr: 2268 +surface: [engine] +invalidates: ["Failed AI read and Look closer calls could disappear from Spending; reported charges now remain visible."] +--- + +AI read and Look closer now keep each reported provider charge on Spending, +including when a generation fails. Successful calls still count once. + +A structural ledger-ownership inventory rejects new unlisted model callers and +retired exception entries. Existing untraced families remain explicit audit debt. diff --git a/internal/factory/scout/scout.go b/internal/factory/scout/scout.go index 6102a2d202..5f66c7b270 100644 --- a/internal/factory/scout/scout.go +++ b/internal/factory/scout/scout.go @@ -91,10 +91,10 @@ func Run(ctx context.Context, it factory.Item, look Look, chat Chat, b Budget) ( final := false for { reply, err := chat(ctx, msgs) + r.out.USD += reply.USD if err != nil { return r.out, err } - r.out.USD += reply.USD if reply.Model != "" { r.out.Model = reply.Model } From a722531d833d9d067484068f82568ae7c5baaa02 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:37:35 -0500 Subject: [PATCH 22/28] spend: a hedge's finished loser counts, with proofs for brief, Follow PR and page AI (#2271) * factory: a write step that changed nothing is not done A write round that reported done while the job's checkout had no change from the run's base counted as done, so test and review ran on an untouched tree and only publishing found nothing to commit. The runner now reads the checkout after a write round; with no change, the round does not pass, says "the checkout has no change from where the run began", and goes round again or asks. Other steps may still finish with no change, and an unreadable checkout never holds a round back. Co-Authored-By: Claude Opus 5.5 * docs: changelog entry for #2158 Co-Authored-By: Claude Opus 5.5 * test: reproduce missing memory spend through the vault interface Co-Authored-By: Claude Opus 5.5 * fix: bank memory receipts and bind job compilation to finished turns The compiler used the provider transport without a ledger consumer, while the process worker owned its lifetime independently of jobs. Bank decoded receipts before validation and keep job sources out of independent compilation. Co-Authored-By: Claude Opus 5.5 * fix: isolate memory snapshots and verify cancellation during compilation Keep compilation boundaries per owner, drain finished-turn journals, and classify older factory conversations from persisted memberships. Preserve projection upgrades and reject a structural bypass that discards the billed context. Co-Authored-By: Claude Opus 5.5 * test: cover retained job sources and large memory snapshots Co-Authored-By: Claude Opus 5.5 * test: refuse memory admission on paused and published jobs Co-Authored-By: Claude Opus 5.5 * fix: drain owned memory snapshots and fence unchanged failed input Reclassify retained job events independently of journal cursors, check stored pause and publication state at admission, and finish every extraction batch in the turn snapshot. New source input or an explicit Build is required to retry failed background compilation. Co-Authored-By: Claude Opus 5.5 * test: reject repeated refinement of unchanged memory input Co-Authored-By: Claude Opus 5.5 * fix: require new evidence or explicit build for further memory refinement Record the Build request consumed by each publication so forced flushes on unchanged sources stay idle. Preserve recovery when extraction produced new unpublished evidence, and let an explicit Build advance retained guide work. Co-Authored-By: Claude Opus 5.5 * test: old job ownership excludes background memory Co-Authored-By: Claude Opus 5.5 * fix: retain old factory ownership in memory inventory Co-Authored-By: Claude Opus 5.5 * test: compile delegated reports with their owning turn Co-Authored-By: Claude Opus 5.5 * test: use the persisted place layout for delegated memory Co-Authored-By: Claude Opus 5.5 * fix: compile delegated evidence on the owning job turn Co-Authored-By: Claude Opus 5.5 * test: measure live job memory cost and stage tail delays Co-Authored-By: Claude Opus 5.5 * test: memory cannot turn successful work into a cap stop Co-Authored-By: Claude Opus 5.5 * fix: preserve successful stage results after deferred spending Co-Authored-By: Claude Opus 5.5 * test: bound stage memory waits and retain cancelled charges Co-Authored-By: Claude Opus 5.5 * test: enforce memory cap only at the next unfinished step Co-Authored-By: Claude Opus 5.5 * fix: record completed phases before checking the next budget Co-Authored-By: Claude Opus 5.5 * test: distinguish memory delay and stop timed-out live jobs Co-Authored-By: Claude Opus 5.5 * fix: bound stage memory waits to the owning job lifetime Co-Authored-By: Claude Opus 5.5 * test: retain skipped stage evidence across job memory handoff Co-Authored-By: Claude Opus 5.5 * fix: recover skipped stage evidence in the next owned memory pass Co-Authored-By: Claude Opus 5.5 * test: measure late memory receipts after job completion Co-Authored-By: Claude Opus 5.5 * fix: keep an empty owned source selection empty Co-Authored-By: Claude Opus 5.5 * test: count overlapping memory waits as wall time Co-Authored-By: Claude Opus 5.5 * test: attribute memory tail wait to its initiating stage Co-Authored-By: Claude Opus 5.5 * test: verify live memory starts stay inside the job lifetime Co-Authored-By: Claude Opus 5.5 * test: stop live measurements with a fresh job version Co-Authored-By: Claude Opus 5.5 * test: count measured memory calls independently of late prices Co-Authored-By: Claude Opus 5.5 * test: read measured job spending through the app wire Co-Authored-By: Claude Opus 5.5 * test: settle cancelled live calls before checking measured spend Co-Authored-By: Claude Opus 5.5 * test: open live factory brief chats through the in-process app Co-Authored-By: Claude Opus 5.5 * test: deliver late memory receipts through the real factory worker Co-Authored-By: Claude Opus 5.5 * test: observe the public ready flag and bound worker fixture cleanup Co-Authored-By: Claude Opus 5.5 * Keep job reporting alive for late memory receipts Co-Authored-By: Claude Opus 5.5 * Test late memory pricing after stopping a factory job Co-Authored-By: Claude Opus 5.5 * Observe finished steps before settling final prices and pin live helpers Co-Authored-By: Claude Opus 5.5 * Reconcile paid memory after a stopped job loses its lease Co-Authored-By: Claude Opus 5.5 * Keep one accounting anchor per run and retry final settlement Co-Authored-By: Claude Opus 5.5 * Test final memory settlement preserves previously counted job spend Co-Authored-By: Claude Opus 5.5 * Measure stage answers separately from automatic brief calls Co-Authored-By: Claude Opus 5.5 * Report per-job memory dollars and bound delays by the next step start Co-Authored-By: Claude Opus 5.5 * Test paid memory survives reopening a finished stage Co-Authored-By: Claude Opus 5.5 * Preserve paid usage in a closed stage journal Co-Authored-By: Claude Opus 5.5 * Test leased memory receipts during actual factory quit Co-Authored-By: Claude Opus 5.5 * Allow bounded accounting settlement after worker cancellation Co-Authored-By: Claude Opus 5.5 * Reproduce the memory tail fixture accounting observation race Co-Authored-By: Claude Opus 5.5 * Wait for both memory ledger and job accounting in lifecycle fixture The ledger is published before the independent job meter update. Poll both observable totals and name standalone session closure accurately. Co-Authored-By: Claude Opus 5.5 * Test quitting while final worker accounting is already settling Co-Authored-By: Claude Opus 5.5 * Keep bounded accounting alive when quit interrupts settlement Detach the existing settlement context from execution immediately, then cancel it five seconds after quit. This also covers cancellation arriving during the existing receipt wait. Co-Authored-By: Claude Opus 5.5 * Recognize late journal usage in the append funnel guard The journal is a separate store from artifact indexes. Account for both journal append owners while retaining the guard against unknown append paths. Co-Authored-By: Claude Opus 5.5 * Reproduce stage budget exemptions and unowned batch draining Co-Authored-By: Claude Opus 5.5 * Distinguish memory cap crossings and restore bounded ordinary Flush Keep memory in total job spend while excluding only newly charged memory from completed-round cap checks. Preserve model and tool stops, and retain finite draining only for owned turns. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate factory brief charges through the app wire Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8f8015c20a5e74cd8c73ec68d8fe22b4feb5118e) * fix: charge factory brief calls only at the session billing boundary The session already books each provider receipt to the job and ledger. Keeping a brief charged the job again, including rejected drafts. Preserve session booking so failed saves retain their cost and worker spending remains additive. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c2938deaf4b3e6ec8eb9c865a58c229663def4ac) * test: audit manager and title charges beside factory briefs Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bb52801b802fd29a1b6889d55461eeeb69e66f87) * test: exercise title billing on an unnamed job-owned conversation Factory managers have a saved title and correctly skip model naming. Use an unnamed stage for the separate title receipt audit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eee242d5899dd62c6d89e3186a1f83d853a67cda) * test: align the factory keep contract with receipt-based billing Keeping a brief preserves its cost metadata without charging the already billed call again. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3af433d90bf6809c42152de9b480be3ab3b9e99a) * test: reserve the leased brief fixture for its simulated worker The local factory starts a worker too. Pin the fixture to the remote machine so a local runner cannot win its queued lease when the surrounding suite supplies a runnable configuration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ff25c49ffc450b770045ee8a861156447001cd4) * Declare local memory provenance at the reader boundary Check late memory receipts retain their classified share as well as their total charge. Co-Authored-By: Claude Opus 5.5 * Use the existing lane import alias for memory accounting Co-Authored-By: Claude Opus 5.5 * test: check live brief charges against the job header Co-Authored-By: Claude Opus 5.5 * changelog: one entry for the brief charge fix Co-Authored-By: Claude Opus 5.5 * test: the call log's served field names the provider, not the model The recorder already checks the model that answered each call. Co-Authored-By: Claude Opus 5.5 * test: memory crossing the cap does not hide a daily refusal Co-Authored-By: Claude Opus 5.5 * factory: a daily refusal is never read as a memory charge Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: cover factory write retries against the job base Exercise committed earlier work on rerun, retry after failure and resume after Stop, and retain the refusal for an empty first attempt and rerun. Include the waiting question when the retry fixture misses its failed check. Co-Authored-By: Claude Opus 5.5 * test: keep write retry checks at the Test approval gate Include the following test step so the manual approval is reached, and stop the current step so its recorded attempt can be resumed. Check the retained commit and change through the job reply. Co-Authored-By: Claude Opus 5.5 * test: distinguish stopped writes before and after reporting Compare immediate Stop with Stop after the first running receipt, using the same committed change and wire actions. Co-Authored-By: Claude Opus 5.5 * fix: retain the factory write base across stopped attempts Stop can revoke a lease before its first report carries ChangeBase. Keep the original pin on the job branch before edits so its replacement counts earlier committed work instead of comparing against the edited HEAD. Make the three retry fixtures perform their promised write. Co-Authored-By: Claude Opus 5.5 * test: synchronize stopped write reports and refuse an unpinned base Hold periodic reporting on the real lease worker until Stop, then run the replacement with normal reporting. Verify a locked repository config prevents Write from editing before its comparison base is preserved. Co-Authored-By: Claude Opus 5.5 * fix: stop a factory round when its base cannot be preserved A repository config lock must not let edits start without their durable comparison pin. Surface the save error before calling the executor. Keep the deterministic replacement worker on the same fixture machine. Co-Authored-By: Claude Opus 5.5 * test: assert the factory preparation failure receipt A failure to preserve the base is an execution preparation error, so read its failed-step detail rather than an out-of-rounds result. Co-Authored-By: Claude Opus 5.5 * test: keep read-only factory checks independent of Git metadata Exercise the exported runner with a real command in a directory without Git metadata, matching the leased-worker fixtures. Co-Authored-By: Claude Opus 5.5 * fix: discard failed Git reads before preserving a factory base Git errors include stderr as output. Do not mistake that text for a branch or commit and try to persist it. Keep the existing behavior when Git metadata is unavailable, while refusing an actual base save failure. Co-Authored-By: Claude Opus 5.5 * test: reproduce pause after a memory stage tail detaches Co-Authored-By: Claude Opus 5.5 * test: pin one next budget for cap questions and answers Co-Authored-By: Claude Opus 5.5 * test: check memory admission when Pause returns Co-Authored-By: Claude Opus 5.5 * test: complete the paid memory response with valid JSON Co-Authored-By: Claude Opus 5.5 * fix: give cap questions and answers one allowed next budget Select the smallest allowed cap above recorded spend instead of doubling the old cap. Share budget choices with the runner, job reads and budget edits, and refuse a continue when no higher budget remains. Co-Authored-By: Claude Opus 5.5 * test: retain paused memory for a later resumed turn Co-Authored-By: Claude Opus 5.5 * test: measure live Pause memory and provider receipt totals Co-Authored-By: Claude Opus 5.5 * test: keep budget continuation available above thirty dollars Co-Authored-By: Claude Opus 5.5 * test: check settled provider sums and detached memory lifecycle Co-Authored-By: Claude Opus 5.5 * test: pause live memory after a provider generation starts Document paused memory admission, normal resume, and process-owned shutdown. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * fix: admit local job memory against the owning factory state A worker mirror can still say running after Pause returns. Read the owning local factory for memory admission and cancellation while retaining mirror-based receipt settlement. The job conversation checks that owner before worker mirrors too. Co-Authored-By: Claude Opus 5.5 * test: cap resumed memory at one normal scripted pass Co-Authored-By: Claude Opus 5.5 * fix: offer matching budget continuation beyond thirty dollars The fixed budget ladder returned zero after thirty dollars, trapping jobs. Derive the next cap from the job and accept that exact amount at the factory budget door. Co-Authored-By: Claude Opus 5.5 * test: measure only stages that started before Pause A paused job correctly has no conversation identity for its unstarted stages. Skip those stages when measuring memory rather than requiring identities from future work. Co-Authored-By: Claude Opus 5.5 * test: align cap metadata expectations with the next budget rule The whole-package run found two expectations left on the former doubling and zero-limit rules. Expect the next standard five dollar cap at both public seams. Co-Authored-By: Claude Opus 5.5 * test: count a finished memory generation and its receipt once A response can finish at the same time cancellation queues its receipt. Both observations describe the same provider charge, so share the existing once-only charge counter. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: retain unfinished memory after a completed extraction Pending counts unextracted source events. When the already-paid extraction finishes at Pause, those events can be consumed while NeedsUpdate retains the remaining compilation work. Keep the stricter source assertion for interrupted calls. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * test: prove completed factory hedge arms are both charged Co-Authored-By: Claude Opus 5.5 * test: check factory helper and follow-up money through the app wire Co-Authored-By: Claude Opus 5.5 * test: isolate page AI sessions and await the published job turn Co-Authored-By: Claude Opus 5.5 * test: exercise the factory Stop action for final brief accounting Co-Authored-By: Claude Opus 5.5 * test: omit absent schema from priced errand requests Co-Authored-By: Claude Opus 5.5 * fix: account for completed hedge losers before returning the winner Completed losing responses were left in the race result channel without reaching discarded-usage accounting. Drain the cancelled arms and forward their usage before the caller closes its accounting. Co-Authored-By: Claude Opus 5.5 * memory: name its call tag for the replay law and pass the new scan argument Co-Authored-By: Claude Opus 5.5 * test: memory fixtures use a step that isn't held for an unchanged checkout A job's write step with no change now waits for a person. These tests are about memory, so their step is plain work. Co-Authored-By: Claude Opus 5.5 * fix: cancel hedge arms before collecting their final accounting Co-Authored-By: Claude Opus 5.5 * test: supply fixture admission and evidence for factory money proofs Co-Authored-By: Claude Opus 5.5 * test: await page AI admission and run follow-up from a captured PR Co-Authored-By: Claude Opus 5.5 * test: explicitly hold the page AI agent through its public open door Co-Authored-By: Claude Opus 5.5 * test: match the page AI fixture transcript to its direct session opener Co-Authored-By: Claude Opus 5.5 * test: clear omitted errand reply fields before retrying admission Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_money_followup_test.go | 142 ++++++++++++ cmd/codeaf/appfactory_money_helpers_test.go | 143 ++++++++++++ cmd/codeaf/appfactory_money_paths_test.go | 210 ++++++++++++++++++ .../2271-factory-completed-hedge-costs.md | 13 ++ internal/provider/hedge.go | 16 +- 5 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 cmd/codeaf/appfactory_money_followup_test.go create mode 100644 cmd/codeaf/appfactory_money_helpers_test.go create mode 100644 cmd/codeaf/appfactory_money_paths_test.go create mode 100644 docs/changes/unreleased/2271-factory-completed-hedge-costs.md diff --git a/cmd/codeaf/appfactory_money_followup_test.go b/cmd/codeaf/appfactory_money_followup_test.go new file mode 100644 index 0000000000..9890d8b158 --- /dev/null +++ b/cmd/codeaf/appfactory_money_followup_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" + "github.com/Agent-Field/codeaf/internal/session" +) + +// A paid Follow PR attempt and a later manager turn retain one job identity. +// The repair reports needs_you, so this fixture neither publishes nor pushes. +func TestAppFactoryFollowPRAndPostPublishMoney(t *testing.T) { + s := newLocalSetup(t) + t.Setenv("OPENROUTER_API_KEY", "fixture") + var calls atomic.Int64 + server := moneyProvider(t, &calls) + g := newRunRig(t) + freshEngineProcess(t) + freshEngineFactoryStore(t) + t.Setenv(factoryBenchesEnv, "") + stageSettle = 20 * time.Millisecond + t.Cleanup(func() { stageSettle = unattendedRunSettleTick }) + s.backend.emit = s.server + s.backend.local.parent = func() session.Config { + return session.Config{Workspace: g.workspace, Model: "deepseek/deepseek-v4.1-flash", OneModel: true, APIKey: "fixture", BaseURL: server.URL, DeterministicNames: true} + } + s.backend.open = func(ask appOpenAsk) (*appConn, error) { + agent := moneyAgent(t, s, ask, server.URL) + return &appConn{Conv: agent, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() { _ = agent.Close() }}, nil + } + // This local runner uses the captured PR snapshot. Keep the automatic + // worker off so it cannot also lease the same fixture job. + var connected atomic.Bool + var runner *factoryrun.Runner + hook := s.backend.local.seamHook + s.backend.local.seamHook = func(seam factory.Seam) factory.Seam { + seam = hook(seam) + seam.ConnectGitHub = func(context.Context, string) error { return nil } + seam.GitHub = func(context.Context) (factory.GitHubLink, error) { + if !connected.Load() { + return factory.GitHubLink{}, nil + } + return factory.GitHubLink{Login: "owner"}, nil + } + seam.Launch = func(id int) error { return runner.Launch(id) } + return seam + } + st := s.start() + if st.Runner { + t.Fatal("fixture started an automatic worker") + } + connected.Store(true) + if err := s.backend.local.host.seam.ConnectGitHub(context.Background(), "fixture"); err != nil { + t.Fatal(err) + } + s.backend.briefs.stop() + s.backend.briefs = newBriefDebounce(time.Hour, s.backend.writeBriefLater) + floor := s.backend.local.host.floor + parent := s.backend.local.parent() + maker := factoryStageMaker(floor, g.workspace, "", parent) + runner = factoryrun.New(factoryrun.Options{Store: floor, Workdir: func(factory.Item) (string, error) { return g.workspace, nil }, Exec: map[factory.StageKind]factoryrun.Executor{factory.StageChat: factoryChatStage(floor, "", factoryrun.NewChatExecutor(maker))}}) + stopAll(t, floor, runner) + if err := floor.SetCheckout("acme/api", g.workspace); err != nil { + t.Fatal(err) + } + item := factory.Item{Repo: "acme/api", Num: 7, Kind: factory.KindIssue, Title: "Repair the PR retry policy", Tier: factory.TierOwner, Model: "deepseek/deepseek-v4.1-flash", Cap: 1, State: factory.StateNew, + PullRequest: &factory.PullRequest{Number: 73, State: factory.PROpen}, FollowUp: &factory.PRFollowUp{Kind: "pr", State: "queued", Attempt: 1, Limit: 3}} + factory.EnsureFollowPR(&item) + it, err := floor.Create(item) + if err != nil { + t.Fatal(err) + } + params := map[string]any{"factory": st.Factory, "job": it.ID} + var job api.Job + s.must("factory.job", params, &job) + s.must("factory.job_start", map[string]any{"factory": st.Factory, "job": it.ID, "expect": map[string]any{"version": job.Version}}, &appwire.FactoryJobActed{}) + deadline := time.Now().Add(15 * time.Second) + for { + s.must("factory.job", params, &job) + if calls.Load() > 0 && job.State == api.StateNeedsYou { + break + } + if time.Now().After(deadline) { + t.Fatalf("paid repair never settled: calls=%d job=%+v", calls.Load(), job) + } + time.Sleep(10 * time.Millisecond) + } + moneyTotals(t, s, st.Factory, it, &calls, float64(calls.Load())*.01) + // A captured publication receipt is a fixture fact, with no forge writes. + if err := floor.Update(it.ID, func(cur *factory.Item) error { + cur.State = factory.StateShipped + cur.Stream.Ended = time.Now() + cur.PullRequest = &factory.PullRequest{Number: 73, State: factory.PROpen} + return nil + }); err != nil { + t.Fatal(err) + } + var chats appwire.FactoryJobChats + s.must("factory.job_chats", params, &chats) + before := calls.Load() + var turn appwire.TurnResult + s.must("agents.send", map[string]any{"id": chats.Agent, "text": "Explain the published repair."}, &turn) + deadline = time.Now().Add(5 * time.Second) + for { + finished := func() bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, line := range s.lines { + var event struct { + Event string `json:"event"` + Data appwire.TurnDone `json:"data"` + } + if json.Unmarshal([]byte(line), &event) == nil && event.Event == appwire.EventTurnDone && event.Data.Agent == chats.Agent && event.Data.Turn == turn.Turn { + if !event.Data.OK { + t.Fatalf("post-publish turn failed: %+v", event.Data) + } + return true + } + } + return false + }() + if finished { + break + } + if time.Now().After(deadline) { + t.Fatal("post-publish turn never finished") + } + time.Sleep(10 * time.Millisecond) + } + if calls.Load() != before+1 { + t.Fatalf("post-publish calls=%d, want %d", calls.Load(), before+1) + } + + moneyTotals(t, s, st.Factory, it, &calls, float64(calls.Load())*.01) +} diff --git a/cmd/codeaf/appfactory_money_helpers_test.go b/cmd/codeaf/appfactory_money_helpers_test.go new file mode 100644 index 0000000000..48d5bf59d0 --- /dev/null +++ b/cmd/codeaf/appfactory_money_helpers_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "math" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestAppFactoryDebouncedEndMoney(t *testing.T) { + for _, stopped := range []bool{false, true} { + name := "finished" + if stopped { + name = "stopped" + } + t.Run(name, func(t *testing.T) { + s := newLocalSetup(t) + var calls atomic.Int64 + server := moneyProvider(t, &calls) + st := s.start() + s.backend.open = func(ask appOpenAsk) (*appConn, error) { + agent := moneyAgent(t, s, ask, server.URL) + return &appConn{Conv: agent, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() { _ = agent.Close() }}, nil + } + floor := s.backend.local.host.floor + it, err := floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Final brief accounting", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: 1, Stream: &factory.Stream{Phases: []factory.Phase{{Name: "test", Kind: factory.StageCheck, State: factory.PhaseDone, Round: 1, Result: &factory.StageResult{Done: true, Tests: &factory.TestCounts{Passed: 3}}}}}}) + if err != nil { + t.Fatal(err) + } + params := map[string]any{"factory": st.Factory, "job": it.ID} + var chats appwire.FactoryJobChats + s.must("factory.job_chats", params, &chats) + // Keep the production lifecycle callback and debounce, shortening only + // its wait so the test does not spend the user's debounce interval. + s.backend.briefs.stop() + s.backend.briefs = newBriefDebounce(10*time.Millisecond, s.backend.writeBriefLater) + if stopped { + var job api.Job + s.must("factory.job", params, &job) + s.must("factory.job_stop", map[string]any{"factory": st.Factory, "job": it.ID, "expect": map[string]any{"version": job.Version}}, &appwire.FactoryJobActed{}) + } else if err := floor.Update(it.ID, func(cur *factory.Item) error { + cur.State = factory.StateShipped + cur.Stream.Ended = time.Now() + return nil + }); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + var brief appwire.FactoryJobBrief + for { + s.must("factory.job_brief", params, &brief) + if brief.Status == factory.BriefStatusReady || brief.Status == factory.BriefStatusFailed { + break + } + if time.Now().After(deadline) { + t.Fatalf("debounced end did not finish: %+v", brief) + } + time.Sleep(10 * time.Millisecond) + } + want := .01 + if stopped { + want = float64(calls.Load()) * .01 + if calls.Load() == 0 && brief.Status != factory.BriefStatusFailed { + t.Errorf("refused brief must state its failure: %+v", brief) + } + if calls.Load() > 1 { + t.Errorf("stopped brief made %d provider calls, want at most one", calls.Load()) + } + } + moneyTotals(t, s, st.Factory, it, &calls, want) + }) + } +} + +func TestAppFactoryPageAIMoneyStaysOutsideJob(t *testing.T) { + s := newLocalSetup(t) + var calls atomic.Int64 + server := moneyProvider(t, &calls) + st := s.start() + s.backend.open = func(ask appOpenAsk) (*appConn, error) { + agent := moneyAgent(t, s, ask, server.URL) + return &appConn{Conv: agent, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() { _ = agent.Close() }}, nil + } + it, err := s.backend.local.host.floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Viewed job", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: 1}) + if err != nil { + t.Fatal(err) + } + s.backend.mint = appMint + var created struct { + Agent appwire.Agent `json:"agent"` + } + s.must("agents.create", map[string]any{"workspace": t.TempDir()}, &created) + s.must("agents.open", map[string]any{"id": created.Agent.ID}, &struct{}{}) + for _, path := range []struct{ purpose, role string }{{"pipeline_ai", "insight_pipeline"}, {"view_topic", "insight_views"}} { + var answer appwire.Errand + deadline := time.Now().Add(5 * time.Second) + for { + answer = appwire.Errand{} + s.must("agents.open", map[string]any{"id": created.Agent.ID}, &struct{}{}) + s.must("agents.errand", map[string]any{"agent": created.Agent.ID, "purpose": path.purpose, "role": path.role, "system": "Answer briefly.", "messages": []appwire.ErrandMessage{{Role: "user", Text: "Summarize."}}, "max_tokens": 100}, &answer) + if answer.Reason != appwire.ErrandNotOpen { + break + } + if time.Now().After(deadline) { + t.Fatalf("page AI agent never opened: provider calls=%d reply=%+v", calls.Load(), answer) + } + time.Sleep(10 * time.Millisecond) + } + if answer.Reason != "" || answer.Role != path.role || math.Abs(answer.Usage.CostUSD-.01) > 1e-9 { + t.Fatalf("%s: %+v", path.purpose, answer) + } + } + if !session.FlushUsage() { + t.Fatal("ledger did not flush") + } + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + var ledger float64 + purposes := map[string]int{} + for _, line := range lines { + ledger += line.USD + purposes[line.Role] += line.Calls + if line.FactoryJob != "" { + t.Errorf("page AI charged job %q", line.FactoryJob) + } + } + day := time.Now().Format("2006-01-02") + var spending appwire.SpendReportResult + s.must("spend.report", map[string]any{"from": day, "to": day}, &spending) + var header api.Job + s.must("factory.job", map[string]any{"factory": st.Factory, "job": it.ID}, &header) + if calls.Load() != 2 || math.Abs(ledger-.02) > 1e-9 || spending.Totals.CostUSD == nil || math.Abs(*spending.Totals.CostUSD-.02) > 1e-9 || header.Cost != 0 || purposes["pipeline_ai"] != 1 || purposes["view_topic"] != 1 { + t.Fatalf("provider=%.4f ledger=%.4f spending=%+v job=%.4f purposes=%v", float64(calls.Load())*.01, ledger, spending.Totals, header.Cost, purposes) + } + t.Logf("provider=0.0200 ledger=%.4f Spending=%.4f viewed job=%.4f", ledger, *spending.Totals.CostUSD, header.Cost) +} diff --git a/cmd/codeaf/appfactory_money_paths_test.go b/cmd/codeaf/appfactory_money_paths_test.go new file mode 100644 index 0000000000..ef9886d265 --- /dev/null +++ b/cmd/codeaf/appfactory_money_paths_test.go @@ -0,0 +1,210 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + lanes "github.com/Agent-Field/codeaf/internal/lane" + "github.com/Agent-Field/codeaf/internal/lane/control" + "github.com/Agent-Field/codeaf/internal/provider" + "github.com/Agent-Field/codeaf/internal/session" +) + +// Both arms reach their final price before either can return. This exercises +// completed losers, rather than depending on a cancelled arm's later receipt. +type moneyBarrier struct { + priced atomic.Int64 + ready chan struct{} +} + +func (b *moneyBarrier) BeforeAttempt(context.Context, provider.ActualAttemptInfo) (provider.ActualAttemptToken, error) { + return b, nil +} +func (*moneyBarrier) EndTransport(provider.TransportEnd) {} +func (b *moneyBarrier) ObserveFinancial(f provider.FinancialObservation) { + if f.State != provider.FinancialKnown { + return + } + if b.priced.Add(1) == 2 { + close(b.ready) + } + select { + case <-b.ready: + case <-time.After(5 * time.Second): + } +} + +type moneyDoor struct { + *session.Agent + barrier *moneyBarrier +} + +func (d *moneyDoor) RunErrand(ctx context.Context, ask session.ErrandAsk) (session.ErrandAnswer, error) { + if d.barrier != nil { + var err error + ctx, err = provider.WithActualAttemptObserver(ctx, d.barrier) + if err != nil { + return session.ErrandAnswer{}, err + } + } + return d.Agent.RunErrand(ctx, ask) +} + +// Only waiting policy is scripted: the real transport must start a rescue. +type moneyWait struct { + control.Controller + deadline time.Time + fired bool +} + +func (w *moneyWait) Quiet(time.Time) control.Act { + if w.fired { + return control.Act{} + } + w.fired = true + return control.Act{Kind: control.Report, Reason: control.CeilingReason} +} +func (w *moneyWait) Note(control.Reading) control.Act { return control.Act{} } +func (w *moneyWait) Deadline() time.Time { + if w.fired { + return time.Time{} + } + return w.deadline +} + +func TestAppFactoryCompletedHedgeLoserMoney(t *testing.T) { + s := newLocalSetup(t) + var calls atomic.Int64 + server := moneyProvider(t, &calls) + lanes.HeardPrefsCarried(server.URL) + old := lanes.SetController(func(plan control.Plan) control.Controller { + return &moneyWait{Controller: control.New(plan), deadline: time.Now().Add(20 * time.Millisecond)} + }) + t.Cleanup(func() { lanes.SetController(old) }) + st := s.start() + floor := s.backend.local.host.floor + barrier := &moneyBarrier{ready: make(chan struct{})} + s.backend.open = func(ask appOpenAsk) (*appConn, error) { + agent := moneyAgent(t, s, ask, server.URL) + return &appConn{Conv: &moneyDoor{Agent: agent, barrier: barrier}, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() { _ = agent.Close() }}, nil + } + it, err := floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Hedge accounting", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: 1}) + if err != nil { + t.Fatal(err) + } + var chats appwire.FactoryJobChats + s.must("factory.job_chats", map[string]any{"factory": st.Factory, "job": it.ID}, &chats) + var answer appwire.Errand + s.must("agents.errand", map[string]any{"agent": chats.Agent, "purpose": "hedge_money", "role": "brief", "system": "Answer briefly.", "messages": []appwire.ErrandMessage{{Role: "user", Text: "Check accounting."}}, "max_tokens": 100, "deadline_ms": 10000}, &answer) + if answer.Reason != "" { + t.Fatalf("errand: %+v", answer) + } + if calls.Load() != 2 || barrier.priced.Load() != 2 { + t.Fatalf("completed arms=%d priced=%d, want two", calls.Load(), barrier.priced.Load()) + } + moneyTotals(t, s, st.Factory, it, &calls, .02) +} + +func moneyProvider(t *testing.T, calls *atomic.Int64) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + fmt.Fprint(w, `{"data":[]}`) + return + } + calls.Add(1) + var request struct { + Stream bool `json:"stream"` + Tools []struct { + Function struct { + Name string `json:"name"` + } `json:"function"` + } `json:"tools"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + message := map[string]any{"content": briefDraft} + choice := map[string]any{"finish_reason": "stop"} + if request.Stream { + choice["delta"] = message + } else { + choice["message"] = message + } + for _, tool := range request.Tools { + if tool.Function.Name == "stage_result" { + message = map[string]any{"tool_calls": []any{map[string]any{"index": 0, "id": "repair-report", "type": "function", "function": map[string]any{"name": "stage_result", "arguments": `{"done":false,"output":"Repair needs a decision.","repair":{"decision":"needs_you","reason":"Choose the retry policy."}}`}}}} + choice["finish_reason"] = "tool_calls" + if request.Stream { + choice["delta"] = message + } else { + choice["message"] = message + } + break + } + } + raw, _ := json.Marshal(map[string]any{"model": "deepseek/deepseek-v4.1-flash", "choices": []any{choice}, "usage": map[string]any{"prompt_tokens": 20, "completion_tokens": 5, "cost": .01}}) + if request.Stream { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", raw) + } else { + w.Write(raw) + } + })) + t.Cleanup(server.Close) + return server +} + +func moneyAgent(t *testing.T, s *localSetup, ask appOpenAsk, base string) *session.Agent { + t.Helper() + agent, err := session.New(session.Config{Workspace: ask.Workspace, SessionFile: ask.Transcript, ProfileDir: filepath.Join(s.home, "money-profile"), Model: "deepseek/deepseek-v4.1-flash", APIKey: "fixture", BaseURL: base, Routing: provider.RoutingLatency, JobChat: jobChatDoorOf(s.backend.local.host.floor), Unattended: true, DeterministicNames: true}) + if err != nil { + t.Fatal(err) + } + return agent +} + +func moneyTotals(t *testing.T, s *localSetup, factoryID string, it factory.Item, calls *atomic.Int64, want float64) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if !session.FlushUsage() { + t.Fatal("ledger did not flush") + } + lines, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatal(err) + } + var ledger float64 + var count int + for _, line := range lines { + if line.FactoryJob == factory.LedgerName(it) { + ledger += line.USD + count += line.Calls + } + } + var header api.Job + s.must("factory.job", map[string]any{"factory": factoryID, "job": it.ID}, &header) + providerUSD := float64(calls.Load()) * .01 + if math.Abs(providerUSD-want) < 1e-9 && math.Abs(ledger-want) < 1e-9 && math.Abs(header.Cost-want) < 1e-9 && int64(count) == calls.Load() { + t.Logf("provider=%.4f ledger=%.4f header=%.4f calls=%d", providerUSD, ledger, header.Cost, count) + return + } + if time.Now().After(deadline) { + t.Fatalf("provider=%.4f ledger=%.4f header=%.4f calls=%d; want %.4f once per call", providerUSD, ledger, header.Cost, count, want) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/docs/changes/unreleased/2271-factory-completed-hedge-costs.md b/docs/changes/unreleased/2271-factory-completed-hedge-costs.md new file mode 100644 index 0000000000..30d4b8b2f5 --- /dev/null +++ b/docs/changes/unreleased/2271-factory-completed-hedge-costs.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: Factory jobs count completed calls discarded by a provider race +pr: 2271 +surface: [engine, factory, chat] +invalidates: + - "Was true: when both provider racers completed, the discarded answer could be missing from Spending and the job budget. Is true now: both completed answers are counted once against the same job." +--- + +When a slow request races another provider, the job includes the cost of both +completed calls, even though only one answer is used. Spending and the job header +retain the same charges. Settlement stops outstanding requests, then waits up to +one second for their accounting. No app wire methods change. diff --git a/internal/provider/hedge.go b/internal/provider/hedge.go index 125ae69a3f..23ac3659ed 100644 --- a/internal/provider/hedge.go +++ b/internal/provider/hedge.go @@ -407,9 +407,9 @@ func (r *hedgeRace) run(ctx context.Context, messages []ai.Message, options ...a // arm that cannot report at all, which is exactly the shape issue #264 was // found in. // -// NOBODY WAITS OUT THIS GRACE ANY MORE. It is paid on the goroutine -// [hedgeRace.accountForTheAbandoned] runs on and never by the caller — see -// [hedgeRace.abandon] for the measurement that moved it. +// Abandoned callers never wait for this grace: [hedgeRace.abandon] accounts +// on its own goroutine. Settlement waits for cancelled arms within this bound +// so completed losing responses reach the caller's accounting before return. const abandonGrace = time.Second // abandon leaves the race because its caller did. @@ -1963,6 +1963,15 @@ func (r *hedgeRace) spend(arm int) (arms int, waste float64, note, refused strin func (r *hedgeRace) settle(result armResult, seen map[int]armResult) (*ai.Response, bool, error) { r.withdraw() arms, model := r.armsAndModel() + // Settlement can also follow an account refusal, before a winner cancels + // the other arms. Stop every request before waiting for its accounting. + for _, arm := range arms { + arm.cancel() + } + // A loser may already have completed when the winner cancels it. Collect + // its response before returning so discarded usage reaches the same call's + // ledger and budget, rather than disappearing with the race. + r.drainArms(seen) // The model as the ANSWER spelled it, because a router may pin a variant // the config never named and a belief keyed on the wrong id is a belief // about nothing. @@ -1993,6 +2002,7 @@ func (r *hedgeRace) settle(result armResult, seen map[int]armResult) (*ai.Respon tokens := 0 if done, ok := seen[arm.index]; ok && done.response != nil && done.response.Usage != nil { tokens = done.response.Usage.CompletionTokens + noteDiscardedUsage(r.base, done.response) } // A HEDGE IS A MEASUREMENT, whichever way it landed: the loser's // first-token wait is the only cheap evidence there is about the lane From 252ecb82bf2167447eaad0e0fac5632a47eba418 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:37:48 -0500 Subject: [PATCH 23/28] factory: a plan approval stays on the job through later gates (#2276) * test: preserve personal approval across later gates Co-Authored-By: Claude Opus 5.5 * test: supply the scripted worker name Co-Authored-By: Claude Opus 5.5 * test: use the manual Test step gate Co-Authored-By: Claude Opus 5.5 * fix: retain settled approvals while later gates wait Later questions deleted the saved plan decision before the job read synthesized an empty waiting approval. Keep the decision for display and use the existing current-question checks for fresh gate consent. Co-Authored-By: Claude Opus 5.5 * test: reopen while the later approval gate waits Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * manual: approval heading doesn't take the restart question Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_approval_stays_test.go | 123 ++++++++++++++++++ .../changes/unreleased/2276-approval-stays.md | 13 ++ internal/factory/api/codec.go | 2 +- internal/factory/jobs.go | 10 ++ internal/factory/run/loop.go | 1 - internal/factory/run/report.go | 15 --- internal/factory/serve/approve_reask_test.go | 10 +- 7 files changed, 152 insertions(+), 22 deletions(-) create mode 100644 cmd/codeaf/appfactory_approval_stays_test.go create mode 100644 docs/changes/unreleased/2276-approval-stays.md diff --git a/cmd/codeaf/appfactory_approval_stays_test.go b/cmd/codeaf/appfactory_approval_stays_test.go new file mode 100644 index 0000000000..3a94db2e23 --- /dev/null +++ b/cmd/codeaf/appfactory_approval_stays_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/client" +) + +// A scripted worker asks successive gates through the real report door. The +// app reads the decisions through factory.job, including after a fresh launch. +func TestAppFactoryApprovalSurvivesLaterGatesAndRestart(t *testing.T) { + s := newLocalSetup(t) + st := s.start() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + worker := client.Open(st.URL, s.backend.factoryHub().tokenFor(st.Factory)).Worker("") + if _, err := worker.Attach(ctx, api.WorkerAttach{Machine: "approval-fixture", Name: "fixture", Handle: "owner", Slots: 1}); err != nil { + t.Fatal(err) + } + it, err := s.backend.local.host.floor.Create(factory.Item{ + Repo: "acme/api", Num: 54, Kind: factory.KindIssue, Title: "Keep my plan approval", State: factory.StateQueued, + Autonomy: factory.AutonomyManual, Starter: "owner", Cap: 10, + Stages: []factory.Stage{ + {Name: "plan", Kind: factory.StageCheck, On: true}, + {Name: "approve", Kind: factory.StageGate, On: true}, + {Name: "write", Kind: factory.StageCheck, On: true}, + {Name: "test", Kind: factory.StageCheck, On: true}, + }, + }) + if err != nil { + t.Fatal(err) + } + pulled, ok, err := worker.Pull(ctx) + if err != nil || !ok || pulled.Job.ID != it.ID { + t.Fatalf("pull: ok=%v err=%v job=%+v", ok, err, pulled.Job) + } + active := s.factoryWire + params := map[string]any{"factory": st.Factory, "job": it.ID} + read := func(w *factoryWire) api.Job { + t.Helper() + var j api.Job + w.must("factory.job", params, &j) + return j + } + report := func(r api.WorkerReport) { + t.Helper() + if err := worker.Report(ctx, pulled.Lease.ID, r); err != nil { + t.Fatal(err) + } + } + approve := func() factory.Approval { + t.Helper() + j := read(active) + if !j.Can.Approve { + t.Fatalf("current gate cannot be approved: %+v", j) + } + var acted appwire.FactoryJobActed + active.must("factory.job_approve", map[string]any{"factory": st.Factory, "job": it.ID, "expect": map[string]any{"version": j.Version}}, &acted) + if acted.Approval == nil || acted.Approval.State != "approving" || acted.Approval.DecidedBy != "owner" { + t.Fatalf("personal decision = %+v", acted.Approval) + } + time.Sleep(factory.ApprovalUndo + 20*time.Millisecond) + settled := read(active) + if settled.Approval == nil || settled.Approval.State != "approved" || settled.Approval.DecidedBy != "owner" { + t.Fatalf("settled decision = %+v", settled.Approval) + } + return *settled.Approval + } + assertDecision := func(where string, j api.Job, want factory.Approval) { + t.Helper() + if j.Approval == nil || !reflect.DeepEqual(*j.Approval, want) { + t.Fatalf("%s: decided approval lost: got %+v, want %+v", where, j.Approval, want) + } + } + report(api.WorkerReport{Steps: []api.StepReport{{Name: "plan", State: api.StepDone}}, Question: &api.Question{Text: factory.ApproveQuestion("plan"), FromStep: "approve", Kind: factory.QKindApprove}}) + first := approve() + report(api.WorkerReport{Steps: []api.StepReport{{Name: "write", State: api.StepRunning}}}) + assertDecision("later work", read(s.factoryWire), first) + report(api.WorkerReport{Steps: []api.StepReport{{Name: "write", State: api.StepDone}}, Question: &api.Question{Text: factory.NextStepQuestion("test"), FromStep: "test", Kind: factory.QKindApprove}}) + waiting := read(s.factoryWire) + assertDecision("Test gate", waiting, first) + if waiting.NeedsYou == nil || !waiting.Can.Approve || waiting.Steps[3].State != "waiting" { + t.Fatalf("Test must retain its own waiting gate: %+v", waiting) + } + if err := worker.Done(ctx, pulled.Lease.ID, api.WorkerDone{State: api.StateNeedsYou}); err != nil { + t.Fatal(err) + } + s.backend.Close() + w, _ := reopened(t, s.backend.factoryHub()) + var list appwire.FactoryList + w.must("factory.list", struct{}{}, &list) + after := read(w) + assertDecision("restart and reopen at Test gate", after, first) + if after.NeedsYou == nil || !after.Can.Approve || after.Steps[3].State != "waiting" { + t.Fatalf("restart passed or lost the Test gate: %+v", after) + } + active = w + second := approve() + if !second.At.After(first.At) { + t.Fatal("later gate reused the earlier consent") + } + var status appwire.FactoryLocalStatus + w.must("factory.local_status", map[string]any{"caller": "w1"}, &status) + worker = client.Open(status.URL, w.backend.factoryHub().tokenFor(st.Factory)).Worker("") + if _, err := worker.Attach(ctx, api.WorkerAttach{Machine: "approval-fixture", Name: "fixture", Handle: "owner", Slots: 1}); err != nil { + t.Fatal(err) + } + pulled, ok, err = worker.Pull(ctx) + if err != nil || !ok || pulled.Job.ID != it.ID { + t.Fatalf("resumed pull: ok=%v err=%v job=%+v", ok, err, pulled.Job) + } + report(api.WorkerReport{Steps: []api.StepReport{{Name: "test", State: api.StepDone}}}) + assertDecision("later decision", read(w), second) + if err := worker.Done(ctx, pulled.Lease.ID, api.WorkerDone{State: api.StateNeedsYou, Ready: true}); err != nil { + t.Fatal(err) + } +} diff --git a/docs/changes/unreleased/2276-approval-stays.md b/docs/changes/unreleased/2276-approval-stays.md new file mode 100644 index 0000000000..631b14f8f4 --- /dev/null +++ b/docs/changes/unreleased/2276-approval-stays.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: codeaf keeps plan approval visible at later gates +pr: 2276 +surface: [engine] +invalidates: + - "A later approval gate erased the earlier settled plan decision and who approved it. The job now keeps the latest settled decision visible until another approval begins, while each later gate still requires its own consent." +--- + +Your plan approval stays on the job through later steps, waiting gates and +reopening the app. A later approval has its own undo window and replaces the +single approval summary; Activity keeps the earlier receipts. Wire fields are +unchanged. diff --git a/internal/factory/api/codec.go b/internal/factory/api/codec.go index dd36054d70..b46b262ddb 100644 --- a/internal/factory/api/codec.go +++ b/internal/factory/api/codec.go @@ -149,7 +149,7 @@ func FromItem(it factory.Item) Job { HeadCommit: it.HeadCommit, Publishing: it.PublishStatus, Change: it.Change, Publish: it.Publish, CommentURL: it.CommentURL, Runner: it.WorktreeMachine, JobID: factory.JobKey(it), Ref: factory.JobRef(it), Priority: factory.PriorityFor(it), Size: factory.JobSize(it), Budget: it.Cap, StopAfterStep: it.StopAfterStep, Model: it.Model, AutomaticModels: it.AutomaticModels, FactoryModel: it.FactoryModel, Reason: factory.JobReason(it), DoneHow: factory.JobDoneHow(it), DoneAt: it.DoneAt, - DismissedBy: append([]string(nil), it.DismissedBy...), Approval: factory.JobApproval(it), + DismissedBy: append([]string(nil), it.DismissedBy...), Approval: factory.JobApprovalSummary(it), State: factory.JobState(it), Ready: it.State == factory.StateLanded, Gate: gate(it), AssignedTo: it.Person(), Assignee: it.Person(), AskedFirstReason: it.AskedFirstReason, diff --git a/internal/factory/jobs.go b/internal/factory/jobs.go index 307073af5f..33b51330fd 100644 --- a/internal/factory/jobs.go +++ b/internal/factory/jobs.go @@ -319,6 +319,16 @@ func JobReason(it Item) string { return "" } +// JobApprovalSummary keeps the latest settled decision visible while a later +// gate waits. JobApproval separately decides whether that gate needs consent. +func JobApprovalSummary(it Item) *Approval { + if it.Approval != nil && it.Approval.State == "approved" { + a := *it.Approval + return &a + } + return JobApproval(it) +} + func JobApproval(it Item) *Approval { kind := "" diff --git a/internal/factory/run/loop.go b/internal/factory/run/loop.go index 5ef695f14e..7ceb020e93 100644 --- a/internal/factory/run/loop.go +++ b/internal/factory/run/loop.go @@ -576,7 +576,6 @@ func (lp *floorLoop) askIn(ctx context.Context, c *loopCtl, i int, pending, qkin // the worker sends names the gate and not the step before it. // A worker still cannot pass the gate: only the person's door does. it.Stream.Cur = i - dropSettledPlanApproval(it) } if note != "" { ph.Note = note diff --git a/internal/factory/run/report.go b/internal/factory/run/report.go index 97334e361f..7d0f1c144b 100644 --- a/internal/factory/run/report.go +++ b/internal/factory/run/report.go @@ -293,18 +293,6 @@ func mergeClaims(it *factory.Item, claims []factory.Claim) { } } -// dropSettledPlanApproval frees the item's decision slot when a new approve -// question arrives after a plan approval already settled: THE OLD CONSENT NEVER -// CARRIES TO A NEW QUESTION, so the new ask needs a new approval with its own -// five-second window. Only a settled plan approval goes; one still inside its -// window ("approving") belongs to undo and settle, and any other kind is not -// this question's. The factory record keeps the earlier consent. -func dropSettledPlanApproval(it *factory.Item) { - if a := it.Approval; a != nil && a.Kind == "plan" && a.State == "approved" { - it.Approval = nil - } -} - func reportQuestion(it *factory.Item, question *factory.Asked, kind string, now time.Time) error { if question == nil { return nil @@ -326,9 +314,6 @@ func reportQuestion(it *factory.Item, question *factory.Asked, kind string, now setPhaseState(&it.Stream.Phases[i], factory.PhaseWaiting, now) } it.Asking = &q - if kind == factory.QKindApprove { - dropSettledPlanApproval(it) - } it.State = factory.StateNeedsYou it.Question, it.QKind = q.Question, kind if it.QKind == "" { diff --git a/internal/factory/serve/approve_reask_test.go b/internal/factory/serve/approve_reask_test.go index ed8ac244c3..2f0568dde3 100644 --- a/internal/factory/serve/approve_reask_test.go +++ b/internal/factory/serve/approve_reask_test.go @@ -13,9 +13,9 @@ import ( // TestSoloSettledPlanApprovalNeverCoversTheNextAsk pins the old-consent law over // HTTP in solo mode: a plan approval that settled answers the question it was -// given for, and when the worker asks the gate again the job reads "waiting" -// with Approve available, the second approve writes a SECOND receipt behind its -// own five-second window, and Undo works only inside that window. +// given for. A later ask keeps the decided approval visible and offers Approve +// for the new ask. A second approval writes its own receipt behind its own +// five-second window, and Undo works only inside that window. func TestSoloSettledPlanApprovalNeverCoversTheNextAsk(t *testing.T) { f := newV2Jobs(t) f.h.Close() @@ -42,8 +42,8 @@ func TestSoloSettledPlanApprovalNeverCoversTheNextAsk(t *testing.T) { t.Fatal(err) } j := f.read(t, it) - if j.Approval == nil || j.Approval.State != "waiting" || !j.Can.Approve { - t.Fatalf("the old consent covers the new ask: approval=%+v can=%+v", j.Approval, j.Can) + if j.Approval == nil || j.Approval.State != "approved" || j.Approval.DecidedBy != "sam" || !j.Can.Approve { + t.Fatalf("the decision or the new ask was lost: approval=%+v can=%+v", j.Approval, j.Can) } f.post(t, it, "approve", "{}", 200) From b8c64d811df23bc7a95f947f64202371eb028d40 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:38:55 -0500 Subject: [PATCH 24/28] session: remove stray lines a stacked landing left in usage_ledger.go (#2279) The landing of #2275 auto-merged a duplicate of the usage writer's drain loop into announceUsageWritten, where its variables do not exist, so the package did not build. This restores the file to the tested copy. Co-authored-by: Claude Opus 5.5 --- internal/session/usage_ledger.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/session/usage_ledger.go b/internal/session/usage_ledger.go index 10c218da24..662181b16b 100644 --- a/internal/session/usage_ledger.go +++ b/internal/session/usage_ledger.go @@ -723,20 +723,6 @@ func announceUsageWritten() { for _, watch := range usageWatchers.of { watch() } - for work := range w.queue { - if work.done != nil { - w.writeOverflow(write) - close(work.done) - continue - } - if work.line != nil { - write(work.line) - } - if len(w.queue) == 0 { - w.writeOverflow(write) - } - } - w.writeOverflow(write) } // openUsageLedger opens one ledger for appending, creating its directory, and From ca73f867ca4a1caf9f1a71088566bcaab862630c Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:25:41 -0500 Subject: [PATCH 25/28] Memory list says where to add a model key (#2278) * memory: red test for the no-key words Co-Authored-By: Claude Opus 5.5 * memory: the no-key refusal names where to add a key Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_memory_switch_test.go | 16 ++++++++++++++++ cmd/codeaf/appsettings.go | 3 +++ .../unreleased/2278-memory-no-key-words.md | 10 ++++++++++ 3 files changed, 29 insertions(+) create mode 100644 docs/changes/unreleased/2278-memory-no-key-words.md diff --git a/cmd/codeaf/appfactory_memory_switch_test.go b/cmd/codeaf/appfactory_memory_switch_test.go index e07bd081b6..0958d90120 100644 --- a/cmd/codeaf/appfactory_memory_switch_test.go +++ b/cmd/codeaf/appfactory_memory_switch_test.go @@ -289,3 +289,19 @@ func TestMemoryStoreFailureNamesItsReasonNotOff(t *testing.T) { t.Fatalf("memoryOf error = %v", err) } } + +// With no key at launch the memory list speaks to the person who has to add +// one, and names where. The variable name is for a developer's terminal. +func TestMemoryStoreWithNoKeySaysWhereToAddOne(t *testing.T) { + for _, cause := range []error{ + config.ErrNoAPIKey, + fmt.Errorf("open memory: %w", config.ErrNoAPIKey), + errors.New(config.ErrNoAPIKey.Error()), + } { + b := &appBackend{memoryStore: func() (*store.Store, error) { return nil, cause }} + _, err := b.memoryOf() + if err == nil || strings.Contains(err.Error(), config.APIKeyEnv) || !strings.Contains(err.Error(), "Settings › Accounts › Providers") { + t.Fatalf("memoryOf(%v) = %v", cause, err) + } + } +} diff --git a/cmd/codeaf/appsettings.go b/cmd/codeaf/appsettings.go index 8970b0a627..4b07b34b13 100644 --- a/cmd/codeaf/appsettings.go +++ b/cmd/codeaf/appsettings.go @@ -268,6 +268,9 @@ func (b *appBackend) memoryOf() (*store.Store, error) { return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read: this engine has no saved memory store") } brain, err := b.memoryStore() + if appMissingKey(err) { + return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read without a model key: add one in Settings › Accounts › Providers") + } if err != nil { return nil, appwire.Errorf(appwire.CodeEngine, "saved memory can't be read: %v", err) } diff --git a/docs/changes/unreleased/2278-memory-no-key-words.md b/docs/changes/unreleased/2278-memory-no-key-words.md new file mode 100644 index 0000000000..46cf70b546 --- /dev/null +++ b/docs/changes/unreleased/2278-memory-no-key-words.md @@ -0,0 +1,10 @@ +--- +kind: fixed +title: codeaf app's memory list says where to add a model key +pr: 2278 +surface: [engine] +invalidates: + - "Was true: with no key at launch the memory list read `saved memory can't be read: OPENROUTER_API_KEY is required`." +--- + +With no model key, the memory list now reads "saved memory can't be read without a model key: add one in Settings › Accounts › Providers". From e13ee2a1d3f99beba0e4eb24abf668335ee02890 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:59:40 -0500 Subject: [PATCH 26/28] Nothing automatic spends on an issue closed on GitHub (#2282) * test(factory): prevent automatic spending on closed issues Co-Authored-By: Claude Opus 5.5 * fix(factory): check GitHub before automatic issue spending Check standing before scouting, start-rule writes or paid arrival reads. Share closed answers for the process and keep unavailable GitHub reads permissive. Co-Authored-By: Claude Opus 5.5 * fix(factory): bound concurrent standing checks and compare persisted rows Share in-flight reads within the same deadline, retain only closed answers and include the repository owner in cache identity. Compare rows after their persistence round trip. Co-Authored-By: Claude Opus 5.5 * test(factory): verify closed answers are shared across automatic paths Reuse the existing canonical GitHub item key for the shared cache. Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * triage: a person's Read press still reads a closed issue Co-Authored-By: Claude Opus 5.5 * intake test and manual: the press isn't part of the shared check Co-Authored-By: Claude Opus 5.5 * manual: the automatic check, in the section about closed issues Co-Authored-By: Claude Opus 5.5 * intake test: unused row Co-Authored-By: Claude Opus 5.5 * factory: the automatic closed check lives below triage and intake Co-Authored-By: Claude Opus 5.5 * factory: the check's own package has no import of github Co-Authored-By: Claude Opus 5.5 * factory: a remembered closed answer lasts ten minutes, so a reopened issue is worked again Co-Authored-By: Claude Opus 5.5 * docs: the closed answer's real length Co-Authored-By: Claude Opus 5.5 * Take the gated copy of the union merged docs --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/factory_pipeline.go | 4 + cmd/codeaf/factory_triage.go | 6 +- .../unreleased/2282-closed-no-auto-spend.md | 12 ++ internal/factory/closed_for_automation.go | 71 +++++++++++ .../factory/closed_for_automation_test.go | 32 +++++ internal/factory/intake/closed_test.go | 120 ++++++++++++++++++ internal/factory/intake/intake.go | 9 +- internal/factory/triage/closed_test.go | 97 ++++++++++++++ internal/factory/triage/triage.go | 14 +- internal/manual/chat/factory.md | 4 +- 10 files changed, 363 insertions(+), 6 deletions(-) create mode 100644 docs/changes/unreleased/2282-closed-no-auto-spend.md create mode 100644 internal/factory/closed_for_automation.go create mode 100644 internal/factory/closed_for_automation_test.go create mode 100644 internal/factory/triage/closed_test.go diff --git a/cmd/codeaf/factory_pipeline.go b/cmd/codeaf/factory_pipeline.go index 3c27cbba7f..da438dfd2c 100644 --- a/cmd/codeaf/factory_pipeline.go +++ b/cmd/codeaf/factory_pipeline.go @@ -81,6 +81,10 @@ func factoryIntakeOwned(st *store.Store, owner func(context.Context) string) *in history, _ := record.Open(st, record.Options{}) look := factoryScoutLook(st, config.Load) return intake.New(st, intake.Options{ + Standing: factorygithub.Rechecker(func(ctx context.Context) string { + token, _ := factorygithub.TokenAt(ctx, factoryProfile("")) + return token + }), Owner: owner, Record: history, // An automatic start is looked at closer first, while Read new issues diff --git a/cmd/codeaf/factory_triage.go b/cmd/codeaf/factory_triage.go index 158226f761..a92208ccc7 100644 --- a/cmd/codeaf/factory_triage.go +++ b/cmd/codeaf/factory_triage.go @@ -9,6 +9,7 @@ import ( "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/codeaf/internal/config" + factorygithub "github.com/Agent-Field/codeaf/internal/factory/github" "github.com/Agent-Field/codeaf/internal/factory/intake" "github.com/Agent-Field/codeaf/internal/factory/store" "github.com/Agent-Field/codeaf/internal/factory/triage" @@ -76,7 +77,10 @@ func runFactoryTriage(ctx context.Context, st *store.Store, newIntake func(*stor // surface owns it in a window and stdout is the protocol in an // engine. An item that could not be read simply has no read. in := newIntake(st) - triage.Worker(ctx, st, call, factoryTriageEvery, nil, triage.Options{ + triage.Worker(ctx, st, call, factoryTriageEvery, nil, triage.Options{Standing: factorygithub.Rechecker(func(ctx context.Context) string { + token, _ := factorygithub.TokenAt(ctx, factoryProfile("")) + return token + }), AskFirst: in.AskFirst, AfterTick: in.Once, Off: func() bool { return !factoryReadsOnArrival(st) }, }) diff --git a/docs/changes/unreleased/2282-closed-no-auto-spend.md b/docs/changes/unreleased/2282-closed-no-auto-spend.md new file mode 100644 index 0000000000..513a847df2 --- /dev/null +++ b/docs/changes/unreleased/2282-closed-no-auto-spend.md @@ -0,0 +1,12 @@ +--- +kind: fixed +title: codeaf skips automatic spending on issues closed on GitHub +pr: 2282 +surface: [engine] +invalidates: + - "Was true: an issue closed on GitHub after arrival could still receive a paid read, automatic look or start-rule action. Is true now: a successful GitHub check reporting closed or merged skips that work and leaves the row unchanged." +--- + +Start rules and issue reads check GitHub before doing work. A closed answer is +remembered for ten minutes, so a reopened issue is worked again; Backlog cleanup is unchanged. Checks wait +at most five seconds. Missing access, errors and timeouts let work continue. diff --git a/internal/factory/closed_for_automation.go b/internal/factory/closed_for_automation.go new file mode 100644 index 0000000000..b2d89fe20d --- /dev/null +++ b/internal/factory/closed_for_automation.go @@ -0,0 +1,71 @@ +package factory + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "time" +) + +const ( + automaticStandingWait = 5 * time.Second + // A closed answer is good this long, so an issue reopened on GitHub is + // worked again without a restart. + automaticClosedKeep = 10 * time.Minute +) + +var automaticNow = time.Now + +type automaticKey struct { + root, item string +} + +type automaticStanding struct { + done chan struct{} + closed bool + until time.Time +} + +var automaticStandings sync.Map + +// ClosedForAutomation remembers a closed answer for ten minutes, shared by +// intake and arrival reads in this process. Failures and open answers never stop work. A +// store without a root can still check, but cannot share remembered answers. +func ClosedForAutomation(ctx context.Context, root string, it Item, standing func(context.Context, Item) (string, error)) bool { + if standing == nil || it.Origin != OriginForge || it.Num <= 0 { + return false + } + + ctx, cancel := context.WithTimeout(ctx, automaticStandingWait) + defer cancel() + entry := &automaticStanding{done: make(chan struct{})} + var key automaticKey + if root != "" { + root, _ = filepath.Abs(root) + key = automaticKey{root, fmt.Sprintf("%s/%s/%s#%d", it.Origin, it.Product, it.Repo, it.Num)} + value, loaded := automaticStandings.LoadOrStore(key, entry) + if loaded { + held := value.(*automaticStanding) + select { + case <-ctx.Done(): + return false + case <-held.done: + } + if automaticNow().Before(held.until) { + return held.closed + } + if !automaticStandings.CompareAndSwap(key, held, entry) { + return ClosedForAutomation(ctx, root, it, standing) + } + } + } + defer close(entry.done) + how, err := standing(ctx, it) + entry.closed = err == nil && ctx.Err() == nil && (how == "closed" || how == "merged") + entry.until = automaticNow().Add(automaticClosedKeep) + if !entry.closed && root != "" { + automaticStandings.Delete(key) + } + return entry.closed +} diff --git a/internal/factory/closed_for_automation_test.go b/internal/factory/closed_for_automation_test.go new file mode 100644 index 0000000000..d2230504fb --- /dev/null +++ b/internal/factory/closed_for_automation_test.go @@ -0,0 +1,32 @@ +package factory + +import ( + "context" + "testing" + "time" +) + +func TestAReopenedIssueIsWorkedAgainWithinTheKeep(t *testing.T) { + now := time.Now() + automaticNow = func() time.Time { return now } + t.Cleanup(func() { automaticNow = time.Now }) + how := "closed" + calls := 0 + standing := func(context.Context, Item) (string, error) { calls++; return how, nil } + it := Item{Origin: OriginForge, Repo: "acme/api", Num: 7} + root := t.TempDir() + if !ClosedForAutomation(context.Background(), root, it, standing) { + t.Fatal("a closed issue should stop automatic work") + } + how = "open" + if !ClosedForAutomation(context.Background(), root, it, standing) || calls != 1 { + t.Fatalf("inside the keep: calls=%d, want the remembered answer", calls) + } + now = now.Add(automaticClosedKeep + time.Second) + if ClosedForAutomation(context.Background(), root, it, standing) { + t.Fatal("an issue reopened on GitHub stayed skipped") + } + if calls != 2 { + t.Fatalf("calls=%d, want one fresh read", calls) + } +} diff --git a/internal/factory/intake/closed_test.go b/internal/factory/intake/closed_test.go index 496b64e0f3..adefaac5a0 100644 --- a/internal/factory/intake/closed_test.go +++ b/internal/factory/intake/closed_test.go @@ -2,6 +2,8 @@ package intake_test import ( "context" + "errors" + "reflect" "testing" "time" @@ -10,8 +12,92 @@ import ( "github.com/Agent-Field/codeaf/internal/factory/pipeline" "github.com/Agent-Field/codeaf/internal/factory/record" "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/factory/triage" ) +func TestAutomaticIntakeChecksClosedIssues(t *testing.T) { + for _, policy := range []string{"start:\n labels: [ready]\n", "start:\n labels: [ready]\n ask_first: [changes secrets]\n"} { + for _, standing := range []string{"closed", "merged", "", "error", "nil"} { + t.Run(policy+standing, func(t *testing.T) { + ctx := context.Background() + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + before, err := st.Create(factory.Item{Repo: "owner/repo", Num: 1, Origin: factory.OriginForge, Kind: factory.KindIssue, State: factory.StateNew, Labels: []string{"ready", "size:S"}}) + if err != nil { + t.Fatal(err) + } + f, err := pipeline.Parse(policy) + if err != nil { + t.Fatal(err) + } + history, err := record.Open(st, record.Options{}) + if err != nil { + t.Fatal(err) + } + before, err = st.Get(before.ID) + if err != nil { + t.Fatal(err) + } + + reads, scouts, launches := 0, 0, 0 + opts := intake.Options{Record: history, Load: func(string) (pipeline.File, error) { return f, nil }, Scout: func(context.Context, int) error { scouts++; return nil }, Launch: func(id int) error { + launches++ + return st.Update(id, func(it *factory.Item) error { it.State = factory.StateQueued; return nil }) + }} + if standing != "nil" { + opts.Standing = func(ctx context.Context, _ factory.Item) (string, error) { + reads++ + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 5*time.Second { + t.Error("standing read lacks five second bound") + } + if standing == "error" { + return "", errors.New("unreachable") + } + return standing, nil + } + } + in := intake.New(st, opts) + for range 2 { + if err := in.Once(ctx); err != nil { + t.Fatal(err) + } + } + after, err := st.Get(before.ID) + if err != nil { + t.Fatal(err) + } + events, err := history.Since(ctx, time.Time{}) + if err != nil { + t.Fatal(err) + } + if standing == "closed" || standing == "merged" { + if scouts != 0 || launches != 0 || !reflect.DeepEqual(before, after) || len(events) != 0 { + t.Fatalf("closed issue spent or changed: scouts=%d launches=%d unchanged=%v events=%d", scouts, launches, reflect.DeepEqual(before, after), len(events)) + } + if reads != 1 { + t.Fatalf("closed issue across two ticks: reads=%d, want 1", reads) + } + } else { + ask := len(f.Start.AskFirst) > 0 + if ask { + if after.State != factory.StateNeedsYou || launches != 0 { + t.Fatalf("ask lost: state=%s launches=%d", after.State, launches) + } + } else if launches != 1 || scouts != 1 { + t.Fatalf("start lost: scouts=%d launches=%d", scouts, launches) + } + if standing != "nil" && reads != 1 { + t.Fatalf("decided issue read %d times, want 1", reads) + } + } + }) + } + } +} + func TestClosedJobCannotBeAutomaticallyStartedOrAsked(t *testing.T) { st, err := store.Open(t.TempDir()) if err != nil { @@ -48,3 +134,37 @@ func TestClosedJobCannotBeAutomaticallyStartedOrAsked(t *testing.T) { t.Fatalf("closed job was considered: reads=%d starts=%d state=%s", reads, starts, got.State) } } + +func TestClosedIssueCacheSharedByAutomaticPaths(t *testing.T) { + ctx := context.Background() + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + _, err = st.Create(factory.Item{Repo: "owner/repo", Num: 1, Origin: factory.OriginForge, Kind: factory.KindIssue, State: factory.StateNew, Labels: []string{"ready", "size:S"}}) + if err != nil { + t.Fatal(err) + } + history, err := record.Open(st, record.Options{}) + if err != nil { + t.Fatal(err) + } + reads := 0 + standing := func(context.Context, factory.Item) (string, error) { reads++; return "closed", nil } + opts := intake.Options{Standing: standing, Record: history, Load: func(string) (pipeline.File, error) { return pipeline.Parse("start:\n labels: [ready]\n") }, Launch: func(int) error { t.Error("closed issue launched"); return nil }} + if err := intake.New(st, opts).Once(ctx); err != nil { + t.Fatal(err) + } + // A new store handle and reader still share the process's closed answer. + again, err := store.Open(st.Root()) + if err != nil { + t.Fatal(err) + } + triage.ReadOne(ctx, again, func(context.Context, string) (string, error) { + t.Error("closed issue read") + return "", nil + }, triage.Options{Standing: standing}) + if reads != 1 { + t.Fatalf("shared closed answer: reads=%d", reads) + } +} diff --git a/internal/factory/intake/intake.go b/internal/factory/intake/intake.go index 889ad08ebb..59eef31240 100644 --- a/internal/factory/intake/intake.go +++ b/internal/factory/intake/intake.go @@ -19,8 +19,10 @@ import ( // Options injects repository policy and the existing launch door; tests need no model. type Options struct { - Refresh func(context.Context) error - Load func(string) (pipeline.File, error) + // Standing reads GitHub before an automatic start or ask. Nil keeps the existing policy. + Standing func(context.Context, factory.Item) (string, error) + Refresh func(context.Context) error + Load func(string) (pipeline.File, error) // LoadJob reads ONE JOB'S OWN FILE (pipeline.JobFile): the one it was // decided on ([factory.Item.DecidePipeline]), else its repository's as // Load reads it. Nil reads Load. Intake decides a job's file when it @@ -129,6 +131,9 @@ func (in *Intake) decide(ctx context.Context, it factory.Item, book estimate.Boo return err } decision := f.DecideJob(it, Job(it).Paths, Cost(book, it)) + if (decision.Start || decision.Ask) && factory.ClosedForAutomation(ctx, in.st.Root(), it, in.opts.Standing) { + return nil + } if decision.Start { if looked, ok := in.lookFirst(ctx, it); ok { it = looked diff --git a/internal/factory/triage/closed_test.go b/internal/factory/triage/closed_test.go new file mode 100644 index 0000000000..5e2c61bb03 --- /dev/null +++ b/internal/factory/triage/closed_test.go @@ -0,0 +1,97 @@ +package triage_test + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/factory/triage" +) + +func TestArrivalReadChecksClosedIssues(t *testing.T) { + for _, door := range []string{"tick"} { + for _, standing := range []string{"closed", "merged", "", "error", "nil", "local", "unnumbered"} { + t.Run(door+"/"+standing, func(t *testing.T) { + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + item := factory.Item{Repo: "owner/repo", Num: 1, Origin: factory.OriginForge, Kind: factory.KindIssue, State: factory.StateNew} + if standing == "local" { + item.Origin = factory.OriginChat + } + if standing == "unnumbered" { + item.Num = 0 + } + before, err := st.Create(item) + if err != nil { + t.Fatal(err) + } + before, err = st.Get(before.ID) + if err != nil { + t.Fatal(err) + } + + reads, calls := 0, 0 + opts := triage.Options{} + if standing != "nil" { + opts.Standing = func(ctx context.Context, _ factory.Item) (string, error) { + reads++ + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 5*time.Second { + t.Error("standing read lacks five second bound") + } + if standing == "error" { + return "", errors.New("unreachable") + } + if standing == "local" || standing == "unnumbered" { + t.Error("non-forge or unnumbered row checked") + } + return standing, nil + } + } + call := func(context.Context, string) (string, error) { calls++; return `{"read":"Fix the issue"}`, nil } + for range 2 { + triage.ReadOne(context.Background(), st, call, opts) + } + after, err := st.Get(before.ID) + if err != nil { + t.Fatal(err) + } + if standing == "closed" || standing == "merged" { + if calls != 0 || !reflect.DeepEqual(before, after) { + t.Fatalf("closed issue read or changed: calls=%d unchanged=%v", calls, reflect.DeepEqual(before, after)) + } + if reads != 1 { + t.Fatalf("closed issue across two ticks: reads=%d, want 1", reads) + } + } else if calls != 1 { + t.Fatalf("open or unavailable issue: calls=%d, want 1", calls) + } + }) + } + } +} + +// A person's Read press reads: they asked, and a button that did nothing would lie. +func TestAPersonsReadPressReadsAClosedIssue(t *testing.T) { + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + row, err := st.Create(factory.Item{Repo: "acme/api", Product: "acme", Num: 1, Kind: factory.KindIssue, Origin: factory.OriginForge, Title: "t", State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + calls := 0 + call := func(context.Context, string) (string, error) { calls++; return `{"read":"Fix the issue"}`, nil } + opts := triage.Options{Standing: func(context.Context, factory.Item) (string, error) { return "closed", nil }} + got := triage.ReadThese(context.Background(), st, call, []int{row.ID}, opts) + if calls != 1 || len(got.Read) != 1 { + t.Fatalf("press on a closed issue: calls=%d result=%+v", calls, got) + } +} diff --git a/internal/factory/triage/triage.go b/internal/factory/triage/triage.go index 64c6957896..c2c362bb5c 100644 --- a/internal/factory/triage/triage.go +++ b/internal/factory/triage/triage.go @@ -426,6 +426,9 @@ const CallTries = 3 // that another window read meanwhile, or that a person dismissed. // Options connects policy reading and intake without giving triage launch authority. type Options struct { + // Standing reads GitHub before the worker pays for an arrival read. A person's + // Read press ([ReadThese]) isn't checked: they asked. Nil leaves reads unchanged. + Standing func(context.Context, factory.Item) (string, error) AskFirst func(factory.Item) []string AfterTick func(context.Context) error // Off is "Read new issues as they arrive" turned off: a tick reads @@ -521,7 +524,7 @@ func (w *worker) once(ctx context.Context) tick { var it factory.Item found := false for _, x := range items { - if factory.IsIssueJob(x) && Wants(x) && !sw.OffJob(x) { + if factory.IsIssueJob(x) && Wants(x) && !sw.OffJob(x) && !w.closed(ctx, x) { it, found = x, true break } @@ -532,6 +535,15 @@ func (w *worker) once(ctx context.Context) tick { return w.read(ctx, it, CallTries) } +// closed shares the process cache with intake when the store has a root. +func (w *worker) closed(ctx context.Context, it factory.Item) bool { + root := "" + if st, ok := w.st.(interface{ Root() string }); ok { + root = st.Root() + } + return factory.ClosedForAutomation(ctx, root, it, w.opts.Standing) +} + // read reads one item, marked in flight. A call that fails is stamped (read, // with nothing learned) only at its tries-th failure, so a passing outage is // tried again; tries 1 never stamps a failure, for a person who can retry. diff --git a/internal/manual/chat/factory.md b/internal/manual/chat/factory.md index dd31b55bee..054b562b0b 100644 --- a/internal/manual/chat/factory.md +++ b/internal/manual/chat/factory.md @@ -435,7 +435,7 @@ The manager never changes the plan from a comment on its own yet, even one that ## starting a job checks GitHub first — closed issue, wasn't started, Check GitHub now, last checked -The factory reads GitHub on its own about once a minute, but those reads list open issues only, so a job never learns from them that its issue was closed. So when a person starts a job that came from GitHub (start, run or run again), the factory reads that one issue again first, in one request. If it was closed on GitHub, the job is not started: it is marked done (closed) and the answer is one sentence, such as `#12 was closed on GitHub, so it wasn't started.` If GitHub cannot be reached, the job starts anyway on what the factory already knows. +The factory reads GitHub on its own about once a minute, but those reads list open issues only, so a job never learns from them that its issue was closed. So when a person starts a job that came from GitHub (start, run or run again), the factory reads that one issue again first, in one request. If it was closed on GitHub, the job is not started: it is marked done (closed) and the answer is one sentence, such as `#12 was closed on GitHub, so it wasn't started.` If GitHub cannot be reached, the job starts anyway on what the factory already knows. An arrival read or a start rule checks GitHub once too, and a closed issue gets nothing from them; a Read press still reads. **Check GitHub now** asks the factory to read GitHub at once instead of on its next tick. It answers straight away and does not wait for the read; what the read finds arrives the way every read does. While it reads the factory says so, and afterwards it says when GitHub was last read without a failure, or what went wrong (`token refused`, `not reachable`). A factory whose poll is not running (no token, nothing watched) says nothing about GitHub at all. @@ -698,5 +698,5 @@ Other amounts are refused. Stop keeps the recorded spend and work. Today's figure on Spending counts every model call on this computer's ledger, and it moves as each one lands, not only when a chat turn ends. -The line under it splits today into factory jobs, `Factory AI` and your chats. `Factory AI` is what Factory spent outside any job: triage, scouts, and the AI on Factory's pages (asking about a repo, Build with AI, AI views). A call made for a job counts under factory jobs, whatever made it. These calls count against the one daily limit too. +The line under it splits today into factory jobs, `Factory AI` and your chats. `Factory AI` is what Factory spent outside any job: triage, scouts, and the AI on Factory's pages (AI notes and themes on a repo, asking about a repo, Build with AI, AI views). Afi's small errands count under your chats. A call made for a job counts under factory jobs, whatever made it. These calls count against the one daily limit too. From 19f4df46400691e1274eae535ea803057b2faf37 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:00:12 -0500 Subject: [PATCH 27/28] Repo AI notes and themes count as Factory AI (#2281) * spend: red test, repo notes and themes are Factory AI Co-Authored-By: Claude Opus 5.5 * spend: repo notes, themes, evals and briefs are Factory AI Co-Authored-By: Claude Opus 5.5 * manual: AI notes and themes are Factory AI Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appspendreport_test.go | 6 +++++- docs/changes/unreleased/2281-factory-ai-purposes.md | 10 ++++++++++ internal/session/daily_spend.go | 8 +++++--- 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 docs/changes/unreleased/2281-factory-ai-purposes.md diff --git a/cmd/codeaf/appspendreport_test.go b/cmd/codeaf/appspendreport_test.go index beab85a36c..bed22fca06 100644 --- a/cmd/codeaf/appspendreport_test.go +++ b/cmd/codeaf/appspendreport_test.go @@ -189,6 +189,10 @@ func TestSpendTodayNamesFactoryAI(t *testing.T) { {Role: "repo_ask", USD: .0061, Input: 9}, {Role: "pipeline_ai", USD: .0041, Input: 9}, {Role: "view_topic", USD: .0014, Input: 9}, + {Role: "repo_notes", USD: .0024, Input: 9}, + {Role: "repo_themes", USD: .0037, Input: 9}, + {Role: "repo_eval", USD: .0005, Input: 9}, + {Role: "factory_brief", USD: .0002, Input: 9}, {Role: "markreader", USD: .0109, Input: 9}, {USD: .02, Input: 9}, {Role: "factory triage", USD: .03, Input: 9, FactoryJob: "acme/api#1"}, @@ -203,7 +207,7 @@ func TestSpendTodayNamesFactoryAI(t *testing.T) { if err != nil { t.Fatal(err) } - if spend.FactoryAIUSD == nil || math.Abs(*spend.FactoryAIUSD-.1234) > 1e-9 || math.Abs(spend.TodayUSD-.1843) > 1e-9 { + if spend.FactoryAIUSD == nil || math.Abs(*spend.FactoryAIUSD-.1302) > 1e-9 || math.Abs(spend.TodayUSD-.1911) > 1e-9 { t.Fatalf("spend.today = %+v", spend) } } diff --git a/docs/changes/unreleased/2281-factory-ai-purposes.md b/docs/changes/unreleased/2281-factory-ai-purposes.md new file mode 100644 index 0000000000..55c8321d5f --- /dev/null +++ b/docs/changes/unreleased/2281-factory-ai-purposes.md @@ -0,0 +1,10 @@ +--- +kind: fixed +title: Repo notes and themes count as Factory AI on Spending +pr: 2281 +surface: [engine] +invalidates: + - "Was true: `factory_ai_usd` left out repo notes, repo themes, repo evals and a brief with no job, so they read as your chats." +--- + +Repo notes, repo themes, repo evals and a brief with no job now count in `factory_ai_usd`, not under your chats. Afi's small errands still count under your chats. diff --git a/internal/session/daily_spend.go b/internal/session/daily_spend.go index 35b69fcdc0..e31283668f 100644 --- a/internal/session/daily_spend.go +++ b/internal/session/daily_spend.go @@ -73,11 +73,13 @@ func ReadDailySpend(path string, now time.Time) (DailySpendReading, error) { // spendTally is one ledger's day as far as it has been read: the day summed, // the file read, how many bytes of whole lines, and the last of those bytes, // which prove the file still holds what was read. -// factoryAIRoles are the ledger roles of Factory's own AI: triage, scouts and the -// page AI. They aren't chats, so they don't count under "your chats". +// factoryAIRoles are the ledger roles of Factory's own AI: triage, scouts, the +// page AI and briefs. They aren't chats, so they don't count under "your chats". +// Afi's small errands are her own and stay out. var factoryAIRoles = map[string]bool{ "factory triage": true, "factory scout": true, - "repo_ask": true, "pipeline_ai": true, "view_topic": true, + "repo_ask": true, "repo_notes": true, "repo_themes": true, "repo_eval": true, + "pipeline_ai": true, "view_topic": true, "factory_brief": true, } type spendTally struct { From 63cff5f7cf0d6122f9067cdb86f79436dfa39ef8 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:49:34 -0500 Subject: [PATCH 28/28] A job has one manager, and all its spend counts against its budget (#2283) * test: reproduce lost job ownership and duplicated call receipts Co-Authored-By: Claude Opus 5.5 * fix: account for durable job ownership and journal call identity Competing manager opens remain owned after Talk selects another journal. Recognize the saved owner for live budget admission and spending. Match receipt rebuilds to the journal identity as well as the cell identity. Co-Authored-By: Claude Opus 5.5 * test: cover factory identity collisions and seeded session names Co-Authored-By: Claude Opus 5.5 * fix: keep factory budget ownership and seeded identities precise Validate durable ownership against the candidate store. A local worker mirror must share the owning floor manager path before taking an extra manager charge. Seed journal headers and titles from the session folder metadata in either layout. Co-Authored-By: Claude Opus 5.5 * test: racing asks make one manager Co-Authored-By: Claude Opus 5.5 * factory: one manager per job, with a lock across create and join Co-Authored-By: Claude Opus 5.5 * docs: one manager per job Co-Authored-By: Claude Opus 5.5 * changelog: name the pull request Co-Authored-By: Claude Opus 5.5 * factory: the mint lock wraps the work, so no closer is handed out Co-Authored-By: Claude Opus 5.5 * test: reproduce foreign manager charging a local budget Co-Authored-By: Claude Opus 5.5 * fix: bind manager budget ownership to the issue and creation time Old ownership marks keep their root, job and repository matching. Co-Authored-By: Claude Opus 5.5 * test: reproduce duplicate aliases hiding a new receipt Co-Authored-By: Claude Opus 5.5 * fix: count saved calls once across ledger session aliases Match call facts across aliases, retaining repeated calls under a single alias and leaving historical rows untouched. Co-Authored-By: Claude Opus 5.5 * test: reproduce unrelated mint blocked by recipe network read Co-Authored-By: Claude Opus 5.5 * test: reproduce remote Talk reusing an identical local row Co-Authored-By: Claude Opus 5.5 * fix: read recipes before locking manager minting Keep the held-manager fast path and recheck after locking. Network reads no longer hold the floor-wide mint lock. Co-Authored-By: Claude Opus 5.5 * fix: keep remote Talk on the worker mirror GitHub issues supply Created (github/source.go issueItem); Store.Create preserves it and api transport carries it. Independent factories can therefore share ID, Repo, Num and Created. The remote door must use its mirror; owned runners still consult their floor. Remote manager deduplication remains unsupported. Co-Authored-By: Claude Opus 5.5 * test: verify receipt refresh through the exported handoff seam Assert historical ledger bytes remain unchanged and repeated refresh imports no saved call twice. Co-Authored-By: Claude Opus 5.5 * test: distinguish saved calls by time and row identity Exercise exported Refresh with separate equal-cost calls, microdollar rounding, and repeated ledger IDs. Normalize Go formatting. Co-Authored-By: Claude Opus 5.5 * test: cover both owner identity fields and legacy marks Co-Authored-By: Claude Opus 5.5 * fix: match saved calls against distinct ledger row identities Match each alias to saved calls in order, using receipt precision for cost and allowing live usage to precede its seal. Count repeated row IDs once and append only missing calls, preserving historical ledger bytes. Co-Authored-By: Claude Opus 5.5 * fix: preserve legacy aggregate call identity and finish formatting Record the legacy prefix assumption for calls sharing a seal. Final review retains no blocking findings. Co-Authored-By: Claude Opus 5.5 * test: a remote job's conversation works in the floor's folder Co-Authored-By: Claude Opus 5.5 * factory: a remote job's conversation keeps the floor's folder Co-Authored-By: Claude Opus 5.5 * cellindex: count a chat's held rows by its fuller id, so nothing is booked again Co-Authored-By: Claude Opus 5.5 * test: rows under both ids are one prefix, as the rebuild writes them Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/codeaf/appfactory_owned_budget_test.go | 73 +++++++++ cmd/codeaf/factory_chat_owner.go | 13 +- cmd/codeaf/factory_chatspend.go | 43 ++++- cmd/codeaf/factory_foreign_owner_test.go | 61 +++++++ cmd/codeaf/factory_mint_network_test.go | 72 +++++++++ cmd/codeaf/factory_remote_doors.go | 2 +- cmd/codeaf/factory_remote_talk_test.go | 83 ++++++++++ cmd/codeaf/factory_talk.go | 118 +++++++++++--- cmd/codeaf/factory_talk_test.go | 47 ++++++ .../2283-job-agent-cost-ownership.md | 22 +++ internal/cellindex/usage.go | 65 ++++++-- .../cellindex/usage_duplicate_alias_test.go | 152 ++++++++++++++++++ internal/cellindex/usage_identity_test.go | 47 ++++++ .../cellindex/usage_legacy_aggregate_test.go | 52 ++++++ internal/manual/chat/factory.md | 2 + internal/session/talk_seed.go | 5 +- internal/session/talk_seed_identity_test.go | 29 ++++ 17 files changed, 835 insertions(+), 51 deletions(-) create mode 100644 cmd/codeaf/appfactory_owned_budget_test.go create mode 100644 cmd/codeaf/factory_foreign_owner_test.go create mode 100644 cmd/codeaf/factory_mint_network_test.go create mode 100644 cmd/codeaf/factory_remote_talk_test.go create mode 100644 docs/changes/unreleased/2283-job-agent-cost-ownership.md create mode 100644 internal/cellindex/usage_duplicate_alias_test.go create mode 100644 internal/cellindex/usage_identity_test.go create mode 100644 internal/cellindex/usage_legacy_aggregate_test.go create mode 100644 internal/session/talk_seed_identity_test.go diff --git a/cmd/codeaf/appfactory_owned_budget_test.go b/cmd/codeaf/appfactory_owned_budget_test.go new file mode 100644 index 0000000000..9af0d58db9 --- /dev/null +++ b/cmd/codeaf/appfactory_owned_budget_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + + "github.com/Agent-Field/codeaf/internal/appwire" + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/session" +) + +// A competing manager remains job-owned even when another conversation wins Talk. +func TestFactoryOwnedManagerChargesAfterTalkMoves(t *testing.T) { + for _, collision := range []bool{false, true} { + t.Run(strconv.FormatBool(collision), func(t *testing.T) { + s := newLocalSetup(t) + var calls atomic.Int64 + server := moneyProvider(t, &calls) + st := s.start() + floor := s.backend.local.host.floor + s.backend.open = func(ask appOpenAsk) (*appConn, error) { + return &appConn{Conv: &fakeConversation{}, Workspace: ask.Workspace, Transcript: ask.Transcript, Release: func() {}}, nil + } + it, err := floor.Create(factory.Item{Repo: "acme/api", Num: 7, Title: "Owned manager", Kind: factory.KindIssue, State: factory.StateNeedsYou, Cap: .005}) + if err != nil { + t.Fatal(err) + } + orphan, err := talkMaker(floor, t.TempDir(), filepath.Join(s.home, "profile"))(context.Background(), it) + if err != nil { + t.Fatal(err) + } + // A manager left by a race before the mint lock: on disk and owned, + // not the one the job chose. + if err := floor.Update(it.ID, func(cur *factory.Item) error { cur.Talk = ""; return nil }); err != nil { + t.Fatal(err) + } + var chats appwire.FactoryJobChats + s.must("factory.job_chats", map[string]any{"factory": st.Factory, "job": it.ID}, &chats) + if collision { + mirror, err := store.Open(filepath.Join(floor.Root(), "workers", strconv.Itoa(os.Getpid()), "other-factory")) + if err != nil { + t.Fatal(err) + } + foreign := it + foreign.Talk = filepath.Join(t.TempDir(), "foreign.jsonl") + foreign.Cap = .0001 + foreign.ChatSpent = .0001 + if err := mirror.Save(foreign); err != nil { + t.Fatal(err) + } + } + + a := moneyAgent(t, s, appOpenAsk{Workspace: filepath.Join(floor.Root(), "chats"), Transcript: orphan}, server.URL) + defer a.Close() + ask := session.ErrandAsk{Purpose: "ownership", Role: "brief", System: "Answer briefly.", Messages: []session.ErrandMessage{{Role: "user", Text: "Check ownership."}}, MaxTokens: 100} + answer, err := a.RunErrand(context.Background(), ask) + if err != nil || answer.Reason != "" { + t.Fatalf("first call: %+v %v", answer, err) + } + moneyTotals(t, s, st.Factory, it, &calls, .01) + _, _ = a.RunErrand(context.Background(), ask) + if calls.Load() != 1 { + t.Fatalf("job cap allowed %d provider calls, want 1", calls.Load()) + } + + }) + } +} diff --git a/cmd/codeaf/factory_chat_owner.go b/cmd/codeaf/factory_chat_owner.go index 36514848d8..abc27cf915 100644 --- a/cmd/codeaf/factory_chat_owner.go +++ b/cmd/codeaf/factory_chat_owner.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/Agent-Field/codeaf/internal/factory" "github.com/Agent-Field/codeaf/internal/factory/store" @@ -17,10 +18,12 @@ import ( const factoryOwnerFile = "factory-owner.json" type factoryChatOwner struct { - Step string `json:"step,omitempty"` - Root string `json:"root"` - Job int `json:"job"` - Repo string `json:"repo"` + Step string `json:"step,omitempty"` + Num int `json:"num,omitempty"` + Created time.Time `json:"created,omitempty"` + Root string `json:"root"` + Job int `json:"job"` + Repo string `json:"repo"` } // Ownership lives beside the journal, not in a crew that can be replaced by a @@ -29,7 +32,7 @@ func markFactoryChat(st *store.Store, it factory.Item, file string) error { if st == nil || it.ID <= 0 || strings.TrimSpace(file) == "" { return nil } - data, err := json.Marshal(factoryChatOwner{Root: chatKey(st.Root()), Job: it.ID, Repo: it.Repo}) + data, err := json.Marshal(factoryChatOwner{Root: chatKey(st.Root()), Job: it.ID, Repo: it.Repo, Num: it.Num, Created: it.Created}) if err != nil { return err } diff --git a/cmd/codeaf/factory_chatspend.go b/cmd/codeaf/factory_chatspend.go index 155ace8e3a..472999ffbb 100644 --- a/cmd/codeaf/factory_chatspend.go +++ b/cmd/codeaf/factory_chatspend.go @@ -65,7 +65,7 @@ func (d jobChatDoor) Chat(transcript string) session.JobChat { if talk.is(it.Talk) { return &jobChat{floor: d.floor, id: it.ID, talk: talk, name: factory.LedgerName(it)} } - if ownsStageChat(it, talk) { + if ownsBudgetChat(d.floor, it, talk) { return ownedStageBudget{&jobChat{floor: d.floor, id: it.ID, talk: talk, name: factory.LedgerName(it)}} } } @@ -79,7 +79,7 @@ func (d jobChatDoor) Chat(transcript string) session.JobChat { if talk.is(it.Talk) { return &jobChat{floor: d.floor, id: it.ID, talk: talk, name: factory.LedgerName(it)} } - if ownsStageChat(it, talk) { + if ownsBudgetChat(mirror, it, talk) { return ownedStageBudget{&jobChat{floor: d.floor, id: it.ID, talk: talk, name: factory.LedgerName(it)}} } } @@ -110,7 +110,7 @@ func (c ownedStageBudget) FactoryAuto() bool { func (c *jobChat) FactoryAuto() bool { for _, st := range c.holders() { it, err := st.Get(c.id) - if err == nil && (c.talk.is(it.Talk) || ownsStageChat(it, c.talk)) { + if err == nil && (c.owns(st, it)) { return factory.Automatic(it) } } @@ -162,7 +162,7 @@ func (c *jobChat) SpendModel(usd float64, model string, calls, input, output int func (c *jobChat) book(usd float64, model string, calls, input, output int) { for _, st := range c.holders() { err := st.Annotate(c.id, func(it *factory.Item) error { - if !c.talk.is(it.Talk) && !ownsStageChat(*it, c.talk) { + if !c.owns(st, *it) { return errNotThisChat } factory.AddChatSpend(it, usd) @@ -208,7 +208,7 @@ func (c *jobChat) readOf(st *store.Store) (chatRead, bool) { if err != nil { return chatRead{}, false } - r := chatRead{file: info, ours: c.talk.is(it.Talk) || ownsStageChat(it, c.talk)} + r := chatRead{file: info, ours: c.owns(st, it)} if r.ours { r.refusal = factoryrun.ChatRefusal(it) } @@ -357,6 +357,37 @@ func (c *jobChat) MaintainPR(ctx context.Context, words string) (string, error) return "Follow PR resumed with your request. Verified repairs will update the existing PR; merge remains yours.", nil } +// A local job's worker mirror carries the same saved manager as its floor. +// Matching that path keeps another factory's job with the same number and +// repository from taking a charge belonging to this floor. +func (c *jobChat) owns(st *store.Store, it factory.Item) bool { + if c.talk.is(it.Talk) || ownsBudgetChat(st, it, c.talk) { + return true + } + if chatKey(st.Root()) == chatKey(c.floor.Root()) { + return false + } + local, err := c.floor.Get(c.id) + return err == nil && talkPath(local.Talk).is(it.Talk) && ownsBudgetChat(c.floor, local, c.talk) +} + +// A manager made alongside another opener still belongs to its job after +// Talk chooses the other conversation. Its durable ownership also outlives +// stage history, so every owned conversation spends from the same budget. +func ownsBudgetChat(floor *store.Store, it factory.Item, talk talkKey) bool { + if ownsStageChat(it, talk) { + return true + } + owner, ok := readFactoryOwner(talk.clean) + if !ok || owner.Job != it.ID || owner.Repo != it.Repo || + (owner.Num != 0 && owner.Num != it.Num) || + (!owner.Created.IsZero() && !owner.Created.Equal(it.Created)) { + return false + } + root := chatKey(floor.Root()) + return owner.Root == root +} + func ownsStageChat(it factory.Item, talk talkKey) bool { if it.Stream == nil { return false @@ -377,7 +408,7 @@ func ownsStageChat(it factory.Item, talk talkKey) bool { func (c *jobChat) MemoryRefusal() string { // Admission follows the owning factory before a worker's delayed mirror. for _, st := range append([]*store.Store{c.floor}, liveMirrors(c.floor, strconv.Itoa(c.id)+".json")...) { - if it, err := st.Get(c.id); err == nil && (c.talk.is(it.Talk) || ownsStageChat(it, c.talk)) { + if it, err := st.Get(c.id); err == nil && (c.owns(st, it)) { return jobMemoryRefusal(it) } } diff --git a/cmd/codeaf/factory_foreign_owner_test.go b/cmd/codeaf/factory_foreign_owner_test.go new file mode 100644 index 0000000000..11ff0525b0 --- /dev/null +++ b/cmd/codeaf/factory_foreign_owner_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" +) + +func TestReviewForeignManagerDoesNotSpendLocalBudgetAfterWorkerLeaves(t *testing.T) { + for _, mode := range []string{"issue", "created", "legacy"} { + t.Run(mode, func(t *testing.T) { + st, web, profile := talkLab(t) + local, err := st.Create(factory.Item{Repo: "web", Num: 7, Title: "local", Cap: 10}) + if err != nil { + t.Fatal(err) + } + foreign := local + if mode == "issue" { + foreign.Num = 42 + } else { + foreign.Created = foreign.Created.Add(time.Hour) + } + foreign.Title = "foreign job" + chat, err := managerMaker(st, web, profile)(context.Background(), foreign) + if err != nil { + t.Fatal(err) + } + if mode == "legacy" { + body, err := json.Marshal(map[string]any{"root": chatKey(st.Root()), "job": local.ID, "repo": local.Repo}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(chat), factoryOwnerFile), body, 0600); err != nil { + t.Fatal(err) + } + } + // A remote worker has finished and no longer has a live mirror here. + budget := jobChatDoorOf(st).Chat(chat) + if budget != nil { + budget.Spend(.25) + } + got, err := st.Get(local.ID) + if err != nil { + t.Fatal(err) + } + want := 0.0 + if mode == "legacy" { + want = .25 + } + if got.ChatSpent != want { + t.Fatalf("foreign manager charged local job $%v; want $%v", got.ChatSpent, want) + } + + }) + } +} diff --git a/cmd/codeaf/factory_mint_network_test.go b/cmd/codeaf/factory_mint_network_test.go new file mode 100644 index 0000000000..cc531f8ed0 --- /dev/null +++ b/cmd/codeaf/factory_mint_network_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + factorygithub "github.com/Agent-Field/codeaf/internal/factory/github" + forge "github.com/Agent-Field/codeaf/internal/praf/github" +) + +func TestReviewMintNetworkDoesNotBlockOtherJob(t *testing.T) { + st, web, profile := talkLab(t) + entered, release := make(chan struct{}, 1), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case entered <- struct{}{}: + default: + } + <-release + json.NewEncoder(w).Encode(map[string]string{"type": "file", "encoding": "base64", "content": base64.StdEncoding.EncodeToString([]byte("default_budget: 1\n")), "sha": "review-mint"}) + })) + defer srv.Close() + loader := factorygithub.NewPipelineLoader(func(context.Context) factorygithub.PipelineAPI { return forge.NewClient("stub").WithBaseURL(srv.URL) }) + factoryRuleCaches.Lock() + factoryRuleCaches.profiles[factoryProfile("")] = loader + factoryRuleCaches.Unlock() + first, err := st.Create(factory.Item{Repo: "owner/repo", Kind: factory.KindIssue, Title: "network read", State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + second, err := st.Create(factory.Item{Repo: "web", Kind: factory.KindIssue, Title: "local job", State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + mint := talkMaker(st, web, profile) + firstDone := make(chan error, 1) + go func() { _, err := mint(context.Background(), first); firstDone <- err }() + select { + case <-entered: + case <-time.After(5 * time.Second): + close(release) + <-firstDone + t.Fatal("maker did not reach stub forge") + } + secondDone := make(chan error, 1) + go func() { _, err := mint(context.Background(), second); secondDone <- err }() + blocked := false + select { + case err := <-secondDone: + if err != nil { + t.Error(err) + } + case <-time.After(300 * time.Millisecond): + blocked = true + } + close(release) + if err := <-firstDone; err != nil { + t.Error(err) + } + if blocked { + if err := <-secondDone; err != nil { + t.Error(err) + } + t.Fatal("unrelated local job blocked on mint lock while another job waited for forge HTTP response") + } +} diff --git a/cmd/codeaf/factory_remote_doors.go b/cmd/codeaf/factory_remote_doors.go index fc8923269d..5dbd8f5309 100644 --- a/cmd/codeaf/factory_remote_doors.go +++ b/cmd/codeaf/factory_remote_doors.go @@ -126,7 +126,7 @@ func factoryLocalJobs(floor *store.Store, base, workspace, profileDir string, sn } seam := factory.LocalSeam(mirror, time.Now(), factoryPipelineOption(nil), factory.WithRepoDirs(factoryRepoDirs(floor, workspace)), - factory.WithTalk(talkMakerReading(floor, nil, workspace, profileDir)), + factory.WithTalk(talkMakerHolding(mirror, floor, nil, workspace, profileDir)), factory.WithRefetch(factorygithub.Refetcher(mirror, func(ctx context.Context) *factorygithub.Source { return factoryGitHub(ctx, floor, func(ctx context.Context) string { token, _ := factorygithub.TokenAt(ctx, factoryProfile(profileDir)) diff --git a/cmd/codeaf/factory_remote_talk_test.go b/cmd/codeaf/factory_remote_talk_test.go new file mode 100644 index 0000000000..44c7abd730 --- /dev/null +++ b/cmd/codeaf/factory_remote_talk_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/factory" + "github.com/Agent-Field/codeaf/internal/factory/api" + "github.com/Agent-Field/codeaf/internal/factory/store" +) + +func TestRemoteTalkDoesNotReuseIdenticalLocalJob(t *testing.T) { + floor, web, profile := talkLab(t) + local, err := floor.Create(factory.Item{Repo: "web", Num: 9, Created: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Kind: factory.KindIssue, State: factory.StateNew, Title: "same forge issue"}) + if err != nil { + t.Fatal(err) + } + first, err := talkMaker(floor, web, profile)(context.Background(), local) + if err != nil { + t.Fatal(err) + } + mirror, err := store.Open(filepath.Join(floor.Root(), "workers", strconv.Itoa(os.Getpid()))) + if err != nil { + t.Fatal(err) + } + remote := api.ToItem(api.FromItem(local)) + if err := mirror.Save(remote); err != nil { + t.Fatal(err) + } + base := "https://factory.example" + factoryLocalLease(mirror)(base, api.WorkerLease{ID: "remote-lease", Job: remote.ID, Worker: "here", ExpiresAt: time.Now().Add(time.Hour)}) + doors := factoryLocalJobs(floor, base, web, profile, factory.Snapshot{Items: []factory.Item{remote}}, nil) + door := doors[remote.ID].seam.Talk + if door == nil { + t.Fatal("remote job has no Talk door") + } + second, err := door(context.Background(), remote.ID) + if err != nil { + t.Fatal(err) + } + if second == first { + t.Fatalf("remote factory job got local factory manager %q (same ID, Repo, Num, Created)", second) + } + held, err := managerMaker(floor, web, profile)(context.Background(), local) + if err != nil { + t.Fatal(err) + } + if held != first { + t.Fatalf("owned runner lost held manager: got %q, want %q", held, first) + } +} + +// A worker's mirror goes when the worker ends; the conversation's folder stays +// with the floor. +func TestRemoteTalkWorksInTheFloorsFolder(t *testing.T) { + floor, web, profile := talkLab(t) + mirror, err := store.Open(filepath.Join(floor.Root(), "workers", strconv.Itoa(os.Getpid()))) + if err != nil { + t.Fatal(err) + } + remote := factory.Item{ID: 41, Repo: "elsewhere", Title: "remote", Kind: factory.KindIssue, State: factory.StateNew} + if err := mirror.Save(remote); err != nil { + t.Fatal(err) + } + base := "https://factory.example" + factoryLocalLease(mirror)(base, api.WorkerLease{ID: "remote-lease", Job: remote.ID, Worker: "here", ExpiresAt: time.Now().Add(time.Hour)}) + doors := factoryLocalJobs(floor, base, web, profile, factory.Snapshot{Items: []factory.Item{remote}}, nil) + chat, err := doors[remote.ID].seam.Talk(context.Background(), remote.ID) + if err != nil { + t.Fatal(err) + } + want, err := v3ProjectDir(filepath.Join(floor.Root(), "chats", strconv.Itoa(remote.ID))) + if err != nil { + t.Fatal(err) + } + if filepath.Dir(filepath.Dir(chat)) != want { + t.Fatalf("the conversation is in %q, want the floor's %q", chat, want) + } +} diff --git a/cmd/codeaf/factory_talk.go b/cmd/codeaf/factory_talk.go index 8fe8da5c6f..523d84bbf2 100644 --- a/cmd/codeaf/factory_talk.go +++ b/cmd/codeaf/factory_talk.go @@ -14,6 +14,7 @@ import ( "github.com/Agent-Field/codeaf/internal/factory/lease" factoryrun "github.com/Agent-Field/codeaf/internal/factory/run" "github.com/Agent-Field/codeaf/internal/factory/store" + "github.com/Agent-Field/codeaf/internal/filelock" "github.com/Agent-Field/codeaf/internal/home" "github.com/Agent-Field/codeaf/internal/session" "github.com/Agent-Field/codeaf/internal/teams" @@ -112,34 +113,99 @@ func talkMaker(st *store.Store, workspace, profileDir string) func(context.Conte // factory runs here), which carries the file it was decided on, so this // machine's own copy is never read for it, nor cleared by that read. func talkMakerReading(st, copies *store.Store, workspace, profileDir string) func(context.Context, factory.Item) (string, error) { - dirs := factoryRepoDirs(st, workspace) + return talkMakerHolding(st, st, copies, workspace, profileDir) +} + +// talkMakerHolding is [talkMakerReading] whose jobs live in st, while the +// conversation's folder and team belong to floor: a worker's mirror holds a +// remote job's row and goes when the worker ends; the conversation stays. +func talkMakerHolding(st, floor, copies *store.Store, workspace, profileDir string) func(context.Context, factory.Item) (string, error) { + dirs := factoryRepoDirs(floor, workspace) return func(_ context.Context, it factory.Item) (string, error) { - factoryReconcileOnce(profileDir, st) - where, err := talkFolder(st, dirs, it) - if err != nil { - return "", err - } - bucket, err := v3ProjectDir(where) - if err != nil { - return "", err - } - place, err := v3MintSession(bucket, where, v3StampLaunchDir(v3LaunchDir(), where), false) - if err != nil { - return "", err - } - transcript := place.Transcript() - if err := markFactoryChat(st, it, transcript); err != nil { - return "", err - } - name := talkTeamName(it) - if err := session.SeedConversation(transcript, where, name, talkBrief(it, factoryItemRecipe(copies, dirs, it))); err != nil { - return "", err - } - if err := talkJoinTeam(profileDir, it, transcript, where, name); err != nil { - return "", err - } - return transcript, nil + factoryReconcileOnce(profileDir, floor) + if _, held, err := talkHeld(st, it); held != "" || err != nil { + return held, err + } + // Recipe reads can reach GitHub. Finish the brief before the floor lock. + brief := talkBrief(it, factoryItemRecipe(copies, dirs, it)) + return withTalkMintLock(st, func() (string, error) { + ours, held, err := talkHeld(st, it) + if held != "" || err != nil { + return held, err + } + where, err := talkFolder(floor, dirs, it) + if err != nil { + return "", err + } + bucket, err := v3ProjectDir(where) + if err != nil { + return "", err + } + place, err := v3MintSession(bucket, where, v3StampLaunchDir(v3LaunchDir(), where), false) + if err != nil { + return "", err + } + transcript := place.Transcript() + if err := markFactoryChat(st, it, transcript); err != nil { + return "", err + } + name := talkTeamName(it) + if err := session.SeedConversation(transcript, where, name, brief); err != nil { + return "", err + } + if err := talkJoinTeam(profileDir, it, transcript, where, name); err != nil { + return "", err + } + if ours { + err = st.Update(it.ID, func(cur *factory.Item) error { + if strings.TrimSpace(cur.Talk) == "" { + cur.Talk = transcript + } + return nil + }) + if err != nil { + return "", err + } + } + return transcript, nil + }) + } +} + +// withTalkMintLock lets one maker at a time make a job's manager, across every +// process on this floor (the runner and the app's job chats race when a job +// starts). A second manager costs money and answers its team's replies, so the +// loser must never be created, not merely lose the write to Talk. +func withTalkMintLock(st *store.Store, mint func() (string, error)) (string, error) { + root := st.Root() + if root == "" { + return mint() + } + file, err := os.OpenFile(filepath.Join(root, "talk.lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return "", err + } + defer file.Close() + if err := filelock.Lock(file, true, false); err != nil { + return "", err + } + defer func() { _ = filelock.Unlock(file) }() + return mint() +} + +// talkHeld reads st's own row for it: ours says the row is +// this job, and held is the manager an earlier maker already kept on it. A job +// another factory handed over can share a number with a row here; that row's +// conversation is never its manager. +func talkHeld(st *store.Store, it factory.Item) (ours bool, held string, err error) { + cur, err := st.Get(it.ID) + if err != nil || cur.Repo != it.Repo || cur.Num != it.Num || !cur.Created.Equal(it.Created) { + return false, "", nil + } + if err := factory.RefuseDropped(cur); err != nil { + return true, "", err } + return true, strings.TrimSpace(cur.Talk), nil } // managerMaker is the runner's Options.Manager: the item's own conversation, diff --git a/cmd/codeaf/factory_talk_test.go b/cmd/codeaf/factory_talk_test.go index 4c3a5145f3..9c29bef1bb 100644 --- a/cmd/codeaf/factory_talk_test.go +++ b/cmd/codeaf/factory_talk_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -320,3 +321,49 @@ func TestFactoryItemDoorLaysARemovedOrMovedStepOutOfTheRun(t *testing.T) { }) } } + +// One manager per job: asks that race, as the runner and the app's job chats do +// when a job starts, all get the same conversation, and no other is made to join +// the job's team and wake on its replies. +func TestRacingAsksMakeOneManager(t *testing.T) { + st, web, profile := talkLab(t) + id, err := st.Create(factory.Item{Repo: "web", Kind: factory.KindIssue, Title: "fix the ledger", State: factory.StateNew}) + if err != nil { + t.Fatal(err) + } + maker := talkMaker(st, web, profile) + var wg sync.WaitGroup + chats := make([]string, 3) + begin := make(chan struct{}) + for i := range chats { + wg.Add(1) + go func() { + defer wg.Done() + <-begin + chat, err := maker(context.Background(), id) + if err != nil { + t.Error(err) + } + chats[i] = chat + }() + } + close(begin) + wg.Wait() + if chats[0] == "" || chats[0] != chats[1] || chats[1] != chats[2] { + t.Fatalf("three asks made %q", chats) + } + got, err := st.Get(id.ID) + if err != nil { + t.Fatal(err) + } + if got.Talk != chats[0] { + t.Fatalf("the job holds %q, the asks got %q", got.Talk, chats[0]) + } + made, err := os.ReadDir(filepath.Dir(filepath.Dir(chats[0]))) + if err != nil { + t.Fatal(err) + } + if len(made) != 1 { + t.Fatalf("%d conversations on disk, want 1", len(made)) + } +} diff --git a/docs/changes/unreleased/2283-job-agent-cost-ownership.md b/docs/changes/unreleased/2283-job-agent-cost-ownership.md new file mode 100644 index 0000000000..22dadb77f7 --- /dev/null +++ b/docs/changes/unreleased/2283-job-agent-cost-ownership.md @@ -0,0 +1,22 @@ +--- +kind: fixed +title: a job has one manager, and all its spend counts against its budget +pr: 2283 +surface: [engine] +invalidates: + - "Asks that raced when a job started could each make a manager, and each joined the job's team. Now one is made and every ask gets it." + - "Only the manager currently linked from a job counted toward its cost and cap. Every durably owned job conversation now counts, including competing manager opens." + - "Reopening a seeded conversation could repeat saved call receipts under its folder identity. Rebuild now recognizes its journal identity as well." +--- + +Starting a job owned by this factory and opening its chat at the same moment +now make one manager. Recipe reads finish before taking the mint lock, so a slow +GitHub read does not hold up an unrelated job. Remote Talk reads its worker +mirror, never a matching job on this floor. Remote Talk and runner opens may +still make separate managers; remote manager deduplication is not supported. +Extra managers left by earlier builds charge the job and stop at its budget, +even before a repository checkout exists. Reopening saved conversations keeps +calls already recorded from being charged twice. Existing duplicate ledger rows +are left unchanged and count once across folder and journal identities when +checking for new receipts. New ownership marks also check the issue number and +creation time before charging; older marks retain their previous matching. diff --git a/internal/cellindex/usage.go b/internal/cellindex/usage.go index e0efea65b1..6ac707f1b2 100644 --- a/internal/cellindex/usage.go +++ b/internal/cellindex/usage.go @@ -3,10 +3,12 @@ package cellindex import ( "bufio" "bytes" + "encoding/json" "errors" "io/fs" "math" "os" + "strings" "time" "github.com/Agent-Field/codeaf/internal/cell" @@ -29,25 +31,28 @@ func (usageIndex) Name() string { return "usage.jsonl" } // Present reports whether the ledger already names the session in any row. func (usageIndex) Present(c cell.Cell) bool { - return ledgerRows(session.UsageLedgerPath(), c.ID) > 0 + path := session.UsageLedgerPath() + if ledgerRows(path, c.ID) > 0 { + return true + } + return ledgerRows(path, usageJournalID(c)) > 0 } // Behind reports whether the chat spent on calls the ledger has no row for: the // copy this machine held was older than the one a take just brought. func (usageIndex) Behind(c cell.Cell, d session.Digest) bool { lines, err := usageLines(c, d) - return err == nil && len(lines) > ledgerRows(session.UsageLedgerPath(), c.ID) + return err == nil && len(lines) > heldRows(session.UsageLedgerPath(), c.ID, d.ID) } -// Build appends the rows the ledger lacks. Rows are in call order and a copy -// holds a prefix of them, so the rows already held are the first ones. +// Build appends the saved calls past the ones the ledger holds ([heldRows]). func (usageIndex) Build(c cell.Cell, d session.Digest) error { lines, err := usageLines(c, d) if err != nil { return err } path := session.UsageLedgerPath() - for _, line := range lines[min(ledgerRows(path, c.ID), len(lines)):] { + for _, line := range lines[min(heldRows(path, c.ID, d.ID), len(lines)):] { session.RecordUsage(path, line) } session.FlushUsage() @@ -73,13 +78,20 @@ func usageOfCall(c cell.Cell, d session.Digest, s cellstore.SealedCall) session. return session.UsageLine{ At: time.UnixMilli(s.SealedAtMs), Model: s.Model, Role: s.Role, Calls: 1, Input: int(s.TokensIn), Output: int(s.TokensOut), USD: float64(s.CostMicroUSD) / 1e6, - Session: c.ID, Workspace: d.Workspace, + Session: usageOwner(c, d), Workspace: d.Workspace, } } +// heldRows is how many of a chat's calls the ledger holds. Saved calls are in +// call order and a chat's rows under either id are a prefix of them, so old rows +// doubled under both ids count once, and a row is never booked again. +func heldRows(path, folderID, journalID string) int { + return max(ledgerRows(path, folderID), ledgerRows(path, journalID)) +} + // ledgerRows counts the ledger rows that name the session, without parsing any // row. An unreadable ledger answers as if it were full: it is not ours to append to. -func ledgerRows(path, session string) int { +func ledgerRows(path string, sessions ...string) int { f, err := os.Open(path) if errors.Is(err, fs.ErrNotExist) { return 0 @@ -88,16 +100,49 @@ func ledgerRows(path, session string) int { return math.MaxInt } defer f.Close() - marker := []byte(`"session":"` + session + `"`) + markers := make([][]byte, 0, len(sessions)) + for _, id := range sessions { + if id != "" { + markers = append(markers, []byte(`"session":"`+id+`"`)) + } + } n := 0 r := bufio.NewReaderSize(f, 1<<20) for { line, err := r.ReadBytes('\n') - if bytes.Contains(line, marker) { - n++ + for _, marker := range markers { + if bytes.Contains(line, marker) { + n++ + break + } } if err != nil { return n } } } + +// Seeded cells from older builds can have a journal identity different from +// their folder. Live calls use the journal identity; recognize both names so +// reopening never books the receipt again under its folder's name. +func usageJournalID(c cell.Cell) string { + f, err := os.Open((session.Place{Dir: c.Root}).Transcript()) + if err != nil { + return "" + } + defer f.Close() + var header struct { + ID string `json:"id"` + } + if json.NewDecoder(f).Decode(&header) != nil { + return "" + } + return strings.TrimSpace(header.ID) +} + +func usageOwner(c cell.Cell, d session.Digest) string { + if d.ID != "" { + return d.ID + } + return c.ID +} diff --git a/internal/cellindex/usage_duplicate_alias_test.go b/internal/cellindex/usage_duplicate_alias_test.go new file mode 100644 index 0000000000..ed431ff143 --- /dev/null +++ b/internal/cellindex/usage_duplicate_alias_test.go @@ -0,0 +1,152 @@ +package cellindex + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/executor" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestReviewDuplicateIdentityRowsDoNotHideNewReceipt(t *testing.T) { + bin := celltest.Engine(t) + t.Setenv(home.EnvVar, t.TempDir()) + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.Sandboxed}) + if err != nil { + t.Fatal(err) + } + path := (session.Place{Dir: c.Root}).Transcript() + if err := session.SeedConversation(path, "", "Job manager", "Manage this job."); err != nil { + t.Fatal(err) + } + d, err := session.ReadDigest(path) + if err != nil { + t.Fatal(err) + } + rec, err := cellstore.NewRecorder(nil, cellstore.Engine{Binary: bin, DataRoot: t.TempDir()}, c, filepath.Join(t.TempDir(), "wal"), cellstore.Options{}) + if err != nil { + t.Fatal(err) + } + for _, micro := range []int64{1000, 9000} { + rec.NoteModelCall(executor.ModelCall{Model: "m", TokensIn: 1, CostMicroUSD: micro}) + if err := rec.Around(context.Background(), executor.Call{Tool: "edit", Args: []byte(`{}`)}, executor.EffectLocal, cellstore.AgentRun, func() ([]byte, bool) { return nil, false }); err != nil { + t.Fatal(err) + } + } + calls, err := cellstore.ModelCalls(c) + if err != nil { + t.Fatal(err) + } + // Older versions wrote this first call once live, once during rebuild. + for _, id := range []string{d.ID, c.ID} { + session.RecordUsage(session.UsageLedgerPath(), session.UsageLine{At: time.UnixMilli(calls[0].SealedAtMs), Session: id, Model: "m", Calls: 1, Input: 1, USD: .001}) + } + session.FlushUsage() + before, err := os.ReadFile(session.UsageLedgerPath()) + if err != nil { + t.Fatal(err) + } + if _, err := Refresh(c, ""); err != nil { + t.Fatal(err) + } + if got := totalOf(t).micro; got != 11000 { + t.Fatalf("ledger microUSD = %d; want 11000 (existing duplicate 2000 + new real call 9000)", got) + } + after, err := os.ReadFile(session.UsageLedgerPath()) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(after, before) { + t.Fatal("refresh changed historical ledger rows") + } + if _, err := Refresh(c, ""); err != nil { + t.Fatal(err) + } + again, err := os.ReadFile(session.UsageLedgerPath()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, again) { + t.Fatal("refresh imported a saved call twice") + } +} + +func TestUsageAliasRowIdentity(t *testing.T) { + for _, mode := range []string{"exact duplicate", "rounded duplicate"} { + t.Run(mode, func(t *testing.T) { + t.Setenv(home.EnvVar, t.TempDir()) + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.Sandboxed}) + if err != nil { + t.Fatal(err) + } + path := (session.Place{Dir: c.Root}).Transcript() + if err := session.SeedConversation(path, "", "Manager", "Manage."); err != nil { + t.Fatal(err) + } + d, err := session.ReadDigest(path) + if err != nil { + t.Fatal(err) + } + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + saved := []session.UsageLine{ + {At: at, Model: "m", Calls: 1, Input: 1, USD: .001}, + {At: at.Add(time.Second), Model: "m", Calls: 1, Input: 1, USD: .009}, + } + rows := []session.UsageLine{saved[0], saved[0]} + const want = int64(11000) + switch mode { + case "rounded duplicate": + rows[0].USD = .0010004 + rows[0].At = at.Add(-time.Second) + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + for _, line := range saved { + err = json.NewEncoder(f).Encode(map[string]any{"type": "usage", "timestamp": line.At.Format(time.RFC3339Nano), "usage": map[string]any{"model": line.Model, "calls": 1, "input": 1, "costUsd": line.USD}}) + if err != nil { + t.Fatal(err) + } + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + rows[0].Session = d.ID + rows[1].Session = c.ID + for _, row := range rows { + session.RecordUsage(session.UsageLedgerPath(), row) + } + session.FlushUsage() + if _, err := Refresh(c, ""); err != nil { + t.Fatal(err) + } + if got := totalOf(t).micro; got != want { + t.Fatalf("ledger microUSD = %d; want %d", got, want) + } + before, err := os.ReadFile(session.UsageLedgerPath()) + if err != nil { + t.Fatal(err) + } + if _, err := Refresh(c, ""); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(session.UsageLedgerPath()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("refresh imported a call twice") + } + }) + } +} diff --git a/internal/cellindex/usage_identity_test.go b/internal/cellindex/usage_identity_test.go new file mode 100644 index 0000000000..646e77abf9 --- /dev/null +++ b/internal/cellindex/usage_identity_test.go @@ -0,0 +1,47 @@ +package cellindex + +import ( + "context" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/executor" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestUsageRebuildRecognizesSeededJournalIdentity(t *testing.T) { + bin := celltest.Engine(t) + t.Setenv(home.EnvVar, t.TempDir()) + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.Sandboxed}) + if err != nil { + t.Fatal(err) + } + path := (session.Place{Dir: c.Root}).Transcript() + if err := session.SeedConversation(path, "", "Job manager", "Manage this job."); err != nil { + t.Fatal(err) + } + d, err := session.ReadDigest(path) + if err != nil { + t.Fatal(err) + } + rec, err := cellstore.NewRecorder(nil, cellstore.Engine{Binary: bin, DataRoot: t.TempDir()}, c, filepath.Join(t.TempDir(), "wal"), cellstore.Options{}) + if err != nil { + t.Fatal(err) + } + rec.NoteModelCall(executor.ModelCall{Model: "m", TokensIn: 100, TokensOut: 7, CostMicroUSD: 1234}) + if err := rec.Around(context.Background(), executor.Call{Tool: "edit", Args: []byte(`{}`)}, executor.EffectLocal, cellstore.AgentRun, func() ([]byte, bool) { return nil, false }); err != nil { + t.Fatal(err) + } + session.RecordUsage(session.UsageLedgerPath(), session.UsageLine{Model: "m", Session: d.ID, Calls: 1, Input: 100, Output: 7, USD: .001234}) + session.FlushUsage() + if _, err := Rebuild(c, ""); err != nil { + t.Fatal(err) + } + if got := totalOf(t); got != (total{in: 100, out: 7, micro: 1234}) { + t.Fatalf("one provider call rebuilt twice: %+v", got) + } +} diff --git a/internal/cellindex/usage_legacy_aggregate_test.go b/internal/cellindex/usage_legacy_aggregate_test.go new file mode 100644 index 0000000000..a35914695a --- /dev/null +++ b/internal/cellindex/usage_legacy_aggregate_test.go @@ -0,0 +1,52 @@ +package cellindex + +import ( + "encoding/json" + "os" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestRefreshDoesNotRebookALegacyTurnAggregate(t *testing.T) { + t.Setenv(home.EnvVar, t.TempDir()) + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.Sandboxed}) + if err != nil { + t.Fatal(err) + } + path := (session.Place{Dir: c.Root}).Transcript() + if err := session.SeedConversation(path, "", "Legacy chat", "hello"); err != nil { + t.Fatal(err) + } + d, err := session.ReadDigest(path) + if err != nil { + t.Fatal(err) + } + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // A pre-model-call-receipt transcript stores one same-model aggregate per turn. + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + err = json.NewEncoder(f).Encode(map[string]any{"type": "usage", "timestamp": at.Add(time.Minute).Format(time.RFC3339Nano), "usage": map[string]any{"model": "m", "calls": 2, "input": 30, "output": 10, "costUsd": 0.75}}) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + // The live ledger already booked each of the two provider calls. + for i, usd := range []float64{0.25, 0.50} { + session.RecordUsage(session.UsageLedgerPath(), session.UsageLine{At: at.Add(time.Duration(i) * time.Second), Session: d.ID, Model: "m", Calls: 1, Input: 10 * (i + 1), Output: 5, USD: usd}) + } + session.FlushUsage() + if _, err := Refresh(c, ""); err != nil { + t.Fatal(err) + } + if got := totalOf(t).micro; got != 750000 { + t.Fatalf("refresh charged %d microUSD; want 750000: both saved calls were already booked", got) + } +} diff --git a/internal/manual/chat/factory.md b/internal/manual/chat/factory.md index 054b562b0b..ea223728ce 100644 --- a/internal/manual/chat/factory.md +++ b/internal/manual/chat/factory.md @@ -392,6 +392,8 @@ no running slot. **Run again** starts a job that says `Step stopped` over, taking the stop mark off; a job stopped with no mark has **Start** instead. It begins a new run at the first step. What the stopped run's steps reported is not carried over, and the job's spend starts again from nothing; its budget stays what it was. What is kept is the job's worktree and branch on the runner that ran it: the new run works in that same folder, on that branch, with every commit and every unsaved edit the stopped run left there. A different runner that picks the job up makes a fresh branch from the repository's default branch, because a job's branch leaves its machine only when the pull request is opened. Your notes on the job are kept, and every step's brief still carries them. +A job owned by this factory has one manager, however many asks race to open it. A remote job running here may still get separate managers from its Talk door and its runner; those opens are not deduplicated. Every conversation owned by a job, including an extra manager left by an earlier build, spends from that job’s budget. Its model calls appear on the job and count toward its cap, wherever its working folder is. Reopening a conversation does not charge its saved calls again. + To go on from a step that was cut instead, keeping the steps before it and the money spent, run that step again (try again) rather than starting the job. ## Stop after the job moves on or starts publishing diff --git a/internal/session/talk_seed.go b/internal/session/talk_seed.go index 486faf3147..0526d9afb2 100644 --- a/internal/session/talk_seed.go +++ b/internal/session/talk_seed.go @@ -2,7 +2,6 @@ package session import ( "errors" - "path/filepath" "strings" ) @@ -39,7 +38,7 @@ func SeedConversation(path, cwd, title, note string) error { if !journal.appendNote(textMessage("user", note), noteMarks{}) { return errors.New("the opening note could not be written") } - dir := filepath.Dir(path) + dir := DirOf(path) if title = strings.TrimSpace(title); title != "" { if meta, err := LoadMeta(dir); err == nil && strings.TrimSpace(meta.ID) != "" { meta.Title = title @@ -53,7 +52,7 @@ func SeedConversation(path, cwd, title, note string) error { // meta id when the journal sits in a session folder, and "" (a fresh id) when // it does not, which is [openSessionFile]'s own rule for a flat file. func sessionIDOfFolder(path string) string { - meta, err := LoadMeta(filepath.Dir(path)) + meta, err := LoadMeta(DirOf(path)) if err != nil { return "" } diff --git a/internal/session/talk_seed_identity_test.go b/internal/session/talk_seed_identity_test.go new file mode 100644 index 0000000000..2eac228820 --- /dev/null +++ b/internal/session/talk_seed_identity_test.go @@ -0,0 +1,29 @@ +package session_test + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/session" +) + +func TestSeededCellKeepsItsSessionIdentity(t *testing.T) { + c, err := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.Sandboxed}) + if err != nil { + t.Fatal(err) + } + if err := session.SaveMeta(c.Root, session.Meta{ID: c.ID}); err != nil { + t.Fatal(err) + } + path := (session.Place{Dir: c.Root}).Transcript() + if err := session.SeedConversation(path, "", "Manager", "Manage the job."); err != nil { + t.Fatal(err) + } + d, err := session.ReadDigest(path) + if err != nil { + t.Fatal(err) + } + if d.ID != c.ID { + t.Fatalf("seeded journal identity %q differs from session folder %q", d.ID, c.ID) + } +}