Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions eval/harness/agent-cli.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// What the agent runs in its directory: `./check` (the arm's compiler feedback) and `./test` (the visible tests).
// Every run is appended to `.runlog` (one JSON line each), which the score reads for iterations and when green came.
// Every run is appended to `.runlog` (one JSON line each: `at` when it ended, `ms` how long it took), which the score
// reads for iterations, when green came, and how much of the agent's time went to these tools.
// Usage: node harness/agent-cli.mjs <check|test> <dir>
import { appendFileSync } from "node:fs";
import path from "node:path";
Expand All @@ -8,7 +9,8 @@ import { runFile } from "./test.mjs";

const [cmd, dirArg] = process.argv.slice(2);
const dir = path.resolve(dirArg);
const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), cmd, ...entry }) + "\n");
const started = Date.now();
const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), ms: Date.now() - started, cmd, ...entry }) + "\n");

if (cmd === "check") {
const r = check(dir);
Expand Down
10 changes: 9 additions & 1 deletion eval/harness/score.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
// Scores finished runs: for each run directory (named <task>-<arm>-<n>), the arm's check, the visible and the hidden
// tests, and the agent's own log of `./check` and `./test` runs. Writes results.json and prints a summary table.
// Time: `toolMs` is the agent's time inside `./check` and `./test` (each `.runlog` entry's `ms`), `greenAfterMs` the time
// from its first run to its first passing `./test`; `agentMs`, the whole run's wall time, comes from `<run>/.agent.json`
// (`{ "ms": … }`, written by whoever ran the agent) when there is one. The model's own time is agentMs - toolMs.
// With --overlap, each task's hidden-overlap.mjs (round 3) runs too and counts toward the defect.
// Tests named in `alongside` are run and reported but do not count toward the defect (round 4 pre-registration).
// Usage: node harness/score.mjs <runs-dir> [out.json] [--overlap]
Expand Down Expand Up @@ -30,6 +33,10 @@ for (const name of readdirSync(runs).sort()) {
const alongside = round5.has(task) ? round5Alongside : round4Alongside;
const log = existsSync(path.join(dir, ".runlog")) ? readFileSync(path.join(dir, ".runlog"), "utf8").trim().split("\n").filter(Boolean).map(l => JSON.parse(l)) : [];
const firstGreen = log.findIndex(e => e.cmd === "test" && e.ok);
const startOf = e => Date.parse(e.at) - (e.ms ?? 0);
const toolMs = log.every(e => e.ms != null) && log.length ? log.reduce((t, e) => t + e.ms, 0) : null;
const greenAfterMs = firstGreen < 0 || !log.length ? null : Date.parse(log[firstGreen].at) - startOf(log[0]);
const agent = existsSync(path.join(dir, ".agent.json")) ? JSON.parse(readFileSync(path.join(dir, ".agent.json"), "utf8")) : null;
const checked = check(dir);
const visible = await runFile(dir, path.join(tasksDir, task, "visible.mjs"));
const hidden = await runFile(dir, path.join(tasksDir, task, "hidden.mjs"));
Expand All @@ -41,6 +48,7 @@ for (const name of readdirSync(runs).sort()) {
run: name, task, arm, n: Number(n),
checks: log.filter(e => e.cmd === "check").length, tests: log.filter(e => e.cmd === "test").length,
cyclesToGreen: firstGreen < 0 ? null : firstGreen + 1,
agentMs: agent?.ms ?? null, toolMs, greenAfterMs,
checkOk: checked.ok, built: hidden.build.ok,
visible: `${visible.results.filter(r => r.ok).length}/${visible.results.length}`,
hidden: `${hidden.results.length - failedHidden.length}/${hidden.results.length}`,
Expand All @@ -51,6 +59,6 @@ for (const name of readdirSync(runs).sort()) {
failedVisible: visible.results.filter(r => !r.ok).map(r => ({ name: r.name, error: r.error })),
});
const r = rows.at(-1);
console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}${r.failedAlongside.length ? " (alongside failed)" : ""}`);
console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.agentMs != null ? ` agent ${(r.agentMs / 60000).toFixed(1)}m` : ""}${r.toolMs != null ? ` tools ${(r.toolMs / 60000).toFixed(1)}m` : ""}${r.shippedDefect ? " DEFECT" : ""}${r.failedAlongside.length ? " (alongside failed)" : ""}`);
}
writeFileSync(outArg ? path.resolve(outArg) : path.join(runs, "results.json"), JSON.stringify(rows, null, 2) + "\n");
27 changes: 27 additions & 0 deletions eval/results/desk13-exact-2026-11-13/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# desk9 in Exact, Opus, 3 apps, our harness (2026-11-13)

Not preregistered. The `contract` arm (ported from expo/exact's bench patch, #59) run the way we ran the .rx probe:
Opus as general-purpose subagents, `--brief plain --inline-docs` (Exact's `docs/start-here.md` inlined, the rest of
its docs readable), `./check`/`./test` as the only build feedback (15 runs). Exact at main 8c742e6a, from a
`git archive` with `bench/` removed, so agents could not read Exact's own desk9 reference.

| app | hidden | overlap | fuzz holes | defects | lines (non-blank, non-comment) | subagent tokens | minutes |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| contract-1 | 77 | 37 | 0 | 1 | 605 | 155k | 12.4 |
| contract-2 | 76 | 37 | 0 | 2 | 568 | 145k | 11.4 |
| contract-3 | 77 | 38 | 0 | **0** | 605 | 168k | 10.5 |

Mean **1.0**. For comparison, the last three .rx apps (rx-7..9, same harness, guide with the lifetime rule, Solid
next f00ce6f52) score 2, 0, 0 = **0.67**, at 99-107k tokens, about 5 minutes and 411-452 lines each. Three apps per
arm: neither difference means anything statistically.

## The defects

- **Settings save, contract-1 and contract-2** (overlap "a later save failing while an earlier one is out shows its
error and then the earlier result"). Every issue write is a `parallel` mutation, but both apps left
`mutation saved` on Exact's default mode, under which a later save replaces the earlier one, so the earlier
result never shows. contract-3 made its save `parallel` too (`mutation saved … parallel refreshes settings`) and passes.
- **contract-2, "rows: row errors and detail errors leave each other alone"**: row actions and the detail share an
error path (agent).

No fuzz holes in any Exact app.
7 changes: 7 additions & 0 deletions eval/results/desk13-exact-2026-11-13/contract-1.grade.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
== hidden
77/77 passed
== overlap
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
37/38 passed
== fuzz
desk13-contract-1: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk13-exact-2026-11-13/contract-2.grade.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
✗ rows: row errors and detail errors leave each other alone
76/77 passed
== overlap
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
37/38 passed
== fuzz
desk13-contract-2: 0/20 seeds found a hole
6 changes: 6 additions & 0 deletions eval/results/desk13-exact-2026-11-13/contract-3.grade.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
== hidden
77/77 passed
== overlap
38/38 passed
== fuzz
desk13-contract-3: 0/20 seeds found a hole
23 changes: 23 additions & 0 deletions eval/results/desk13-probe-2026-11-13/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,26 @@ and no error boundary. Setup:

When the other action lands (its re-ask goes out), and then the detail lands, the `<h2>` renders empty while
`detail.body` beside it shows. Without the other action the test passes.

## On next f00ce6f52 + solidjs/solid#4835 (the #4834 fix, still an open PR)

Built at c1949ba3a. Both repros (`repro/blank-rows.spec.tsx`, `repro/blank-heading.spec.tsx`) pass. Grades in `graded-4835/`.

| app | hidden | overlap | fuzz holes | defects | before |
| --- | --- | --- | --- | --- | --- |
| rx-1 | 76 | 37 | 1 | 3 | 4 |
| rx-2 | 76 | 37 | 0 | 2 | 2 |
| rx-3 | 76 | 37 | 2 | 4 | 5 |
| rx-4 | 77 | 37 | 0 | 1 | 2 |
| rx-5 | 77 | 37 | 1 | 2 | 2 |
| rx-6 | 77 | 36 | 0 | 2 | 3 |
| rx-7 | 75 | 38 | 0 | 2 | 2 |
| rx-8 | 77 | 38 | 0 | 0 | 0 |
| rx-9 | 77 | 38 | 0 | 0 | 0 |
| reference | 77 | 38 | 0 | 0 | 0 |

"Before" is the last grade of each app: rx-1, rx-3 and rx-8 on f00ce6f52, the others on the Solid of their first grade.
The blank heading (seed 15 in rx-1 and rx-3) is gone. rx-4 s13 and rx-6 s16 (count outside the list's region) no
longer fire; rx-5 s17 still does. Everything left is agent or guide-caused, except two untraced holes: rx-1 s11
(the #3 detail shows #4's guessed title "Bravo"; the trail replayed alone ends correctly, so it may depend on timing)
and rx-3 s11/s19 (page hangs). Apps on the current guide (rx-7..9): 2, 0, 0.
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
== hidden
77/77 passed
== overlap
38/38 passed
== fuzz
ref-desk9: 0/20 seeds found a hole
10 changes: 10 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-1.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
== hidden
✗ base: tabs: the settings page loads again when reopened, with a fresh draft
76/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk13-rx-1: 1/20 seeds found a hole
seed 11 step 43: detail title: "#3 Bravofooter says 2019finding related helpretry assign to ", server "Delta"
tab help ; tab issues ; tab help ; land loadSettings: ; tab issues ; filter closed ; tab help ; land fetchIssues:open ; land fetchIssues:closed ; tab issues ; tab help ; tab issues ; focus ; land fetchIssues:open ; select #4 ; rename Alpha ; rename Bravo ; post me too ; post fixed? ; land fetchIssue:4 ; row-assign 0 ; row-close 0 ; select #3 ; focus ; close ; land fetchComments:4 ; row-close 0
8 changes: 8 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-2.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
✗ base: tabs: the settings page loads again when reopened, with a fresh draft
76/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk13-rx-2: 0/20 seeds found a hole
12 changes: 12 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-3.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
== hidden
✗ base: tabs: the settings page loads again when reopened, with a fresh draft
76/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk13-rx-3: 2/20 seeds found a hole
seed 11 step -1: page crash: session timed out (120 s): the page hangs

seed 19 step -1: page crash: session timed out (120 s): the page hangs

7 changes: 7 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-4.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
== hidden
77/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk13-rx-4: 0/20 seeds found a hole
9 changes: 9 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-5.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
== hidden
77/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk13-rx-5: 1/20 seeds found a hole
seed 17 step 8: count: tab Issues (2), 1 open rows shown
user bob ; land fetchIssues:open ; row-assign 1 ; land assign:3,bob ; land fetchIssues:open ; row-close 2 ; land closeIssue:4 ; row-close 1
8 changes: 8 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-6.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
77/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
✗ rows: a row action that answers after the issues page was left changes nothing on its return
36/38 passed
== fuzz
desk13-rx-6: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-7.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
✗ base: help: selecting after a failure shows loading…, not the old failure
✗ base: help: selecting after not found shows loading…, not the old message
75/77 passed
== overlap
38/38 passed
== fuzz
desk13-rx-7: 0/20 seeds found a hole
6 changes: 6 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-8.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
== hidden
77/77 passed
== overlap
38/38 passed
== fuzz
desk13-rx-8: 0/20 seeds found a hole
6 changes: 6 additions & 0 deletions eval/results/desk13-probe-2026-11-13/graded-4835/rx-9.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
== hidden
77/77 passed
== overlap
38/38 passed
== fuzz
desk13-rx-9: 0/20 seeds found a hole
52 changes: 52 additions & 0 deletions eval/results/desk14-2026-11-13/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Round 14 (exploratory): frontier .rx vs React vs Exact, 2026-11-13

Not preregistered. 18 Opus apps on the desk9 brief (`--brief plain --inline-docs`), 6 per arm, run in three waves of
six (two per arm per wave). Graded with the usual suite: hidden 77, overlap 38, 20 fuzz seeds (`--r9`).

- Solid next dba2aadbb (includes #4831 and #4835, the #4834 fix). rxc built from main + this branch (54853d0).
- React arm: React 19 + TypeScript, @tanstack/react-query installed and offered.
- Exact arm: Exact `contract` (app.contract + app.ts) from the checkout described in `desk13-exact-2026-11-13`.
- The .rx guide is the one-lifetime-rule guide from the desk13 probe.
- The hand-written .rx reference grades 0 (77/77, 38/38, 0/20).

## Results

| arm | defects per app | mean | apps at 0 | mean minutes | mean tokens |
| --- | --- | --- | --- | --- | --- |
| .rx | 1, 0, 0, 0, 0, 1 | **0.33** | 4/6 | 7.1 | 112k |
| React | 2, 3, 1, 3, 2, 2 | 2.17 | 0/6 | 6.7 | 89k |
| Exact | 2, 2, 2, 1, 0, 3 | 1.67 | 1/6 | 9.9 | 150k |

Minutes are each agent's wall time (from its task notification, in `<desk>/.agent.json`). Three waves of six ran on
one machine, so a wave's builds and browsers shared the CPU.

## What the defects are

- **.rx (2): both are page hangs, and both are a Solid bug.** rx-1 seed 2 and rx-6 seed 19 time out (120 s): selecting
another issue while a guess is out (a rename, or an assign) never returns. Reduced to
`repro/guess-select-loop.spec.tsx` (plain Solid, about 35 lines; filed solidjs/solid#4840): a detail rendered per selection through `<For>`
under a component that passes its children by getter, a heading memo reading a `createOptimistic` guess, and a
selection change while that guess's action is out. Solid's dev build reports "Potential Infinite Loop Detected. Kept
alive by scheduled work; last staged node: <For>"; the production build spins. rx-6 s19 is the rename case exactly;
rx-1 s2 has assign guesses out instead (not reduced separately). The same loop is behind desk13 rx-3 s11/s19.
- **React (13): mostly one class.** 6/6 lose a guessed comment or label when the list it sits in reloads while the
post or add is out (hidden "labels that load while an add is out keep the add" 6/6, "comments ... keep the post"
5/6). react-2 and react-4 also show a posted comment twice at rest (fuzz s16).
- **Exact (10): mostly the settings save.** 4/6 get overlapping saves wrong ("a later save failing while an earlier
one is out"; contract-3 also "an earlier save failing after a later one succeeded"), the default mutation mode
replacing an earlier write with a later one, as in the desk13 Exact probe. 2/6 let a row error and a detail error
clear each other; contract-1 doesn't reload settings when the page reopens (2 tests); contract-6 asks the server
for an invalid name.

## Also traced: desk13 rx-1 seed 11 (Solid, filed solidjs/solid#4841)

The detail of issue #3 showed issue #4's data. Instrumenting the app showed the detail memo computing #3's answer
while the screen kept #4's. `repro/wrong-detail-after-error.spec.tsx` (plain Solid): a heading memo reading a rename
guess; select #3 while closes and renames are out; #3's detail fails, a rename's refresh asks again, and the answer
lands while a close is still out. The screen shows "#3 Alpha (detail of #4)" instead of "#3 Delta (detail of #3)".

## Deviations

`deviations.txt`. Most agents in every arm added their own tests to `tests/visible.mjs` and restored the file;
contract-6 kept its extra tests in `.exact/extra.mjs`. rx-6 removed an unused import after its last test run (it
type-checks; grading builds it fine).
11 changes: 11 additions & 0 deletions eval/results/desk14-2026-11-13/deviations.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
rx-1: added 9 own tests to tests/visible.mjs, then restored it (agent says exactly)
rx-2: added 14 own tests to tests/visible.mjs, then restored it
contract-1: added 17 own tests to tests/visible.mjs, then restored it
contract-2: added 12 own tests to tests/visible.mjs, then restored it
contract-4: added 11 own tests to tests/visible.mjs, then removed
react-3: added 12 own tests to tests/visible.mjs, then restored
contract-3: added 13 own tests, then removed
rx-6: last edit (removed unused import) checked but not re-tested
react-6: added 9 own tests, then restored. contract-6: extra tests kept in .exact/extra.mjs (not graded)
contract-5: 15 extra tests, then restored
rx-5: 19 extra tests, then restored
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-1.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
✗ base: tabs: the settings page loads again when reopened, with a fresh draft
76/77 passed
== overlap
✗ base: tabs: a save that answers after the page was left shows the reloaded settings
37/38 passed
== fuzz
desk14-contract-1: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-2.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
✗ rows: row errors and detail errors leave each other alone
76/77 passed
== overlap
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
37/38 passed
== fuzz
desk14-contract-2: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-3.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
77/77 passed
== overlap
✗ base: settings: an earlier save failing after a later one succeeded leaves the later one
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
36/38 passed
== fuzz
desk14-contract-3: 0/20 seeds found a hole
7 changes: 7 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-4.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
== hidden
77/77 passed
== overlap
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
37/38 passed
== fuzz
desk14-contract-4: 0/20 seeds found a hole
6 changes: 6 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-5.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
== hidden
77/77 passed
== overlap
38/38 passed
== fuzz
desk14-contract-5: 0/20 seeds found a hole
9 changes: 9 additions & 0 deletions eval/results/desk14-2026-11-13/graded/contract-6.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
== hidden
✗ base: settings: saving an invalid name asks nothing
✗ rows: row errors and detail errors leave each other alone
75/77 passed
== overlap
✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
37/38 passed
== fuzz
desk14-contract-6: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-1.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
77/77 passed
== overlap
✗ base: comments that load while a post is out keep the post
✗ labels that load while an add is out keep the add
36/38 passed
== fuzz
desk14-react-1: 0/20 seeds found a hole
10 changes: 10 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-2.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
== hidden
77/77 passed
== overlap
✗ base: comments that load while a post is out keep the post
✗ labels that load while an add is out keep the add
36/38 passed
== fuzz
desk14-react-2: 1/20 seeds found a hole
seed 16 step 37: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping]
user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign
7 changes: 7 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-3.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
== hidden
77/77 passed
== overlap
✗ labels that load while an add is out keep the add
37/38 passed
== fuzz
desk14-react-3: 0/20 seeds found a hole
10 changes: 10 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-4.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
== hidden
77/77 passed
== overlap
✗ base: comments that load while a post is out keep the post
✗ labels that load while an add is out keep the add
36/38 passed
== fuzz
desk14-react-4: 1/20 seeds found a hole
seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping]
user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-5.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
77/77 passed
== overlap
✗ base: comments that load while a post is out keep the post
✗ labels that load while an add is out keep the add
36/38 passed
== fuzz
desk14-react-5: 0/20 seeds found a hole
8 changes: 8 additions & 0 deletions eval/results/desk14-2026-11-13/graded/react-6.grade
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
== hidden
77/77 passed
== overlap
✗ base: comments that load while a post is out keep the post
✗ labels that load while an add is out keep the add
36/38 passed
== fuzz
desk14-react-6: 0/20 seeds found a hole
Loading