Skip to content

Commit aa85cbd

Browse files
authored
Merge pull request #14 from ryansolid/claude/project-thread-t6465s
eval rounds 2 and 3, round 4 pre-registration; MO-1d, refresh under a whole-value guess, Solid repin ac28157a
2 parents e25b81b + 4ba4c71 commit aa85cbd

227 files changed

Lines changed: 12508 additions & 116 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎compiler/Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎compiler/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ regex = "1"
2626
serde_json = { version = "1", features = ["preserve_order"] }
2727
# CP-1b: Solid's compiler, at the commit ref/ runs on (ref/SOLID_NEXT), and the Oxc release it is built against (CP-1c:
2828
# the program crosses the seam, so the versions must match)
29-
solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "d2323b077c7dda32ebced7843c2ce2b788a91e8d", default-features = false }
29+
solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "ac28157aba6d03b806f0043bce70a87d515e953e", default-features = false }
3030
oxc_allocator = "0.144"
3131
oxc_ast = "0.144"
3232
oxc_span = "0.144"

‎compiler/src/check.rs‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ impl Diagnostic {
4040
}
4141
}
4242

43-
const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"];
43+
const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"];
4444
/// EF-5's cleanup shapes: a `return` of a closure anywhere in the body, a closure as its last expression, or an expression body that is one.
4545
fn returns_closure(f: &Expr) -> bool {
4646
fn in_stmts(ss: &[Stmt]) -> bool {
@@ -765,6 +765,8 @@ pub struct Checker<'a> {
765765
apps: Vec<AppRec<'a>>,
766766
actions: Vec<ActionRec<'a>>,
767767
provisions: Vec<Provision<'a>>,
768+
/// MO-1d: the symbols import declarations bound, which no declaration in a nested block may shadow.
769+
import_syms: HashSet<SId>,
768770
later: Vec<Later<'a>>,
769771
comp_syms: HashMap<RId, SId>,
770772
rows_busy: HashSet<String>,
@@ -838,6 +840,7 @@ impl<'a> Checker<'a> {
838840
apps: vec![],
839841
actions: vec![],
840842
provisions: vec![],
843+
import_syms: HashSet::new(),
841844
later: vec![],
842845
comp_syms: HashMap::new(),
843846
rows_busy: HashSet::new(),
@@ -1007,13 +1010,20 @@ impl<'a> Checker<'a> {
10071010
// 1. names (E-7: one declaration per name in a block)
10081011
for d in decls {
10091012
self.line = d.line() as i64;
1010-
if let Decl::Import { resolved: Some(_), .. } = d {
1011-
self.import_rx(d, env);
1012-
continue;
1013-
}
1014-
if let Decl::Import { declared: Some(_), .. } = d {
1015-
self.import_declared(d, env);
1016-
continue;
1013+
if let Decl::Import { names, .. } = d {
1014+
if matches!(d, Decl::Import { resolved: Some(_), .. }) {
1015+
self.import_rx(d, env);
1016+
} else if matches!(d, Decl::Import { declared: Some(_), .. }) {
1017+
self.import_declared(d, env);
1018+
}
1019+
if matches!(d, Decl::Import { resolved: Some(_), .. } | Decl::Import { declared: Some(_), .. }) {
1020+
for n in names {
1021+
if let Some(&s) = self.envs[env].vars.get(&n.alias) {
1022+
self.import_syms.insert(s);
1023+
}
1024+
}
1025+
continue;
1026+
}
10171027
}
10181028
match d {
10191029
Decl::Import { names, pure, fails, side, line, .. } => {
@@ -1022,7 +1032,8 @@ impl<'a> Checker<'a> {
10221032
s.io = !pure;
10231033
s.rejects = Some(fails.clone().unwrap_or_else(|| vec!["Error".into()]).into_iter().collect());
10241034
s.side = *side;
1025-
self.bind(env, &n.alias, s);
1035+
let s = self.bind(env, &n.alias, s);
1036+
self.import_syms.insert(s);
10261037
}
10271038
continue;
10281039
}
@@ -1059,6 +1070,11 @@ impl<'a> Checker<'a> {
10591070
self.err("DUPLICATE_DECLARATION", "E-7", format!("{} is declared twice in one block", name));
10601071
}
10611072
}
1073+
if let Some(shadowed) = self.envs[env].parent.and_then(|p| self.lookup(p, name)) {
1074+
if self.import_syms.contains(&shadowed) {
1075+
self.err("SHADOWED_IMPORT", "MO-1d", format!("{} shadows the imported {}: rename one (`import {{ {} as … }}`)", name, name, name));
1076+
}
1077+
}
10621078
let kind = match d {
10631079
Decl::Fn { .. } => SymKind::Fn,
10641080
Decl::Component { .. } => SymKind::Comp,

‎compiler/src/emit.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ fn fail<T>(msg: impl Into<String>) -> R<T> {
2121
Err(CompileError(msg.into()))
2222
}
2323

24-
const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"];
24+
const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"];
2525
fn builtin(n: &str) -> bool {
2626
BUILTINS.contains(&n)
2727
}

‎compiler/src/project.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pub const PRELUDE: &str = include_str!("prelude.ts");
1818

1919
static ROW_WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^((?:io|pending|lane|fails\([^)]*\))\s+)+").unwrap());
2020
static SOURCE_WORD: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^source\s+").unwrap());
21-
const BUILTINS: [&str; 12] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "log", "until", "onSettled", "onFlush"];
21+
const BUILTINS: [&str; 13] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf", "log", "until", "onSettled", "onFlush"];
2222
fn builtin(n: &str) -> bool {
2323
BUILTINS.contains(&n)
2424
}

‎eval/PREREG-change.md‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Round 4: changing an existing app
2+
3+
Pre-registration. Everything in this file was fixed and committed before the round's first agent run.
4+
5+
## Why this round
6+
7+
Rounds 2 and 3 (`results/plain-2026-10-27/`, `results/overlap-2026-10-27/`) had agents write small apps from
8+
scratch. Neither found a defect gap. This round asks the other half of Ryan's question: does an agent keep an app
9+
correct when it **changes** one it did not write, where the new feature adds writes that overlap the app's existing
10+
ones?
11+
12+
## Design
13+
14+
- **Bases**: round 3's frontier-model tracker apps, `results/overlap-2026-10-27/runs/big/tracker-<arm>-<n>`. Run n of
15+
each task and arm starts from base n of that arm, for both models. On Solid next @ ac28157a every base passes all
16+
of the tracker's visible, hidden and hidden-overlap tests.
17+
- **Tasks** (`tasks/<task>/brief-plain.md`, the change; `BASE.md`, the tracker's plain brief, which the app must keep
18+
meeting). None names a race.
19+
- rename: rename the selected issue (`renameIssue`), shown in the list and the detail straight away.
20+
- close: close an issue (`closeIssue`); it leaves the list straight away and comes back if declined.
21+
- comments: load the selected issue's comments (`fetchComments`) and post one (`addComment`), shown straight away
22+
with `(sending…)`.
23+
- **Tests**: per task `visible.mjs` (the tracker's 4 plus 2 new), `hidden.mjs` (new tests plus the tracker's hidden
24+
tests as regressions) and `hidden-overlap.mjs` (new overlap tests plus the tracker's overlap tests). rename 16 + 7,
25+
close 15 + 6, comments 17 + 6 hidden and overlap tests.
26+
- **Prompt**: `harness/prompt-change.md`. Each run keeps scratch files in its own `.scratch` folder (round 3's leak).
27+
- **Toolchain**: Solid next @ ac28157a (2.0.0-rc.19 plus the #4329 fix, solidjs/solid#4330) in both arms, through
28+
`eval/package.json`. The .rx compiler and runtime from this branch: MO-1d `SHADOWED_IMPORT` (taken by default
29+
2026-10-27), and `refresh` re-asking the question under a whole-value guess (`ref/compile/rt-web.ts`). The .rx
30+
arm gets `docs/spec` and `docs/guide/async.md` as in round 3, unchanged.
31+
- 3 tasks × 2 arms × 2 models (frontier, Haiku) × 3 runs = 36 fresh runs, 15 `./check` + `./test` each.
32+
33+
Hidden test files, SHA-256 at freezing:
34+
35+
```
36+
869e02eceb679c5907a67571a26f061157eefd86e8e38a1a7aec8db7d9e65fc0 tasks/rename/hidden.mjs
37+
532e2caf837f3effaf77c5979af2651ed4d0ec2973b496d76846035d45831403 tasks/rename/hidden-overlap.mjs
38+
86aebac10d51c9244504e5da3b368c356e6b8c65dfab5d9dbc35b03adb66604d tasks/close/hidden.mjs
39+
669c606395e539e031ca7f376304a42607d15ffccb823b291b5d616f30ddce42 tasks/close/hidden-overlap.mjs
40+
25d241a2309310943ed24d72d26c2a7e638a6feecbd3397b39123ebf5e956dab tasks/comments/hidden.mjs
41+
3d95d4274efea4cd7d793140d2daab3f6b087bcd22c6b4ff7e5a8492d4d361ec tasks/comments/hidden-overlap.mjs
42+
c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e tasks/tracker/hidden.mjs
43+
5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e tasks/tracker/hidden-overlap.mjs
44+
```
45+
46+
**References** (`reference/<task>/<arm>`, not subjects) were written on the bases. Both arms pass every test, except
47+
the .rx comments reference on "a post on one issue does not show on another" (below).
48+
49+
**Tests loosened while writing the references** (before freezing). By H-11 two guesses on one slot from two actions
50+
are one change, so a declined guess can stay on screen until the other action settles. Ryan (2026-10-28): that is the
51+
desired behaviour. Four tests were changed to accept it: close's "a declined close keeps an assignment…", "a declined
52+
assignment keeps a close…", "two closes at once: a declined one does not undo the other", and comments' "two posts at
53+
once: a declined one does not take the other with it". Each still checks the end state after both settle.
54+
55+
## Hypothesis and test
56+
57+
- **H1**: the React arm ships more defects than the .rx arm.
58+
- **Primary measure**: defects shipped: a failed build or any failed visible, hidden or hidden-overlap test at the
59+
agent's "done" (`node harness/score.mjs <dir> --overlap`), except the one test below. One number per arm, pooled
60+
over tasks and models (18 runs each).
61+
- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React
62+
rate is higher and p < 0.05. **H1 reversed** is reported as plainly.
63+
- **Outside the count, reported alongside**: comments' "a post on one issue does not show on another". The .rx
64+
idiom for it (a whole-value guess on the comments source) shows solidjs/solid#4336 (filed 2026-10-28): the guess
65+
stays on screen after the selected issue changes, until the post settles. `score.mjs` runs it and lists it per run
66+
under `failedAlongside`, for both arms, and does not count it.
67+
- **Other runtime bugs** found during the round count as defects, as in rounds 2 and 3, and are listed separately
68+
with the primary result recomputed without them.
69+
- **Reported alongside**: the same table per model and per task; base regressions (tracker tests failed) apart from
70+
new-feature failures; each failure classified by hand; agent tokens and cycles.
71+
- Published as is, whatever the outcome.
72+
73+
## Known limits
74+
75+
- Isolation is still by instruction.
76+
- The bases are frontier-model apps; the small model changes code it would not have written.
77+
- The new tests were written after round 3's results were known, and tuned on the references.
78+
- 18 runs per arm detects a large effect only.

‎eval/PREREG-overlap.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Round 3: overlapping writes
2+
3+
Pre-registration. Everything in this file was fixed and committed before the round's first agent run.
4+
5+
## Why this round
6+
7+
Round 2 (`results/plain-2026-10-27/`) found no defect gap: with briefs that don't name the races, React shipped 0
8+
of 18 defects and .rx 1 of 18. Current models guard a single stale answer in React unprompted. What the agents
9+
reported as unsure, and what no hidden test probed, was **overlapping writes**: two writes in flight at once, and a
10+
write racing a re-load of the data it changed. Exploratory probes on round 2's finished apps (`probes/`, not a
11+
test of anything) showed failures there on both sides. This round tests that question on fresh runs.
12+
13+
## What changes from round 2
14+
15+
- **Hidden tests**: each task gets `tasks/<task>/hidden-overlap.mjs`, run in addition to the unchanged
16+
`hidden.mjs`. Each test checks something the plain brief already states, in a case where two things are in
17+
flight:
18+
- search (1): a retry's answer landing after another selection must not show. The brief says the pane shows the
19+
selected hit's item.
20+
- tracker (2): two issues assigned at once, one declined; and the list loading while an assignment is out. The
21+
brief says a declined row "goes back to the assignee it had before" and that the row shows the new assignee
22+
"straight away" and "keeps" it if the server accepts.
23+
- settings (3): an earlier save failing after a later one succeeded; a later save failing while an earlier one is
24+
out; an earlier save landing while a later one is out. The brief says the saved line shows the values being
25+
saved with `(saving…)`, the saved settings when a save succeeds, and goes back and shows `save-error` when it
26+
fails.
27+
- **The .rx guide** teaches guessing one row of a list (`optimistic xs[i].field := v`, with `indexOf` to find the
28+
row) instead of the whole list. A whole-list guess makes two actions one change (H-11). The example is a generic
29+
todo list, not a task's code. **The prelude** gains `indexOf(xs, f)` (Ryan, 2026-10-27), and the .rx prompt's
30+
prelude list names it.
31+
- Everything else is as in round 2: plain briefs, prompt, visible tests, toolchain, budget, 3 tasks × 2 arms ×
32+
2 models (frontier, Haiku) × 3 runs = 36 fresh runs.
33+
34+
Hidden test files, SHA-256 at freezing:
35+
36+
```
37+
5776f8888055c4f5c13261b8637e048ebbb4f919602afb6f294a78ae4339b4d2 tasks/search/hidden-overlap.mjs
38+
5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e tasks/tracker/hidden-overlap.mjs
39+
aa1d42209d188ea4c54220c816d0ffe718e7ebe65c2d9b9d3fbf5fa23949aade tasks/settings/hidden-overlap.mjs
40+
1d77a1daa81d15d40045d1f4665f9fdcebbe9abb6356db5b3a6eb8b29d30d0ae tasks/search/hidden.mjs
41+
c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e tasks/tracker/hidden.mjs
42+
58d2a1daa4a1b8d2ea559ab56bb37ee9193dcf3d31d03e362107e25ee3b106b6 tasks/settings/hidden.mjs
43+
```
44+
45+
The references are not subjects, and neither passes everything: both fail some overlap tests (`reference/` is
46+
unchanged from the pilot). The tests were checked to run and to fail for a stated reason, not to be passable by the
47+
references.
48+
49+
## Hypothesis and test
50+
51+
- **H1**: the React arm ships more defects than the .rx arm.
52+
- **Primary measure**: defects shipped, where a defect is a failed build or any failed visible, hidden or
53+
hidden-overlap test at the agent's "done" (`node harness/score.mjs <dir> --overlap`). One number per arm, pooled
54+
over tasks and models (18 runs each).
55+
- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React rate
56+
is higher and p < 0.05. **H1 reversed** (the .rx rate higher, p < 0.05) is reported as plainly.
57+
- **Reported alongside**: the same table per model and per task; overlap tests failed per run; each failure
58+
classified by hand (stale answer, lost rollback, wrong revert, error lost, runtime bug, build failure).
59+
- **Runtime bugs**: solidjs/solid#4329 (a per-key guess dropped when a re-ask lands mid-action) is live in rc.19 and
60+
touches the tracker's "list loading while an assignment is out" test for a .rx app that guesses per row. As in
61+
round 2, such a failure counts as a defect in the primary measure (the user gets the bug either way) and is
62+
listed separately with the primary result recomputed without it.
63+
- Published as is, whatever the outcome.
64+
65+
## Known limits
66+
67+
- Isolation is still by instruction.
68+
- The overlap tests were chosen after the exploratory probes had been run on round 2's apps. They run here only on
69+
fresh runs, and four of the six are the probes as they were run.
70+
- 18 runs per arm detects a large effect only (round 2's PREREG gives the arithmetic).

0 commit comments

Comments
 (0)