Skip to content

Commit 7ed2467

Browse files
authored
Merge pull request #59 from solidjs/claude/project-thread-xp2irv
eval: desk13 Opus probe on the new guide; limits on module data
2 parents 7e5bc81 + cd9746b commit 7ed2467

24 files changed

Lines changed: 903 additions & 14 deletions

‎eval/harness/build.mjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@
66
// vanilla arm: `src/main.ts` default-exports `mount(root: HTMLElement)`, called once with the `#app` container
77
// foldkit arm: `src/main.ts` exports Model, init, update and view; booted with Runtime.makeApplication and
88
// Runtime.run, as Foldkit's own `src/entry.ts` does
9+
// contract arm: an Exact app, built by its own `exact.mjs web-build` (harness/contract.mjs)
910
// The app's API is `src/api.js` over `src/flights.js`: every call is a flight the test lands or fails.
1011
// Usage: node harness/build.mjs <dir> [rx|react] (the arm defaults to what `<dir>/arm` says)
1112
import { execFileSync } from "node:child_process";
1213
import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
1314
import path from "node:path";
1415
import { fileURLToPath, pathToFileURL } from "node:url";
16+
import { buildContract } from "./contract.mjs";
1517

1618
const here = path.dirname(fileURLToPath(import.meta.url));
1719
const evalDir = path.resolve(here, "..");
@@ -35,6 +37,7 @@ async function solidPlugin() {
3537

3638
export async function build(dir, arm = armOf(dir)) {
3739
dir = realpathSync(path.resolve(dir));
40+
if (arm === "contract") return buildContract(dir);
3841
const work = path.join(dir, ".build");
3942
rmSync(work, { recursive: true, force: true }); mkdirSync(work, { recursive: true });
4043
// the app's bare imports resolve from the harness's packages

‎eval/harness/check.mjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
// solid: tsc --noEmit -p . and eslint (eslint-plugin-solid's v2 config)
55
// vanilla: tsc --noEmit -p . (strict)
66
// foldkit: tsc --noEmit -p . with Foldkit's template tsconfig (its lint, oxlint + effect-tsgo, is not on npm)
7+
// contract: `exact.mjs contract build app.contract`, which also type-checks app.ts
78
// Usage: node harness/check.mjs <dir>
89
import { execFileSync } from "node:child_process";
910
import { existsSync, symlinkSync } from "node:fs";
1011
import path from "node:path";
1112
import { fileURLToPath, pathToFileURL } from "node:url";
1213
import { RXC, armOf } from "./build.mjs";
14+
import { checkContract } from "./contract.mjs";
1315

1416
const here = path.dirname(fileURLToPath(import.meta.url));
1517
const evalDir = path.resolve(here, "..");
@@ -21,6 +23,7 @@ const run = (cmd, args, cwd) => {
2123

2224
export function check(dir) {
2325
dir = path.resolve(dir);
26+
if (armOf(dir) === "contract") return checkContract(dir);
2427
if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules"));
2528
if (armOf(dir) === "rx") return run(RXC, ["check", "src/main.rx"], dir);
2629
const bin = n => path.join(evalDir, "node_modules/.bin", n);

‎eval/harness/contract.mjs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// The contract arm: an Exact app (https://github.com/expo/exact), `app.contract` for the interface and `app.ts` for its
2+
// data module, which imports the task's `src/api.js`. A desk is the directory `exact new` makes, with the eval's files
3+
// beside it. EXACT2 names the Exact checkout the desk is made from; the desk's own `exact.mjs` keeps that path.
4+
//
5+
// The build is Exact's web JS target, a release build (`exact.mjs web-build --out`). The data module runs in the page's
6+
// realm, so `src/flights.js` puts `__flights` on the page as it does in every other arm, and the driver is unchanged.
7+
// BUN names the Bun to run `exact.mjs` with (default: `bun` on PATH).
8+
import { execFileSync, spawnSync } from "node:child_process";
9+
import { cpSync, existsSync, readFileSync, rmSync } from "node:fs";
10+
import path from "node:path";
11+
12+
const bun = () => process.env.BUN ?? "bun";
13+
const exactEnv = () => ({ ...process.env, EXACT_FEEDBACK: "0", NO_COLOR: "1" });
14+
15+
export function exact2() {
16+
const dir = process.env.EXACT2;
17+
if (!dir || !existsSync(path.join(dir, "scripts/exact.mjs"))) throw new Error("EXACT2 must name an Exact checkout (its scripts/exact.mjs)");
18+
return path.resolve(dir);
19+
}
20+
21+
/** `<dir>/dist`, built by the desk's `exact.mjs` (Exact keeps a build inside an app in `dist`, which its source scans
22+
* skip); a compile error is the `compile` stage's output */
23+
export function buildContract(dir) {
24+
const dist = path.join(dir, "dist");
25+
const r = spawnSync(bun(), ["exact.mjs", "web-build", "--out", dist], { cwd: dir, env: exactEnv(), encoding: "utf8", maxBuffer: 64 << 20 });
26+
if (r.status !== 0) return { ok: false, stage: "compile", output: `${r.stdout ?? ""}${r.stderr ?? ""}`.trim() };
27+
return { ok: true, dist };
28+
}
29+
30+
/** the compiler's diagnostics, warnings included, as the agent's `./check` shows them */
31+
export function checkContract(dir) {
32+
const r = spawnSync(bun(), ["exact.mjs", "contract", "build", "app.contract"], { cwd: dir, env: exactEnv(), encoding: "utf8", maxBuffer: 64 << 20 });
33+
return { ok: r.status === 0, out: `${r.stdout ?? ""}${r.stderr ?? ""}`.trim() || "ok" };
34+
}
35+
36+
/** `exact new <dir>`, then the scaffold's own files go in beside it. The placeholder test is removed: `./test` is the
37+
* desk's test command, as in every arm. */
38+
export function newContractDesk(dir) {
39+
if (existsSync(dir)) throw new Error(`${dir} exists; a contract desk starts from \`exact new\``);
40+
execFileSync(bun(), [path.join(exact2(), "scripts/exact.mjs"), "new", dir], { env: exactEnv(), stdio: "pipe" });
41+
rmSync(path.join(dir, "app.test.contract"), { force: true });
42+
}
43+
44+
/** Exact's one required guide, for the prompt's inline docs */
45+
export const startHere = () => readFileSync(path.join(exact2(), "docs/start-here.md"), "utf8");
46+
47+
/** copies a reference app's files over a desk's placeholders */
48+
export function copyContractReference(from, dir) {
49+
for (const f of ["app.contract", "app.ts"]) cpSync(path.join(from, f), path.join(dir, f));
50+
}

‎eval/harness/driver.mjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,13 @@ export async function serve(dist) {
2323
}
2424

2525
/** CHROMIUM=/path/to/chromium drives a system browser (NixOS and the like, where Playwright's download does not run). */
26-
export async function launch() { return chromium.launch(process.env.CHROMIUM ? { executablePath: process.env.CHROMIUM } : {}); }
26+
// Without CHROMIUM, a Playwright whose own browser is missing falls back to a preinstalled one (/opt/pw-browsers/chromium,
27+
// as in cloud containers), so an agent's ./test runs there too.
28+
export async function launch() {
29+
if (process.env.CHROMIUM) return chromium.launch({ executablePath: process.env.CHROMIUM });
30+
try { return await chromium.launch(); }
31+
catch (e) { const alt = "/opt/pw-browsers/chromium"; if (existsSync(alt)) return chromium.launch({ executablePath: alt }); throw e; }
32+
}
2733

2834
/** One test: a fresh page on the app, `run(app)` with the helpers below. Returns { ok, error }. */
2935
export async function runTest(browser, url, test) {

‎eval/harness/prompt-arms.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,5 +58,17 @@
5858
"LANG_RULE": "plain TypeScript and the browser DOM, with no UI framework or library: `src/main.ts` must default-export `function mount(root: HTMLElement): void`, which the harness calls once with the page's `#app` element. You may add files under `src/`. No packages are installed for you to use.",
5959
"API_TYPES": "`src/api.d.ts`",
6060
"CHECKER": "the TypeScript compiler (strict)"
61+
},
62+
"contract": {
63+
"DOCS_RULE": ", except that you may read Exact's documentation in {{EXACT2}}/docs and look names up with `bun exact.mjs contract vocab <name>` and `bun exact.mjs docs <heading words>`",
64+
"LANG_RULE": "Exact, with Contract for the interface and TypeScript for its data module. This directory is an Exact app that `exact new` made from the Exact checkout at {{EXACT2}}; the tests run its web build in a browser. Exact's guide is `{{EXACT2}}/docs/start-here.md`, the only required reading; the rest of `{{EXACT2}}/docs` is lookup. `app.contract` is the interface and `app.ts` its data module, whose sources call the backend: import it with `import * as api from './src/api.js'`. Give an element a test id with `testId=\"…\"`. The only commands that build, test, drive or serve the app are `./check` and `./test` below: do not run the `bun exact.mjs` verbs that do (`web`, `web-build`, `serve`, `test`, `verify`, `agent`, or a platform's).",
65+
"API_TYPES": "`src/api.d.ts`",
66+
"CHECKER": "the Contract compiler (`bun exact.mjs contract build app.contract`), which also type-checks `app.ts`"
67+
},
68+
"contract-inline": {
69+
"DOCS_RULE": ", except that you may read Exact's documentation in {{EXACT2}}/docs and look names up with `bun exact.mjs contract vocab <name>` and `bun exact.mjs docs <heading words>`",
70+
"LANG_RULE": "Exact, with Contract for the interface and TypeScript for its data module. This directory is an Exact app that `exact new` made from the Exact checkout at {{EXACT2}}; the tests run its web build in a browser. Exact's guide, `docs/start-here.md`, is at the end of this message: it is the only required reading, and the rest of `{{EXACT2}}/docs` is lookup. `app.contract` is the interface and `app.ts` its data module, whose sources call the backend: import it with `import * as api from './src/api.js'`. Give an element a test id with `testId=\"…\"`. The only commands that build, test, drive or serve the app are `./check` and `./test` below: do not run the `bun exact.mjs` verbs that do (`web`, `web-build`, `serve`, `test`, `verify`, `agent`, or a platform's).",
71+
"API_TYPES": "`src/api.d.ts`",
72+
"CHECKER": "the Contract compiler (`bun exact.mjs contract build app.contract`), which also type-checks `app.ts`"
6173
}
6274
}

‎eval/harness/scaffold.mjs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, symlinkSync, writeFileSync } from "node:fs";
1919
import path from "node:path";
2020
import { fileURLToPath, pathToFileURL } from "node:url";
21+
import { copyContractReference, exact2, newContractDesk, startHere } from "./contract.mjs";
2122

2223
const here = path.dirname(fileURLToPath(import.meta.url));
2324
const evalDir = path.resolve(here, "..");
@@ -30,6 +31,7 @@ const statelyDocs = process.env.STATELY_DOCS ?? path.resolve(evalDir, "../../sta
3031
export function scaffold(task, arm, dir, { reference = false, brief = null, base = null, inline = false } = {}) {
3132
const t = path.join(evalDir, "tasks", task);
3233
dir = path.resolve(dir);
34+
if (arm === "contract") newContractDesk(dir);
3335
mkdirSync(path.join(dir, "src"), { recursive: true }); mkdirSync(path.join(dir, "tests"), { recursive: true });
3436
writeFileSync(path.join(dir, "arm"), `${arm}\n`);
3537
cpSync(path.join(t, brief ? `brief-${brief}.md` : "brief.md"), path.join(dir, "BRIEF.md"));
@@ -69,7 +71,8 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base
6971
}
7072
// Solid's own docs ship in its package: the agent can read them there before its first build
7173
if (arm === "solid" && !existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules"));
72-
if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true });
74+
if (reference && arm === "contract") copyContractReference(path.join(evalDir, "reference", task, arm), dir);
75+
else if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true });
7376
if (base) {
7477
cpSync(path.join(t, "BASE.md"), path.join(dir, "BASE.md"));
7578
// everything the earlier app had (a chain step's base is the previous agent's src/, which may hold more than
@@ -85,11 +88,13 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base
8588
export function prompt(arm, dir, { plain = false, change = false, inline = false } = {}) {
8689
const docs = inline && arm !== "react";
8790
const fill = { ...JSON.parse(readFileSync(path.join(here, "prompt-arms.json"), "utf8"))[docs ? `${arm}-inline` : arm] };
91+
if (arm === "contract") fill.EXACT2 = exact2();
8892
fill.BACKEND_NOTE = plain ? "" : " whose answers the tests control";
8993
let text = readFileSync(path.join(here, change ? "prompt-change.md" : "prompt.md"), "utf8");
9094
for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]);
9195
if (docs && arm === "rx") for (const f of ["language.md", "async.md", "reference.md"]) text += `\n---\n\n${readFileSync(path.join(evalDir, "../guide", f), "utf8")}`;
9296
if (docs && arm === "foldkit") text += `\n---\n\n${readFileSync(path.join(foldkitRepo, "packages/create-foldkit-app/templates/base/FOLDKIT.md"), "utf8")}`;
97+
if (docs && arm === "contract") text += `\n---\n\n${startHere()}`;
9398
if (docs && arm === "xstate") text += `\n---\n\n${readFileSync(path.join(statelyDocs, "content/docs/cheatsheet.mdx"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}\n---\n\n${readFileSync(path.join(evalDir, "node_modules/@xstate/react/README.md"), "utf8")}`;
9499
if (docs && arm === "solid") text += `\n---\n\n${readFileSync(path.join(evalDir, "node_modules/solid-js/skills/solid-2.0/SKILL.md"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}`;
95100
return text;

‎eval/harness/score.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ const round5 = new Set(["desk", "labels", "related", "count", "chain1", "chain2"
2424
const rows = [];
2525
for (const name of readdirSync(runs).sort()) {
2626
const dir = path.join(runs, name);
27-
const m = name.match(/^(\w+)-(rx|react|solid|foldkit|vanilla)-(\d+)$/);
27+
const m = name.match(/^(\w+)-(rx|react|solid|foldkit|vanilla|contract)-(\d+)$/);
2828
if (!m) continue;
2929
const [, task, arm, n] = m;
3030
const alongside = round5.has(task) ? round5Alongside : round4Alongside;
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# desk9 in Contract (Exact)
2+
3+
Exact team's hand-written reference, from expo/exact `bench/reactivescript/harness.patch` (main 8c742e6a, 2026-11-13).
4+
MIT License, Copyright (c) 2026 650 Industries, Inc. (Expo).
5+
6+
The `contract` arm (`harness/contract.mjs` and its hooks in build, check, scaffold, prompt-arms and score) comes from
7+
the same patch, ported to this harness. It needs `EXACT2` (an Exact checkout, `bun install` and `cargo fetch` run in
8+
it; the web build compiles Exact's Rust workspace once) and Bun on PATH.
9+
10+
Graded here on 2026-11-13 (`harness/grade.sh`): 77/77 hidden, 38/38 overlap, 0/20 fuzz, 0 defects, as in Exact's own
11+
harness.

0 commit comments

Comments
 (0)