diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox index 40100e113..63745404d 100644 --- a/Dockerfile.sandbox +++ b/Dockerfile.sandbox @@ -7,18 +7,30 @@ ARG PNPM_VERSION=11.1.3 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + ca-certificates curl gnupg \ + && curl --fail --silent --show-error --location https://apt.llvm.org/llvm-snapshot.gpg.key \ + | gpg --dearmor --output /usr/share/keyrings/llvm-snapshot.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" \ + > /etc/apt/sources.list.d/llvm22.list \ + && apt-get update \ && apt-get install --yes --no-install-recommends \ build-essential \ ca-certificates \ ccache \ - clang \ + clang-22 \ cmake \ curl \ git \ - libclang-rt-dev \ - llvm \ + libclang-rt-22-dev \ + libzstd-dev \ + llvm-22-dev \ + ninja-build \ xz-utils \ zlib1g-dev \ + && ln -sf clang-22 /usr/bin/clang \ + && ln -sf clang++-22 /usr/bin/clang++ \ + && test "$(llvm-config-22 --version)" = 22.1.8 \ && rm -rf /var/lib/apt/lists/* RUN curl --fail --silent --show-error --location \ diff --git a/packages/cli/test/bootstrap.test.ts b/packages/cli/test/bootstrap.test.ts index 420f4761c..e30c02b64 100644 --- a/packages/cli/test/bootstrap.test.ts +++ b/packages/cli/test/bootstrap.test.ts @@ -4,12 +4,16 @@ import { release as osRelease, tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; import { expect, test } from "vitest"; +import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js"; +import { platformLinkerSupportsPersistentCache } from "../../compiler/src/backend/linker.js"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../../.."); const bootstrap = join(repoRoot, "packages/cli/dist/bootstrap.js"); const runtimePackHost = process.platform === "darwin" && process.arch === "arm64" && Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24; +const helperTarget = precompiledRuntimePackTarget(); +const persistentExecutable = helperTarget === null || platformLinkerSupportsPersistentCache(process.env, helperTarget); test("bootstrap serves version and help without loading the compiler graph", async () => { const preloadDir = await mkdtemp(join(tmpdir(), "scriptc-bootstrap-preload-")); @@ -83,7 +87,15 @@ test("bootstrap exact builds use the routed cache and source edits fall through" rm(join(cacheRoot, "early-exe-implementation"), { recursive: true, force: true }), ]); expect((await build()).stderr).not.toContain("scriptc lowering"); - await expect(build(true)).resolves.toMatchObject({ stderr: "" }); + if (persistentExecutable) { + await expect(build(true)).resolves.toMatchObject({ stderr: "" }); + } else { + // This target intentionally lacks a persistent native dependency proof. + // It must enter the full compiler to relink, even on a frontend hit. + await expect(build(true)).rejects.toMatchObject({ stderr: expect.stringContaining("compiler graph loaded") }); + expect((await build()).stderr).not.toContain("scriptc lowering"); + expect((await execFileAsync(outPath)).stdout).toBe("one\n"); + } await writeFile(entry, 'console.log("two");\n'); expect((await build()).stderr).toContain("scriptc lowering"); diff --git a/packages/cli/test/executable-cache.test.ts b/packages/cli/test/executable-cache.test.ts index be3113d8a..0dd75f3fb 100644 --- a/packages/cli/test/executable-cache.test.ts +++ b/packages/cli/test/executable-cache.test.ts @@ -5,6 +5,8 @@ import { release as osRelease, tmpdir } from "node:os"; import { delimiter, dirname, join } from "node:path"; import { promisify } from "node:util"; import { expect, test } from "vitest"; +import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js"; +import { platformLinkerSupportsPersistentCache } from "../../compiler/src/backend/linker.js"; const execFileAsync = promisify(execFile); const require = createRequire(import.meta.url); @@ -13,6 +15,8 @@ const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); const runtimePackHost = process.platform === "darwin" && process.arch === "arm64" && Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24; +const helperTarget = precompiledRuntimePackTarget(); +const persistentExecutable = helperTarget === null || platformLinkerSupportsPersistentCache(process.env, helperTarget); test("exact executable repeats skip lowering while edits and damaged outputs stay correct", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-cli-executable-cache-")); @@ -50,10 +54,12 @@ test("exact executable repeats skip lowering while edits and damaged outputs sta const originalBinaryTime = (await stat(outPath)).mtimeMs; const originalTuTime = (await stat(tuPath)).mtimeMs; - // The exact repeat returns from the whole-program cache before tsgo and - // leaves already-correct caller outputs on their existing inodes. + // Every route reuses the frontend. Only routes with a complete native + // dependency proof can also retain the executable inode; other targets + // deliberately relink their cached translation unit. expect((await build()).stderr).not.toContain("scriptc lowering"); - expect((await stat(outPath)).mtimeMs).toBe(originalBinaryTime); + if (persistentExecutable) expect((await stat(outPath)).mtimeMs).toBe(originalBinaryTime); + expect(await run()).toBe("one\n"); expect((await stat(tuPath)).mtimeMs).toBe(originalTuTime); // Output-kind cleanup is honest even on an exact early hit: an IR file diff --git a/packages/cli/test/native-output.test.ts b/packages/cli/test/native-output.test.ts index e4c3c1fce..943d09f1e 100644 --- a/packages/cli/test/native-output.test.ts +++ b/packages/cli/test/native-output.test.ts @@ -35,11 +35,13 @@ function cli(args: string[], env: NodeJS.ProcessEnv = process.env) { }); } -test("unsupported hosts fail with SC3002 before creating an artifact", async () => { - if (supported) return; +test("unsupported targets fail with SC3002 before creating an artifact", async () => { const { dir, entry } = await fixture(); const output = join(dir, "hello.o"); - await expect(cli(["build", entry, "--emit=obj", "-o", output])).rejects.toMatchObject({ + await expect(cli(["build", entry, "--emit=obj", "-o", output], { + ...process.env, + SCRIPTC_TARGET: "scriptc-unsupported-target", + })).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining("SC3002"), }); diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index 5c092d0c3..4c842fa92 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -71,6 +71,7 @@ function request(root: string, packageJson: string, output = join(root, "program outputKind: "obj" as const, sourcePath: "/source/app.ts", target, + helperHost: { platform: "darwin" as const, arch: "arm64" }, resolvePackageJson: () => packageJson, cacheRoot: join(root, "cache"), }; diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index 2c57839ac..ce68d5f35 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -4178,7 +4178,11 @@ test.skipIf( probeSource, "long long public_value(void);\nint main(void) { return public_value() != 7; }\n", ); - execFileSync(clangExecutable!, [probeSource, outPath, "-lm", "-o", probe]); + // The failed linker belongs only to the shard-merge probe. Link the + // resulting archive with the real host tools to verify its fallback. + execFileSync(clangExecutable!, [probeSource, outPath, "-lm", "-o", probe], { + env: { ...process.env, PATH: oldPath }, + }); expect(execFileSync(probe, { encoding: "utf8" })).toBe(""); // The canonical retry is valid for this invocation, but must not occupy a // key describing merged shard bytes. A repaired merge tool retries the diff --git a/scripts/sandbox-bootstrap.sh b/scripts/sandbox-bootstrap.sh index 07aa5e5bc..fe8186ac5 100644 --- a/scripts/sandbox-bootstrap.sh +++ b/scripts/sandbox-bootstrap.sh @@ -9,30 +9,35 @@ if [ -z "$node_version" ] || [ -z "$pnpm_version" ]; then fi # The managed image intentionally floats its distro and language runtimes. -# Install the same LLVM major and exact Node/pnpm tuple as the custom image so +# Install the same LLVM and exact Node/pnpm tuple as the custom image so # differential oracles do not change when the managed image is refreshed. +. /etc/os-release +llvm_distro=${VERSION_CODENAME:?the Sandbox image must identify its Ubuntu release} curl --fail --silent --show-error --location \ https://apt.llvm.org/llvm-snapshot.gpg.key \ --output /tmp/llvm-snapshot.gpg.key gpg --dearmor --yes --output /tmp/llvm-snapshot.gpg /tmp/llvm-snapshot.gpg.key sudo install -D -m 0644 /tmp/llvm-snapshot.gpg /etc/apt/keyrings/llvm-snapshot.gpg -echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \ - | sudo tee /etc/apt/sources.list.d/llvm18.list >/dev/null +echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/${llvm_distro}/ llvm-toolchain-${llvm_distro}-22 main" \ + | sudo tee /etc/apt/sources.list.d/llvm22.list >/dev/null sudo apt-get update --quiet=2 sudo env DEBIAN_FRONTEND=noninteractive apt-get install --quiet=2 --yes --no-install-recommends \ build-essential \ ca-certificates \ ccache \ - clang-18 \ + clang-22 \ cmake \ - libclang-rt-18-dev \ - llvm-18 \ + libclang-rt-22-dev \ + libzstd-dev \ + llvm-22-dev \ + ninja-build \ xz-utils \ zlib1g-dev sudo rm -f /usr/local/bin/clang /usr/local/bin/clang++ -sudo ln -sf clang-18 /usr/bin/clang -sudo ln -sf clang++-18 /usr/bin/clang++ +sudo ln -sf clang-22 /usr/bin/clang +sudo ln -sf clang++-22 /usr/bin/clang++ +test "$(llvm-config-22 --version)" = 22.1.8 node_archive="node-v${node_version}-linux-x64.tar.xz" curl --fail --silent --show-error --location \ diff --git a/scripts/sandbox-command.mjs b/scripts/sandbox-command.mjs new file mode 100644 index 000000000..9d00e592f --- /dev/null +++ b/scripts/sandbox-command.mjs @@ -0,0 +1,22 @@ +export const MAX_INLINE_SANDBOX_COMMAND_BYTES = 768; + +export const shellQuote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; + +/** Keep long shell programs out of the local CLI's argument vector. The + * same script records the remote exit status for either transport. */ +export function sandboxCommand(command, args, exitMarker) { + const statusPath = `/tmp/${exitMarker}.status`; + const script = + `${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` + + `printf '%s\\n' "$scriptc_status" > ${shellQuote(statusPath)}; ` + + `printf '\\n${exitMarker}%s\\n' "$scriptc_status"`; + const scriptPath = `/tmp/${exitMarker}.sh`; + const file = Buffer.byteLength(script, "utf8") > MAX_INLINE_SANDBOX_COMMAND_BYTES; + return { + script, + scriptPath, + statusPath, + file, + argv: file ? ["sh", scriptPath] : ["sh", "-c", script], + }; +} diff --git a/scripts/sandbox-test.mjs b/scripts/sandbox-test.mjs index 5b79432a2..d7b624265 100644 --- a/scripts/sandbox-test.mjs +++ b/scripts/sandbox-test.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { spawn, spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pipeline } from "node:stream/promises"; @@ -23,6 +23,7 @@ import { filterExistingWorktreePaths, workspaceResetCommand, } from "./worktree-files.mjs"; +import { sandboxCommand, shellQuote } from "./sandbox-command.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const laneCaseShardedFiles = [ @@ -371,7 +372,6 @@ const vercel = ([group, command, ...args], options) => }); // `vercel sandbox exec` does not propagate the remote process's exit code. // Print a per-command nonce after it finishes and enforce that status here. -const shellQuote = (value) => `'${value.replaceAll("'", `'\"'\"'`)}'`; const execIn = async ( worker, command, @@ -384,12 +384,22 @@ const execIn = async ( ) => { const envArgs = Object.entries(env).flatMap(([key, value]) => ["--env", `${key}=${value}`]); const exitMarker = `__SCRIPTC_REMOTE_EXIT_${randomBytes(12).toString("hex")}__`; - const statusPath = `/tmp/${exitMarker}.status`; - const script = - `${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` + - `printf '%s\\n' "$scriptc_status" > ${shellQuote(statusPath)}; ` + - `printf '\\n${exitMarker}%s\\n' "$scriptc_status"`; + const prepared = sandboxCommand(command, args, exitMarker); + const { statusPath } = prepared; const label = task ? `${worker.label} ${task}` : worker.label; + if (prepared.file) { + const localScript = join(temp, `${exitMarker}.sh`); + await writeFile(localScript, prepared.script, { mode: 0o600 }); + try { + await vercel(["sandbox", "copy", localScript, `${worker.name}:${prepared.scriptPath}`], { + idleTimeoutMs: 60_000, + label: `${label} command`, + timeoutMs: 2 * 60_000, + }); + } finally { + await rm(localScript, { force: true }); + } + } const commandArgs = [ "sandbox", "exec", @@ -399,9 +409,7 @@ const execIn = async ( workdir, ...envArgs, worker.name, - "sh", - "-c", - script, + ...prepared.argv, ]; try { await vercel(commandArgs, { @@ -411,7 +419,7 @@ const execIn = async ( timeoutMs: wallTimeoutMs, }); } catch (error) { - console.warn(`[${label}] CLI completion was not confirmed; checking the remote command status...`); + console.warn(`[${label}] CLI completion was not confirmed (${error.message}); checking the remote command status...`); const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`; const probeScript = `scriptc_status=125; test ! -f ${shellQuote(statusPath)} || ` + @@ -714,6 +722,10 @@ try { } await execIn(worker, "pnpm", ["install", "--frozen-lockfile"], {}, "", 2 * 60_000); await execIn(worker, "pnpm", ["build"], {}, "", 2 * 60_000); + // Workspace builds deliberately do not rebuild packaged native artifacts. + // Every remote lane needs the Linux helper and runtime from this worktree. + await execIn(worker, "pnpm", ["--filter", "@scriptc/llvm-linux-x64-gnu", "build:native"], {}, "LLVM helper", 5 * 60_000); + await execIn(worker, "pnpm", ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"], { CC: "clang-22", AR: "llvm-ar-22" }, "runtime pack", 5 * 60_000); }, imageConfig.custom ? workers.length : 8); await allWorkers("Testing", async (worker) => { diff --git a/tests/harness/sandbox-command.test.ts b/tests/harness/sandbox-command.test.ts new file mode 100644 index 000000000..6003fe09e --- /dev/null +++ b/tests/harness/sandbox-command.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { sandboxCommand } from "../../scripts/sandbox-command.mjs"; + +const roots: string[] = []; +const statuses: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); + await Promise.all(statuses.splice(0).map((path) => rm(path, { force: true }))); +}); + +test("short commands stay inline and retain the remote exit contract", async () => { + const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_SHORT__`; + const prepared = sandboxCommand("sh", ["-c", "exit 7"], marker); + statuses.push(prepared.statusPath); + + expect(prepared.file).toBe(false); + const output = execFileSync(prepared.argv[0], prepared.argv.slice(1), { encoding: "utf8" }); + expect(output).toBe(`\n${marker}7\n`); + expect(await readFile(prepared.statusPath, "utf8")).toBe("7\n"); +}); + +test("uploaded scripts preserve long and shell-sensitive argument bytes", async () => { + const root = await mkdtemp("/tmp/scriptc-sandbox-command-test-"); + roots.push(root); + const sentinel = join(root, "unexpected"); + const args = [ + "x".repeat(2048), + "quote' and spaces", + `$(touch ${sentinel})`, + "backtick` dollar$ slash\\ newline\nend", + "é".repeat(512), + ]; + const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_LONG__`; + const prepared = sandboxCommand("printf", ["<%s>\n", ...args], marker); + statuses.push(prepared.statusPath); + expect(prepared.file).toBe(true); + expect(prepared.argv).toEqual(["sh", prepared.scriptPath]); + expect(prepared.argv.every((arg) => arg.length < 128)).toBe(true); + + const localScript = join(root, "command.sh"); + await writeFile(localScript, prepared.script); + const output = execFileSync("sh", [localScript], { encoding: "utf8" }); + expect(output).toBe(args.map((arg) => `<${arg}>\n`).join("") + `\n${marker}0\n`); + await expect(readFile(sentinel)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(prepared.statusPath, "utf8")).toBe("0\n"); +}); diff --git a/tests/harness/sandbox-config.test.ts b/tests/harness/sandbox-config.test.ts index e26ea4b00..71f0b4a7c 100644 --- a/tests/harness/sandbox-config.test.ts +++ b/tests/harness/sandbox-config.test.ts @@ -66,11 +66,12 @@ test("the managed-image bootstrap pins the custom image's Node, pnpm, and LLVM", const bootstrap = readFileSync(new URL("../../scripts/sandbox-bootstrap.sh", import.meta.url), "utf8"); expect(bootstrap).toContain("< .node-version"); expect(bootstrap).toContain("ARG PNPM_VERSION="); - expect(bootstrap).toContain("llvm-toolchain-noble-18"); + expect(bootstrap).toContain("llvm-toolchain-${llvm_distro}-22"); expect(bootstrap).toContain("install -D -m 0644"); - expect(bootstrap).toContain("clang-18"); - expect(bootstrap).toContain("libclang-rt-18-dev"); - expect(bootstrap).toContain("llvm-18"); + expect(bootstrap).toContain("clang-22"); + expect(bootstrap).toContain("libclang-rt-22-dev"); + expect(bootstrap).toContain("llvm-22-dev"); + expect(bootstrap).toContain("ninja-build"); }); test("a custom image does not select the Sandbox team or project", () => {