Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion .mise/tasks/extension/artifact/elm-native
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,51 @@ require_snapshot_inputs() {
fi
}

# A guest must not record where it was built. Without these flags it embeds the builder's Cargo home
# and, when the toolchain has rust-src, the path of the toolchain. The second flag gives the path that
# rustc records when rust-src is absent, so that component makes no difference. The flags apply only to
# the guest build: CARGO_ENCODED_RUSTFLAGS keeps a path with spaces whole, and with --target it does not
# reach build scripts and proc macros. Cargo still mixes the build machine's host triple into crate hashes
# through those units, so builds on different host triples give different bytes. See "Build paths and
# digests" in docs/extensions/capability-claims.md.
cargo_home() {
home="${CARGO_HOME:-$HOME/.cargo}"
printf '%s' "${home%/}"
}

guest_path_flags() {
sysroot="$(rustc --print sysroot)"
commit="$(rustc -vV | sed -n 's/^commit-hash: //p')"
if [ -z "$sysroot" ] || [ -z "$commit" ]; then
echo "cannot read the Rust sysroot and commit hash" >&2
return 1
fi
printf '%s\037%s' \
"--remap-path-prefix=$(cargo_home)=/cargo" \
"--remap-path-prefix=$sysroot/lib/rustlib/src/rust=/rustc/$commit"
}

refuse_machine_paths() {
for machine_path in "$(cargo_home)/" "$(rustc --print sysroot)/lib/rustlib/"; do
# A Cargo home at /cargo is its own canonical path: every remapped path starts with it.
[ "$machine_path" = "/cargo/" ] && continue
if LC_ALL=C grep -aqF "$machine_path" "$1"; then
echo "$1 embeds a build machine path: $machine_path" >&2
return 1
fi
done
}

run_pipeline() {
source_root="$1"
output="$2"
git_commit="$3"
(
cd "$source_root"
cargo test --locked -p morphir-elm-binding
cargo build --locked --release -p morphir-elm-binding --target wasm32-unknown-unknown
path_flags="$(guest_path_flags)"
CARGO_ENCODED_RUSTFLAGS="$path_flags" cargo build --locked --release -p morphir-elm-binding --target wasm32-unknown-unknown
refuse_machine_paths "$WASM"
wasm-tools validate "$WASM"
MORPHIR_ELM_GUEST="$WASM" cargo test --locked -p morphir-elm-binding --features wasm-host-tests --test wasm -- --ignored
set -- scripts/package_extension.py \
Expand Down
39 changes: 38 additions & 1 deletion .mise/tasks/extension/artifact/rust
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,51 @@ require_snapshot_inputs() {
fi
}

# A guest must not record where it was built. Without these flags it embeds the builder's Cargo home
# and, when the toolchain has rust-src, the path of the toolchain. The second flag gives the path that
# rustc records when rust-src is absent, so that component makes no difference. The flags apply only to
# the guest build: CARGO_ENCODED_RUSTFLAGS keeps a path with spaces whole, and with --target it does not
# reach build scripts and proc macros. Cargo still mixes the build machine's host triple into crate hashes
# through those units, so builds on different host triples give different bytes. See "Build paths and
# digests" in docs/extensions/capability-claims.md.
cargo_home() {
home="${CARGO_HOME:-$HOME/.cargo}"
printf '%s' "${home%/}"
}

guest_path_flags() {
sysroot="$(rustc --print sysroot)"
commit="$(rustc -vV | sed -n 's/^commit-hash: //p')"
if [ -z "$sysroot" ] || [ -z "$commit" ]; then
echo "cannot read the Rust sysroot and commit hash" >&2
return 1
fi
printf '%s\037%s' \
"--remap-path-prefix=$(cargo_home)=/cargo" \
"--remap-path-prefix=$sysroot/lib/rustlib/src/rust=/rustc/$commit"
}

refuse_machine_paths() {
for machine_path in "$(cargo_home)/" "$(rustc --print sysroot)/lib/rustlib/"; do
# A Cargo home at /cargo is its own canonical path: every remapped path starts with it.
[ "$machine_path" = "/cargo/" ] && continue
if LC_ALL=C grep -aqF "$machine_path" "$1"; then
echo "$1 embeds a build machine path: $machine_path" >&2
return 1
fi
done
}

run_pipeline() {
source_root="$1"
output="$2"
git_commit="$3"
(
cd "$source_root"
cargo test --locked -p morphir-rust-binding
cargo build --locked --release -p morphir-rust-binding --target wasm32-unknown-unknown
path_flags="$(guest_path_flags)"
CARGO_ENCODED_RUSTFLAGS="$path_flags" cargo build --locked --release -p morphir-rust-binding --target wasm32-unknown-unknown
refuse_machine_paths "$WASM"
wasm-tools validate "$WASM"
MORPHIR_RUST_GUEST="$WASM" cargo test --locked -p morphir-rust-binding --features wasm-host-tests --test wasm -- --ignored
set -- scripts/package_extension.py \
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
and `ProcessChild::read_within`, or a `ProcessChannel`.

### Fixed
- Extension WASM guests no longer record the build machine's Cargo home or
toolchain directory. `extension:artifact:rust` and `extension:artifact:elm-native`
remap those paths and fail if a guest still contains them. Builds of one commit
on one host triple now give the same bytes; builds on different host triples
still differ, because Cargo hashes the host triple into crate metadata.
- Concurrent first cache-maintenance saves and mutation leases share one
coordination file without macOS `NotFound` failures. Atomic creation and
existing-file opens retain directory confinement and no-follow checks.
Expand Down
18 changes: 18 additions & 0 deletions docs/extensions/capability-claims.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,24 @@ The claimed extension version must match the registered crate's `Cargo.toml`.
`.github/extensions.toml` remains the declared source of release capabilities;
packaging never replaces a guest claim with a registry value.

### Build paths and digests

The `rust` and `elm-native` tasks build the guest with `--remap-path-prefix`, so
it does not record the builder's Cargo home or toolchain directory. Paths under
the Cargo home become `/cargo/...`, and standard library paths become
`/rustc/<commit>/...`, with or without the `rust-src` component. The task fails
if the guest still contains either directory.

One commit and toolchain give the same WASM on machines with the same host
triple. Machines with different host triples give different bytes: Cargo hashes
the `host:` line of `rustc -vV` for build scripts and proc macros, and that hash
reaches every crate that depends on them, which changes the guest's mangled
symbols. So a run of the task on macOS checks the release steps, but its WASM
digest differs from the release job's, which runs on
`x86_64-unknown-linux-gnu`. Run the task on that host triple to predict the
published digest. `release.json` records the commit and the WASM digest, so its
own digest follows them. Qualify the published assets, not a local build.

A bundle descriptor has this structure. The digest below is illustrative:

```json
Expand Down
17 changes: 17 additions & 0 deletions tests/ci/package_extension_test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,24 @@ def __init__(self, root: Path) -> None:
'build --locked --release -p morphir-rust-binding --target wasm32-unknown-unknown')
mkdir -p "$CARGO_TARGET_DIR/wasm32-unknown-unknown/release"
printf '\\000asm\\001\\000\\000\\000' > "$CARGO_TARGET_DIR/wasm32-unknown-unknown/release/morphir_rust_binding.wasm"
printf '%s' "${CARGO_ENCODED_RUSTFLAGS:-}" > "$ARTIFACT_TASK_LOG.wasm-flags"
printf '%s' "${ARTIFACT_EMBEDDED_PATH:-}" >> "$CARGO_TARGET_DIR/wasm32-unknown-unknown/release/morphir_rust_binding.wasm"
;;
*)
if [ -n "${CARGO_ENCODED_RUSTFLAGS:-}" ]; then
printf '%s\n' "cargo $*" >> "$ARTIFACT_TASK_LOG.leaked-flags"
fi
;;
esac
""",
)
self._write_tool(
"rustc",
"""
case "$*" in
'--print sysroot') printf '%s\n' "${ARTIFACT_SYSROOT-/fixture/sysroot}" ;;
'-vV') printf 'rustc 1.0.0\ncommit-hash: %s\nhost: fixture\n' "${ARTIFACT_RUSTC_COMMIT-abc123}" ;;
*) exit 1 ;;
esac
""",
)
Expand Down
112 changes: 112 additions & 0 deletions tests/ci/test_package_extension_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,118 @@ def test_staging_cleanup_io_error_has_no_traceback(self) -> None:
self.assertNotIn("Traceback", result.stderr)
self.assertFalse(fixture.log.exists())

def test_the_guest_build_remaps_machine_paths_and_native_builds_do_not(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run({"CARGO_HOME": "/fixture/cargo home"})

self.assertEqual(0, result.returncode, result.stderr)
flags = Path(f"{fixture.log}.wasm-flags").read_text(encoding="utf-8")
self.assertEqual(
[
"--remap-path-prefix=/fixture/cargo home=/cargo",
"--remap-path-prefix=/fixture/sysroot/lib/rustlib/src/rust=/rustc/abc123",
],
flags.split("\x1f"),
)
self.assertFalse(Path(f"{fixture.log}.leaked-flags").exists())

def test_cargo_home_defaults_to_the_directory_cargo_uses(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")
environment = {"HOME": "/fixture/home", "CARGO_HOME": ""}

result = fixture.run(environment)

self.assertEqual(0, result.returncode, result.stderr)
flags = Path(f"{fixture.log}.wasm-flags").read_text(encoding="utf-8")
self.assertIn("--remap-path-prefix=/fixture/home/.cargo=/cargo", flags)

def test_a_guest_that_embeds_a_machine_path_fails_before_packaging(self) -> None:
for embedded in ("/fixture/cargo/registry/src/a.rs", "/fixture/sysroot/lib/rustlib/src/rust/b.rs"):
with self.subTest(embedded=embedded), tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run(
{"CARGO_HOME": "/fixture/cargo", "ARTIFACT_EMBEDDED_PATH": embedded}
)

self.assertNotEqual(0, result.returncode)
self.assertIn("embeds a build machine path", result.stderr)
self.assertFalse(any((fixture.root / ".morphir/build/extensions/rust").glob("*.wasm")))

def test_a_cargo_home_that_is_already_the_canonical_path_is_not_a_machine_path(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run(
{"CARGO_HOME": "/cargo", "ARTIFACT_EMBEDDED_PATH": "/cargo/registry/src/a.rs"}
)

self.assertEqual(0, result.returncode, result.stderr)
self.assertTrue(any((fixture.root / ".morphir/build/extensions/rust").glob("*.wasm")))

def test_the_canonical_cargo_home_does_not_hide_a_toolchain_path(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run(
{
"CARGO_HOME": "/cargo",
"ARTIFACT_EMBEDDED_PATH": "/fixture/sysroot/lib/rustlib/src/rust/b.rs",
}
)

self.assertNotEqual(0, result.returncode)
self.assertIn("embeds a build machine path", result.stderr)

def test_a_trailing_slash_on_cargo_home_changes_nothing(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run(
{"CARGO_HOME": "/fixture/cargo/", "ARTIFACT_EMBEDDED_PATH": "/fixture/cargo/registry/a.rs"}
)

self.assertNotEqual(0, result.returncode)
self.assertIn("embeds a build machine path", result.stderr)
flags = Path(f"{fixture.log}.wasm-flags").read_text(encoding="utf-8")
self.assertIn("--remap-path-prefix=/fixture/cargo=/cargo\x1f", flags)

def test_an_unknown_toolchain_fails_instead_of_building_without_remapping(self) -> None:
for environment in ({"ARTIFACT_SYSROOT": ""}, {"ARTIFACT_RUSTC_COMMIT": ""}):
with self.subTest(environment=environment), tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")

result = fixture.run(environment)

self.assertNotEqual(0, result.returncode)
self.assertFalse(Path(f"{fixture.log}.wasm-flags").exists())

def test_every_artifact_task_builds_its_guest_with_the_same_path_rules(self) -> None:
for short_id, package in (("rust", "morphir-rust-binding"), ("elm-native", "morphir-elm-binding")):
with self.subTest(short_id=short_id):
script = (RUST_ARTIFACT_TASK.parent / short_id).read_text(encoding="utf-8")
self.assertIn(
f'CARGO_ENCODED_RUSTFLAGS="$path_flags" cargo build --locked --release -p {package}'
" --target wasm32-unknown-unknown",
script,
)
self.assertLess(
script.index('CARGO_ENCODED_RUSTFLAGS="$path_flags"'),
script.index('refuse_machine_paths "$WASM"'),
)
self.assertLess(
script.index('refuse_machine_paths "$WASM"'),
script.index("set -- scripts/package_extension.py"),
)
def rules(short_id: str) -> str:
script = (RUST_ARTIFACT_TASK.parent / short_id).read_text(encoding="utf-8")
return script[script.index("guest_path_flags() {"):script.index("run_pipeline() {")]

self.assertEqual(rules("rust"), rules("elm-native"))

def test_clean_tree_records_the_head_commit(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
fixture = ArtifactTaskFixture(Path(temporary_directory) / "repository")
Expand Down
Loading