diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 862072329..ccea22fea 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -51,6 +51,13 @@ updates: - dependency-name: "monty" - dependency-name: "monty-types" + # wasmtime is pinned exactly (=48.0.5): the `cpython` feature embeds a + # module precompiled by build.rs, and a precompiled module loads only + # into the same wasmtime version. 49.x also needs a newer rustc than + # rust-toolchain.toml pins. Bump deliberately together with the + # toolchain; see knowledge/runtimes/cpython-wasm.md. + - dependency-name: "wasmtime" + # idna_adapter is deliberately held at 1.0.0. It is not a normal # dependency: the crate exists so the top-level lockfile can choose # `idna`'s Unicode back end, and 1.0.x is the "no back end" stream that diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7c17ddd9..cd20b4ffb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,9 @@ jobs: - name: Install cargo-vet uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2 with: - tool: cargo-vet + # 0.10.2+ parses trusted-publisher entries in imports.lock (wasmtime + # crates publish via GitHub trusted publishing); 0.10.0 rejects them. + tool: cargo-vet@0.10.2 - name: Supply chain audit (cargo-vet) run: cargo vet --locked @@ -300,6 +302,12 @@ jobs: - name: Run doc tests run: cargo test --workspace --doc --features http_client,ssh,sqlite + # CPython (wasm) python3: unit, integration, capability (diffed against + # the runner's python3), security and load tests. Feature-gated, so the + # main test step above does not build it into bashkit. + - name: Run CPython tests + run: cargo test -p bashkit --features cpython --lib --test integration -- cpython + - name: Run realfs tests run: cargo test --features realfs -p bashkit --test realfs_tests -p bashkit-cli @@ -422,6 +430,7 @@ jobs: cargo run --example live_mounts cargo run --example git_workflow --features git cargo run --example python_external_functions --features python + cargo run --example cpython_scripts --features cpython cargo run --example typescript_external_functions --features typescript cargo run --example realfs_readonly --features realfs cargo run --example realfs_readwrite --features realfs @@ -552,7 +561,9 @@ jobs: - name: Verify fuzz targets compile working-directory: crates/bashkit - run: cargo +nightly fuzz build + run: | + cargo +nightly fuzz build + cargo +nightly fuzz build --features cpython cpython_fuzz # Gate job for branch protection — name must stay "Check" check: diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 899dd5859..3e475093b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - target: [parser_fuzz, lexer_fuzz, analyze_fuzz, arithmetic_fuzz, glob_fuzz] + target: [parser_fuzz, lexer_fuzz, analyze_fuzz, arithmetic_fuzz, glob_fuzz, cpython_fuzz] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -60,7 +60,9 @@ jobs: echo "::error::duration must be a positive integer with no leading zeros, got: $DURATION_INPUT" exit 1 fi - cargo +nightly fuzz run ${{ matrix.target }} -- \ + features=() + if [ "${{ matrix.target }}" = cpython_fuzz ]; then features=(--features cpython); fi + cargo +nightly fuzz run "${features[@]}" ${{ matrix.target }} -- \ -max_total_time="$DURATION_INPUT" \ -print_final_stats=1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6b727f26a..019028859 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,10 +13,51 @@ env: CARGO_TERM_COLOR: always jobs: + # Publish bashkit-cpython-wasm (optional dependency of bashkit's `cpython` + # feature; must exist on crates.io before bashkit can be verified) + publish-bashkit-cpython-wasm: + name: Publish bashkit-cpython-wasm + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Verify publish source is on main + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + SOURCE_SHA=$(git rev-parse HEAD) + if ! git merge-base --is-ancestor "$SOURCE_SHA" origin/main; then + echo "Error: publish source $SOURCE_SHA is not reachable from origin/main" + exit 1 + fi + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + + - name: Verify bashkit-cpython-wasm package without credentials + run: | + cargo publish --dry-run -p bashkit-cpython-wasm + # crates.io rejects crates over 10 MiB; the embedded snapshot and + # stdlib keep this one close to that limit. + size=$(stat -c %s target/package/bashkit-cpython-wasm-*.crate) + echo "crate size: $size bytes" + if [ "$size" -ge 10485760 ]; then + echo "Error: bashkit-cpython-wasm crate exceeds the crates.io 10 MiB limit" + exit 1 + fi + + - name: Publish bashkit-cpython-wasm to crates.io + run: cargo publish --no-verify -p bashkit-cpython-wasm + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + # Publish bashkit (core library) to crates.io publish-bashkit: name: Publish bashkit runs-on: ubuntu-latest + needs: publish-bashkit-cpython-wasm environment: release steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -59,6 +100,9 @@ jobs: fi echo "Version verified: $TAG_VERSION" + - name: Wait for crates.io index update + run: sleep 30 + - name: Verify bashkit package without credentials run: cargo publish --dry-run -p bashkit @@ -123,7 +167,7 @@ jobs: verify-publish: name: Verify published versions runs-on: ubuntu-latest - needs: [publish-bashkit, publish-bashkit-cli, publish-bashkit-scripted-tool] + needs: [publish-bashkit-cpython-wasm, publish-bashkit, publish-bashkit-cli, publish-bashkit-scripted-tool] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -136,4 +180,4 @@ jobs: run: | EXPECTED=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/') echo "Expected version: $EXPECTED" - python3 scripts/verify_crates_publish.py --expected "$EXPECTED" bashkit bashkit-cli bashkit-scripted-tool + python3 scripts/verify_crates_publish.py --expected "$EXPECTED" bashkit-cpython-wasm bashkit bashkit-cli bashkit-scripted-tool diff --git a/AGENTS.md b/AGENTS.md index 28f983d91..c07ac31fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | integrations/script-analysis | Static pre-execution script introspection for permission gating | | integrations/git-support | Sandboxed git operations on VFS | | runtimes/python-builtin | Embedded Python via Monty, security, resource limits | +| runtimes/cpython-wasm | Real CPython 3.14 on WASI (Wizer snapshot, Pulley AOT), WASI-on-VFS host, limits | | operations/eval | LLM eval study on the mira framework, dataset format, scoring | | operations/maintenance | Pre-release maintenance requirements | | runtimes/python-package | Python package, PyPI wheels, platform matrix | diff --git a/Cargo.lock b/Cargo.lock index 400765f37..9a5aa6713 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -177,6 +186,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + [[package]] name = "arc-swap" version = "1.9.2" @@ -347,6 +362,7 @@ dependencies = [ "anyhow", "async-trait", "base64 0.23.1", + "bashkit-cpython-wasm", "bigdecimal", "bitflags 2.13.2", "branches", @@ -394,7 +410,7 @@ dependencies = [ "serde_yaml_ng", "serial_test", "sha1", - "sha2", + "sha2 0.11.0", "tempfile", "thiserror", "tokio", @@ -406,6 +422,7 @@ dependencies = [ "url", "vt100", "wasm-bindgen-futures", + "wasmtime", "web-time", "zapcode-core", "zeroize", @@ -464,6 +481,14 @@ dependencies = [ "toml", ] +[[package]] +name = "bashkit-cpython-wasm" +version = "0.18.2" +dependencies = [ + "flate2", + "wasmtime", +] + [[package]] name = "bashkit-eval" version = "0.18.2" @@ -550,7 +575,7 @@ checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" dependencies = [ "blowfish", "pbkdf2", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -611,7 +636,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest", + "digest 0.11.3", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array 0.14.7", ] [[package]] @@ -668,6 +702,9 @@ name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +dependencies = [ + "allocator-api2 0.2.21", +] [[package]] name = "bytecount" @@ -850,7 +887,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "crypto-common 0.2.2", "inout 0.2.2", "zeroize", @@ -1033,6 +1070,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" +[[package]] +name = "cpp_demangle" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -1057,6 +1103,152 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-assembler-x64" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfbb8f30b72ef5cedf2dbd85ee2f167a33c888d492a40e999250187138663c9" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5715fdd9d897798ba2bb834df5d6c1129319d51e82ddcf81812c23231e9b852b" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a266da98b248dd98dfc8a46d45d57810db04a55b81a6d8c7a9c6cf0bd2db7c3" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67c1887ed2194ef3ecc283a9add4fe8a79d249a293063046f2adedc31ae7200" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90103abeafa22cc578c577999bdf1b51c32b8a59e15a09789841b4e9a854588d" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.17.1", + "libm", + "log", + "postcard", + "pulley-interpreter", + "regalloc2", + "rustc-hash", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d73eaff808bc302e2899e42d822c0e64df23f38facb98ecdf695d1817d2063a3" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de19c4fc55ac3cdeb8e153f35ef9f51db197da91a64c9811d0ddd21ed1b9d18d" + +[[package]] +name = "cranelift-control" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbd919579d74a7212e6e849dc24ff015cb823e70051d01559eea828efaf773f" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af201af93fa8a576ac4b04b901ac3f252545d7b703053c922526fce28f65a7be" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69395ceee43706b93bf869b3a6631ab8d9dcb2bf19f9af4d1d2db0cb9e18af18" +dependencies = [ + "cranelift-codegen", + "hashbrown 0.17.1", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d98a81f71e2ddc61849ca1454ff424fbf42a3ee5529365be8c0859ddf5b9a" + +[[package]] +name = "cranelift-native" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c79e0265b754f67ff76c42d6db4856dabf9101e3bb1b23c83255e2438dd02" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" + [[package]] name = "crc32c" version = "0.6.8" @@ -1240,7 +1432,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.3.1", "curve25519-dalek-derive", - "digest", + "digest 0.11.3", "fiat-crypto", "rand_core 0.10.1", "rustc_version", @@ -1344,13 +1536,23 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", "ctutils", @@ -1392,7 +1594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der", - "digest", + "digest 0.11.3", "elliptic-curve", "rfc6979", "signature", @@ -1420,7 +1622,7 @@ dependencies = [ "ed25519", "rand_core 0.10.1", "serde", - "sha2", + "sha2 0.11.0", "signature", "subtle", "zeroize", @@ -1441,7 +1643,7 @@ dependencies = [ "base16ct", "crypto-bigint", "crypto-common 0.2.2", - "digest", + "digest 0.11.3", "ff", "group", "hkdf", @@ -1853,6 +2055,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap", + "stable_deref_trait", +] + [[package]] name = "gloo-timers" version = "0.4.0" @@ -1912,6 +2126,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", + "serde", + "serde_core", +] [[package]] name = "heapless" @@ -1972,7 +2191,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -2272,6 +2491,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.1.0" @@ -2598,6 +2823,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-parse-float" version = "1.0.6" @@ -2721,6 +2952,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "manyhow" version = "0.11.4" @@ -2760,7 +2997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -2775,6 +3012,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memfd" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" +dependencies = [ + "rustix", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -3081,6 +3327,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.1", + "indexmap", + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3345,7 +3603,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3359,7 +3617,7 @@ dependencies = [ "fiat-crypto", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3373,7 +3631,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3408,7 +3666,7 @@ dependencies = [ "futures", "log", "rand 0.10.3", - "sha2", + "sha2 0.11.0", "thiserror", "tokio", "windows", @@ -3470,7 +3728,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", + "digest 0.11.3", "hmac", ] @@ -3627,7 +3885,7 @@ dependencies = [ "pbkdf2", "rand_core 0.10.1", "scrypt", - "sha2", + "sha2 0.11.0", "spki", ] @@ -3860,6 +4118,29 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulley-interpreter" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc9a7073f2530aab972ac0a7ce43624ff59de8ad06b76a4ef4021f7ddd1035a" +dependencies = [ + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", +] + +[[package]] +name = "pulley-macros" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864f92c9538de48325a4f8709ec83e56471446798d48a29073ca094c5447b549" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pyo3" version = "0.29.2" @@ -4130,6 +4411,21 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regalloc2" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" +dependencies = [ + "allocator-api2 0.2.21", + "bumpalo", + "hashbrown 0.17.1", + "log", + "rustc-hash", + "serde", + "smallvec", +] + [[package]] name = "regex" version = "1.13.1" @@ -4247,11 +4543,11 @@ dependencies = [ "const-oid", "crypto-bigint", "crypto-primes", - "digest", + "digest 0.11.3", "pkcs1", "pkcs8", "rand_core 0.10.1", - "sha2", + "sha2 0.11.0", "signature", "spki", "zeroize", @@ -4361,7 +4657,7 @@ dependencies = [ "data-encoding", "delegate", "der", - "digest", + "digest 0.11.3", "ecdsa", "ed25519-dalek", "elliptic-curve", @@ -4398,7 +4694,7 @@ dependencies = [ "scrypt", "sec1", "sha1", - "sha2", + "sha2 0.11.0", "sha3 0.12.0", "signature", "spki", @@ -4436,6 +4732,12 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -4644,7 +4946,7 @@ dependencies = [ "cfg-if", "pbkdf2", "salsa20", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -4695,6 +4997,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "send_wrapper" @@ -4830,7 +5136,7 @@ checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "digest", + "digest 0.11.3", ] [[package]] @@ -4839,6 +5145,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -4847,7 +5164,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "digest", + "digest 0.11.3", ] [[package]] @@ -4856,7 +5173,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest", + "digest 0.11.3", "keccak", ] @@ -4866,7 +5183,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ - "digest", + "digest 0.11.3", "keccak", "sponge-cursor", ] @@ -4932,7 +5249,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ - "digest", + "digest 0.11.3", "rand_core 0.10.1", ] @@ -5080,7 +5397,7 @@ dependencies = [ "bytes", "crypto-bigint", "ctutils", - "digest", + "digest 0.11.3", "pem-rfc7468", "zeroize", ] @@ -5104,7 +5421,7 @@ dependencies = [ "rsa", "sec1", "sha1", - "sha2", + "sha2 0.11.0", "signature", "ssh-cipher", "ssh-encoding", @@ -5262,6 +5579,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.4" @@ -6027,6 +6353,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebaca0b540bdb15d195ff34fb23074255948708b8916a11a339ba4513dc61906" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1880dd9a848aa68f387966256c34ad49abb4607e3cfa35720d2723d0591cd3" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.5.0" @@ -6040,6 +6388,230 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5ff8685b587722894b6f25958fc608d315b93b8d23647bfb567664e7006d757" +dependencies = [ + "bitflags 2.13.2", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wasmprinter" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b03c8f270f3f384f70f42869b5ff7f6253fe4f0aa8590f7edd332c1943e5f51" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser", +] + +[[package]] +name = "wasmtime" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94eda6b1c1d8ade526a67e4ce50cc6234d1c7af68e5d7bf13f40bc07e70daba" +dependencies = [ + "addr2line", + "async-trait", + "bitflags 2.13.2", + "bumpalo", + "cc", + "futures", + "libc", + "log", + "mach2", + "memfd", + "object", + "once_cell", + "postcard", + "pulley-interpreter", + "rayon", + "rustix", + "serde", + "serde_derive", + "smallvec", + "target-lexicon", + "wasmparser", + "wasmtime-environ", + "wasmtime-internal-component-macro", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-environ" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84ca4865ece3a37abb34c2df96c741ff4aed308d35744d84095cff79848752d4" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "hashbrown 0.17.1", + "indexmap", + "log", + "object", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasm-encoder", + "wasmparser", + "wasmprinter", + "wasmtime-internal-component-util", + "wasmtime-internal-core", +] + +[[package]] +name = "wasmtime-internal-component-macro" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae080b05e5e0356a2167c478c264e26f3fe5dbaaf1b285f8f5501301b99facb" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-internal-component-util" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f014d9ad6f6acbc449e9d7c4e7b7757b4b6b1a84a3662f66ea6da2ebed788c" + +[[package]] +name = "wasmtime-internal-core" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc0a0310fc4a3f545427c7880362657f72f838ffcfe4d3b8c864021df3b3488" +dependencies = [ + "hashbrown 0.17.1", + "libm", + "serde", +] + +[[package]] +name = "wasmtime-internal-cranelift" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6c8d2b6f8ed3a784fcd127a1936589db8051febbd228230660175e41b5751d" +dependencies = [ + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools 0.14.0", + "log", + "object", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-fiber" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92b558b5892ca405508103641fa3a3ec2e986b1cc679a083ed47e6bcb08ecf31" +dependencies = [ + "cc", + "libc", + "rustix", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-jit-debug" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0669ce70a8a8c92eb8ad148c1d7b9b771aab37787e9462efd75b2e2b4575406" +dependencies = [ + "cc", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa66405e95fc7b82adb195c14fa8b38f45046c3ad99a273b06262b8eb755e76" +dependencies = [ + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-unwinder" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70555458e519d6fb8790e042b4274132ab58171ae0f2a6917c3dc29ded0184d0" +dependencies = [ + "cranelift-codegen", + "log", + "object", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-internal-versioned-export-macros" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca407e3e2809c0e6568be132065095a16f184c30c6bee4157dfdc497aee10a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "wasmtime-internal-wit-bindgen" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aacdb42fb0b393353e402a1534e0b492a6b0fddf6528bfaec65c5f56b990c43" +dependencies = [ + "anyhow", + "bitflags 2.13.2", + "heck", + "indexmap", + "wit-component", + "wit-parser", +] + [[package]] name = "web-sys" version = "0.3.105" @@ -6304,6 +6876,44 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-component" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c96b3cea6838dbea55173baa56991fcb087896c321b0a770db201e5a0bd3506" +dependencies = [ + "anyhow", + "bitflags 2.13.2", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22672d61bfb9df46a16b46ce38522293aee1cc4749fc84a3464ce4083b3d5fd6" +dependencies = [ + "anyhow", + "hashbrown 0.17.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-ident", + "wasmparser", +] + [[package]] name = "wnaf" version = "0.14.1" diff --git a/Cargo.toml b/Cargo.toml index 224b1709f..74230cd80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,12 @@ keywords = ["bash", "virtual", "ai", "agent", "everruns"] categories = ["command-line-utilities", "parser-implementations"] [workspace.dependencies] +# Embedded CPython runtime (`cpython` feature, crates/bashkit-cpython-wasm). +# Important decision: pinned exactly. Precompiled `.cwasm` artifacts only load +# into the identical wasmtime version. Runtime build: no Cranelift, Pulley +# interpreter only; Cranelift is added as a build-dependency by the artifact +# crate. 48.x is the newest line supporting the pinned rustc 1.95. +wasmtime = { version = "=48.0.5", default-features = false, features = ["runtime", "std", "pulley", "async"] } # Async runtime # Base features are WASM-compatible; crates needing more (rt-multi-thread, fs, net, process) # add them in their own Cargo.toml or via [target.'cfg(...)'.dependencies]. @@ -179,6 +185,19 @@ napi-build = "2" [profile.test] debug = "line-tables-only" +# Important decision: the `cpython` feature runs CPython on wasmtime's Pulley +# interpreter, and its build script compiles a ~30 MB wasm module with +# Cranelift. Unoptimized, both are 10-50x slower, which makes debug test runs +# and builds crawl. Optimize just those crates in every profile. +[profile.dev.package.pulley-interpreter] +opt-level = 3 +[profile.dev.package.cranelift-codegen] +opt-level = 3 +[profile.dev.package.wasmtime-internal-cranelift] +opt-level = 3 +[profile.dev.package.regalloc2] +opt-level = 3 + [profile.release] lto = "thin" codegen-units = 1 diff --git a/README.md b/README.md index 0d9767e47..e06f9275a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Homepage: [bashkit.sh](https://bashkit.sh) - **Async-first** - Built on tokio - **Language bindings** - Python (PyO3) and JavaScript/TypeScript (NAPI-RS) for Node.js, Bun, and Deno - **Experimental: Git support** - Virtual git operations on the virtual filesystem (`git` feature) -- **Experimental: Python support** - Embedded Python interpreter via [Monty](https://github.com/pydantic/monty) (`python` feature) +- **Experimental: Python support** - Embedded Python interpreter via [Monty](https://github.com/pydantic/monty) (`python` feature), or real CPython 3.14 in a WebAssembly sandbox (`cpython` feature) - **Experimental: TypeScript support** - Embedded TypeScript interpreter via [ZapCode](https://github.com/TheUncharted/zapcode) (`typescript` feature) - **Experimental: SQLite support** - Embedded SQLite-compatible engine via [Turso](https://github.com/tursodatabase/turso) (`sqlite` feature) @@ -42,6 +42,7 @@ Optional features: ```bash cargo add bashkit --features git # Virtual git operations cargo add bashkit --features python # Embedded Python interpreter +cargo add bashkit --features cpython # Real CPython 3.14 (WebAssembly sandbox) cargo add bashkit --features typescript # Embedded TypeScript interpreter cargo add bashkit --features sqlite # Embedded SQLite engine (Turso) cargo add bashkit --features realfs # Real filesystem backend @@ -177,7 +178,7 @@ check must consult it. Available in Rust, Node (`bash.analyze()`), and Python | Data formats | `csv`, `json`, `yaml`, `tomlq`, `template`, `envsubst` | | Network | `curl`, `wget` (requires allowlist), `http` | | DevOps | `assert`, `dotenv`, `glob`, `log`, `retry`, `semver`, `verify`, `parallel`, `patch` | -| Experimental | `python`, `python3` (requires `python` feature), `ts`, `typescript`, `node`, `deno`, `bun` (requires `typescript` feature), `git` (requires `git` feature), `ssh`, `scp`, `sftp` (requires `ssh` feature), `sqlite`, `sqlite3` (requires `sqlite` feature) | +| Experimental | `python`, `python3` (requires `python` or `cpython` feature), `ts`, `typescript`, `node`, `deno`, `bun` (requires `typescript` feature), `git` (requires `git` feature), `ssh`, `scp`, `sftp` (requires `ssh` feature), `sqlite`, `sqlite3` (requires `sqlite` feature) | ## Shell Features @@ -390,6 +391,11 @@ Security note: `re` is intentionally disabled due to regex backtracking DoS risk Limitations: file I/O is VFS-scoped, no network, no classes, no third-party imports. See [crates/bashkit/docs/python.md](crates/bashkit/docs/python.md) for the full guide. +For full CPython 3.14 (complete language and stdlib, CPython CLI semantics), enable the +`cpython` feature and call `.cpython()` instead. The interpreter runs as a WebAssembly guest, +one fresh instance per call. See [crates/bashkit/docs/cpython.md](crates/bashkit/docs/cpython.md) +for the trade-offs against Monty and the limitations. + ## Experimental: TypeScript Support Enable the `typescript` feature to embed the [ZapCode](https://github.com/TheUncharted/zapcode) TypeScript interpreter (pure Rust, no V8). diff --git a/crates/bashkit-cli/Cargo.toml b/crates/bashkit-cli/Cargo.toml index 5077506dd..023379c88 100644 --- a/crates/bashkit-cli/Cargo.toml +++ b/crates/bashkit-cli/Cargo.toml @@ -25,6 +25,8 @@ doc = false # Disable to avoid collision with bashkit library docs [features] default = ["python", "sqlite", "interactive"] python = ["bashkit/python"] +# python/python3 via real CPython 3.14 in a wasm sandbox (replaces Monty) +cpython = ["bashkit/cpython"] sqlite = ["bashkit/sqlite"] realfs = ["bashkit/realfs"] interactive = ["dep:rustyline", "dep:terminal_size", "dep:signal-hook"] diff --git a/crates/bashkit-cli/src/main.rs b/crates/bashkit-cli/src/main.rs index 9fe861f1c..a19efffdf 100644 --- a/crates/bashkit-cli/src/main.rs +++ b/crates/bashkit-cli/src/main.rs @@ -76,9 +76,12 @@ struct Args { #[arg(long)] no_git: bool, - /// Disable python builtin (monty backend) - #[cfg_attr(not(feature = "python"), arg(long, hide = true))] - #[cfg_attr(feature = "python", arg(long))] + /// Disable python/python3 builtins (Monty, or CPython with `cpython`) + #[cfg_attr( + not(any(feature = "python", feature = "cpython")), + arg(long, hide = true) + )] + #[cfg_attr(any(feature = "python", feature = "cpython"), arg(long))] no_python: bool, /// Disable sqlite/sqlite3 builtin (turso backend, BETA) @@ -188,6 +191,12 @@ fn configure_bash(args: &Args, mode: CliMode) -> bashkit::BashBuilder { builder = builder.env(PYTHON_INPROCESS_OPT_IN_ENV, "1"); } + // CPython (wasm) replaces Monty for python/python3 when compiled in. + #[cfg(feature = "cpython")] + if !args.no_python { + builder = builder.cpython(); + } + #[cfg(feature = "sqlite")] if !args.no_sqlite { builder = builder.sqlite(); diff --git a/crates/bashkit-cpython-wasm/Cargo.toml b/crates/bashkit-cpython-wasm/Cargo.toml new file mode 100644 index 000000000..68ed4c7a0 --- /dev/null +++ b/crates/bashkit-cpython-wasm/Cargo.toml @@ -0,0 +1,40 @@ +# CPython 3.14 for WebAssembly (wasm32-wasip1), pre-initialized and compiled +# ahead of time to Wasmtime's Pulley bytecode. Consumed by bashkit's +# `cpython` feature; see knowledge/runtimes/cpython-wasm.md. +# +# Decisions: +# - Artifacts (snapshot + stdlib zip) are committed; `guest/build.sh` rebuilds +# them. No network access at build or run time. +# - build.rs compiles the snapshot to Pulley once per build, so no wasm is ever +# compiled at run time and the output is identical on every 64-bit +# little-endian host. Cranelift is a build-dependency only. +# - wasmtime is pinned exactly: a `.cwasm` only loads into the same version +# with the same compile-affecting configuration (`engine_config`). + +[package] +name = "bashkit-cpython-wasm" +version.workspace = true +edition.workspace = true +authors.workspace = true +repository.workspace = true +license = "MIT AND Python-2.0 AND Zlib AND BSD-2-Clause" +description = "CPython 3.14 compiled to WASI with a pre-initialized snapshot, for bashkit's sandboxed python3" +keywords = ["python", "wasm", "sandbox", "bashkit"] +categories = ["wasm", "development-tools"] +readme = "README.md" +include = [ + "src/**", + "build.rs", + "artifacts/**", + "guest/**", + "README.md", + "THIRD_PARTY_LICENSES.md", +] + +[dependencies] +wasmtime = { workspace = true } + +[build-dependencies] +wasmtime = { workspace = true, features = ["cranelift", "parallel-compilation"] } +flate2 = { workspace = true } + diff --git a/crates/bashkit-cpython-wasm/README.md b/crates/bashkit-cpython-wasm/README.md new file mode 100644 index 000000000..c74ce11b5 --- /dev/null +++ b/crates/bashkit-cpython-wasm/README.md @@ -0,0 +1,41 @@ +# bashkit-cpython-wasm + +CPython 3.14 compiled to WebAssembly (`wasm32-wasip1`), pre-initialized with +Wizer and precompiled at build time to Wasmtime's portable Pulley bytecode. + +This crate ships bytes, not behavior: the precompiled interpreter module, the +zipped standard library and the matching `wasmtime::Config`. It is consumed by +[bashkit](https://crates.io/crates/bashkit)'s `cpython` feature, which provides +the WASI host over bashkit's virtual filesystem and the `python3` builtin. +Most users want that feature, not this crate directly: + +```toml +[dependencies] +bashkit = { version = "0.18.2", features = ["cpython"] } +``` + +## Contents + +| Item | Description | +|------|-------------| +| `load_module(&engine)` | The interpreter snapshot as a `wasmtime::Module` (mapped in place on first load) | +| `engine_config()` / `engine()` | The compile-affecting Wasmtime configuration the module was built for | +| `STDLIB_ZIP`, `STDLIB_ZIP_PATH` | Stdlib sources; the host must serve these exact bytes at this guest path | +| `PYTHON_VERSION` | `3.14.8` | + +Guest contract: export `bashkit_run() -> i32` runs one `python3` invocation, +reading argv, environment (including `PWD`) and stdio through WASI preview1; +exactly one preopen, `/`, at fd 3; one instance per call. + +## Rebuilding + +`guest/build.sh` reproduces `artifacts/` from pinned sources (CPython, WASI +SDK, zlib, SQLite, the Wasmtime CLI for `wizer`). `build.rs` compiles the +snapshot to Pulley on every build; no network access is needed at build or +run time. + +## License + +Crate code: MIT. Embedded CPython: Python Software Foundation License +(Python-2.0), plus the bundled libraries listed in +[THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md). diff --git a/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md b/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md new file mode 100644 index 000000000..e9ffefa80 --- /dev/null +++ b/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md @@ -0,0 +1,446 @@ +# Third-party licenses + +`bashkit-cpython-wasm` embeds a WebAssembly build of CPython and the +libraries statically linked into it (see `guest/build.sh` and +`artifacts/MANIFEST` for exact versions). The crate's own code (Rust sources, +`guest/` sources and scripts) is MIT, like the rest of bashkit. + +| Component | Version | License | Where | +|-----------|---------|---------|-------| +| CPython (interpreter and stdlib) | 3.14.8 | Python-2.0 (PSF License Agreement and predecessors) | `artifacts/python.wasm.gz`, `artifacts/python314.zip` | +| libmpdec (`decimal`) | bundled with CPython | BSD-2-Clause | `artifacts/python.wasm.gz` | +| Expat (`pyexpat`, `xml.etree`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` | +| HACL* (`hashlib` md5/sha1/sha2/sha3/blake2, `hmac`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` | +| zlib | 1.3.1 | Zlib | `artifacts/python.wasm.gz` | +| SQLite | 3.50.4 | Public domain (blessing) | `artifacts/python.wasm.gz` | +| wasi-libc (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT; musl parts MIT | `artifacts/python.wasm.gz` | +| compiler-rt builtins (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception | `artifacts/python.wasm.gz` | + +The full license texts that require reproduction follow. + +## CPython + +```text +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +``` + +## libmpdec + +```text +Copyright (c) 2008-2020 Stefan Krah. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. +``` + +## Expat + +```text +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` + +## HACL* + +```text +MIT License + +Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation +Copyright (c) 2022-2023 HACL* Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +``` + +## zlib + +```text +Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu +``` + +## SQLite + +```text +The author disclaims copyright to this source code. In place of a legal +notice, here is a blessing: + + May you do good and not evil. + May you find forgiveness for yourself and forgive others. + May you share freely, never taking more than you give. +``` + +## wasi-libc and musl + +```text +wasi-libc: https://github.com/WebAssembly/wasi-libc/blob/main/LICENSE-MIT +musl: https://git.musl-libc.org/cgit/musl/tree/COPYRIGHT + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` diff --git a/crates/bashkit-cpython-wasm/artifacts/MANIFEST b/crates/bashkit-cpython-wasm/artifacts/MANIFEST new file mode 100644 index 000000000..37673ca25 --- /dev/null +++ b/crates/bashkit-cpython-wasm/artifacts/MANIFEST @@ -0,0 +1,7 @@ +cpython=3.14.8 +wasi_sdk=24 +wasmtime_wizer=49.0.1 +zlib=1.3.1 +sqlite=sqlite-amalgamation-3500400 +9049f2fa9d9bfda5fd44803c0aada4f6c8175fae4db190c9db6735093fb7943f python.wasm.gz +1f6ab556e47987bb7d3b9779e5c9e94022dae90ba1882455123933fa995fd830 python314.zip diff --git a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz new file mode 100644 index 000000000..33b0d7be6 Binary files /dev/null and b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz differ diff --git a/crates/bashkit-cpython-wasm/artifacts/python314.zip b/crates/bashkit-cpython-wasm/artifacts/python314.zip new file mode 100644 index 000000000..fde6c4223 Binary files /dev/null and b/crates/bashkit-cpython-wasm/artifacts/python314.zip differ diff --git a/crates/bashkit-cpython-wasm/build.rs b/crates/bashkit-cpython-wasm/build.rs new file mode 100644 index 000000000..7e236fcce --- /dev/null +++ b/crates/bashkit-cpython-wasm/build.rs @@ -0,0 +1,34 @@ +//! Compile the committed CPython snapshot to Pulley bytecode ahead of time. +//! +//! Decision: compile at build time, never at run time (python3 startup is a +//! product feature). Set `BASHKIT_CPYTHON_CWASM=/path/python.cwasm` to reuse a +//! module precompiled by the same wasmtime version and configuration. + +use std::io::Read; +use std::path::PathBuf; + +include!("src/config.rs"); + +fn main() { + let manifest = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("python.cwasm"); + let snapshot = manifest.join("artifacts/python.wasm.gz"); + println!("cargo:rerun-if-changed={}", snapshot.display()); + println!("cargo:rerun-if-changed=src/config.rs"); + println!("cargo:rerun-if-env-changed=BASHKIT_CPYTHON_CWASM"); + + if let Some(prebuilt) = std::env::var_os("BASHKIT_CPYTHON_CWASM") { + std::fs::copy(&prebuilt, &out).expect("copy BASHKIT_CPYTHON_CWASM"); + return; + } + + let mut wasm = Vec::new(); + flate2::read::GzDecoder::new(std::fs::File::open(&snapshot).expect("open snapshot")) + .read_to_end(&mut wasm) + .expect("decompress snapshot"); + let engine = wasmtime::Engine::new(&engine_config()).expect("engine"); + let cwasm = engine + .precompile_module(&wasm) + .expect("precompile CPython snapshot to Pulley"); + std::fs::write(&out, cwasm).expect("write python.cwasm"); +} diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py new file mode 100644 index 000000000..b4f8a62f7 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py @@ -0,0 +1,352 @@ +"""Bashkit CPython guest driver. + +Runs inside the wasm32-wasip1 CPython snapshot. `preload()` executes once at +build time (wizer); `main()` executes once per `python3` call and emulates +CPython's command line (`-c`, `-m`, script, `-`, stdin) on an interpreter that +is already initialized. + +Decisions: +- Tracebacks hide this driver's frames so output matches real `python3`. +- `random` is re-seeded per call: the snapshot would otherwise hand every + tenant the same sequence. +- The snapshot heap is frozen (`gc.freeze`) so garbage collection only + scans objects created by the current call. +- asyncio's selector loop normally wakes itself through a socketpair. WASI + has no sockets and the guest has no threads or signals that could need a + wakeup, so the self-pipe is disabled (patched once, in the snapshot). +- Options that only matter at interpreter init (-E, -I, -s, -S, -B, -u, -O, + -q, -b, -d, -v, -R, -P, -X) are accepted and ignored; the snapshot already + runs isolated, unbuffered output is irrelevant because the host captures + stdout, and bytecode is never written. +""" + +import sys + +_PRELOAD = ( + "abc", "argparse", "ast", "base64", "binascii", "bisect", "calendar", + "collections", "collections.abc", "contextlib", "copy", "csv", "dataclasses", + "datetime", "decimal", "difflib", "enum", "fnmatch", "fractions", + "functools", "glob", "gzip", "hashlib", "heapq", "html", "io", "itertools", + "json", "keyword", "linecache", "locale", "logging", "math", "numbers", + "operator", "os", "pathlib", "pprint", "random", "re", "shlex", "shutil", + "statistics", "string", "struct", "subprocess", "tempfile", "textwrap", + "time", "token", "tokenize", "traceback", "types", "typing", "unicodedata", + "urllib.parse", "uuid", "warnings", "weakref", "zlib", "runpy", + "_bashkit", "atexit", "builtins", "importlib", "sqlite3", "zipfile", + "asyncio", +) + +_USAGE = """\ +usage: python3 [option] ... [-c cmd | -m mod | file | -] [arg] ... +Options (and corresponding environment variables): +-c cmd : program passed in as string (terminates option list) +-h : print this help message and exit (also -? or --help) +-m mod : run library module as a script (terminates option list) +-V : print the Python version number and exit (also --version) + when given twice, print more information about the build +-W arg : warning control +-x : skip first line of source +file : program read from script file +- : program read from stdin (default; interactive mode if a tty) +arg ...: arguments passed to program in sys.argv[1:] + +Accepted for compatibility and ignored: -b -B -d -E -i -I -O -OO -P -q -R -s -S -u -v -X opt +""" + +_NOARG_FLAGS = set("bBdEiIOPqRsSuvx") + + +def preload(): + """Import common stdlib modules into the snapshot.""" + import importlib + + for name in _PRELOAD: + try: + importlib.import_module(name) + except Exception: # pragma: no cover - optional modules + pass + _patch_asyncio() + # Move every object the snapshot holds into the permanent generation. + # The collector then never traverses (or writes to) snapshot memory, so + # per-call GC cost scales with what the script allocates, and snapshot + # pages stay shared copy-on-write instead of being dirtied by GC headers. + import gc + + gc.collect() + gc.freeze() + + +def _patch_asyncio(): + try: + from asyncio import selector_events + except Exception: # pragma: no cover + return + loop = selector_events.BaseSelectorEventLoop + loop._make_self_pipe = lambda self: None + loop._close_self_pipe = lambda self: None + loop._write_to_self = lambda self: None + + +class _UsageError(Exception): + pass + + +def _parse(args): + """Parse CPython-style options. Returns (mode, value, rest, flags).""" + flags = {"x": False, "V": 0, "warnings": []} + i = 0 + while i < len(args): + arg = args[i] + if arg == "--": + i += 1 + break + if arg in ("--version",): + flags["V"] += 1 + i += 1 + continue + if arg == "--help" or arg == "--help-all": + return ("help", None, [], flags) + if arg.startswith("--"): + if arg == "--check-hash-based-pycs": + i += 2 + continue + raise _UsageError(f"unknown option {arg}") + if not arg.startswith("-") or arg == "-": + break + j = 1 + while j < len(arg): + ch = arg[j] + if ch in ("c", "m"): + value = arg[j + 1 :] + rest_start = i + 1 + if not value: + if i + 1 >= len(args): + raise _UsageError(f"Argument expected for the -{ch} option") + value = args[i + 1] + rest_start = i + 2 + mode = "command" if ch == "c" else "module" + return (mode, value, args[rest_start:], flags) + if ch in ("W", "X"): + value = arg[j + 1 :] + if not value: + if i + 1 >= len(args): + raise _UsageError(f"Argument expected for the -{ch} option") + value = args[i + 1] + i += 1 + if ch == "W": + flags["warnings"].append(value) + break + if ch in ("h", "?"): + return ("help", None, [], flags) + if ch == "V": + flags["V"] += 1 + elif ch == "x": + flags["x"] = True + elif ch not in _NOARG_FLAGS: + raise _UsageError(f"Unknown option: -{ch}") + j += 1 + i += 1 + if flags["V"]: + return ("version", None, [], flags) + if i < len(args): + if args[i] == "-": + return ("stdin", "-", args[i + 1 :], flags) + return ("script", args[i], args[i + 1 :], flags) + return ("stdin", "", [], flags) + + +def _refresh_process_state(): + import os + + import _bashkit + + os.environ.clear() + for item in _bashkit.environ(): + key, sep, value = item.partition("=") + if sep and key: + os.environ[key] = value + limit = os.environ.pop("__BASHKIT_RECURSION_LIMIT", None) + if limit: + try: + sys.setrecursionlimit(max(50, int(limit))) + except (ValueError, RecursionError): + pass + pwd = os.environ.get("PWD") + if pwd: + try: + os.chdir(pwd) + except OSError: + pass + random = sys.modules.get("random") + if random is not None: + random.seed() + + +def _fresh_main(filename=None): + import builtins + import types + + module = types.ModuleType("__main__") + module.__builtins__ = builtins + if filename is not None: + module.__file__ = filename + module.__cached__ = None + sys.modules["__main__"] = module + return module + + +def _strip_driver_frames(tb): + me = __file__ + while tb is not None and tb.tb_frame.f_code.co_filename in (me, ""): + tb = tb.tb_next + return tb + + +def _report(exc): + # The default excepthook formats `exc.__traceback__`, so strip the driver + # frames on the exception itself, not only on the argument. + tb = _strip_driver_frames(exc.__traceback__) + exc = exc.with_traceback(tb) + sys.excepthook(type(exc), exc, tb) + + +def _exit_code(exc): + code = exc.code + if code is None: + return 0 + if isinstance(code, bool): + return int(code) + if isinstance(code, int): + return code & 0xFF + try: + print(code, file=sys.stderr) + except Exception: + pass + return 1 + + +def _release_main(): + # Real CPython finalizes modules at exit, which closes files a script left + # open and flushes their buffers. Drop __main__'s globals and collect so + # those writes reach the filesystem before the host tears the instance down. + try: + import gc + + main = sys.modules.get("__main__") + if main is not None: + main.__dict__.clear() + gc.collect() + except BaseException: + pass + + +def _flush(): + for stream in (sys.stdout, sys.stderr): + try: + if stream is not None: + stream.flush() + except Exception: + pass + + +def _run_source(source, filename, module): + code = compile(source, filename, "exec", dont_inherit=True) + exec(code, module.__dict__) + + +def _execute(mode, value, rest, flags): + import os + + if mode == "help": + sys.stdout.write(_USAGE) + return 0 + if mode == "version": + if flags["V"] > 1: + sys.stdout.write(f"Python {sys.version}\n") + else: + v = sys.version_info + sys.stdout.write(f"Python {v.major}.{v.minor}.{v.micro}\n") + return 0 + if flags["warnings"]: + import warnings + + warnings._processoptions(flags["warnings"]) + + if mode == "command": + sys.argv = ["-c", *rest] + sys.path.insert(0, "") + _run_source(value, "", _fresh_main()) + return 0 + if mode == "module": + import runpy + + sys.argv = [value, *rest] + sys.path.insert(0, os.getcwd()) + _fresh_main() + runpy._run_module_as_main(value, alter_argv=True) + return 0 + if mode == "stdin": + sys.argv = [value, *rest] + sys.path.insert(0, "") + source = sys.stdin.read() + _run_source(source, "", _fresh_main()) + return 0 + # script + path = value + sys.argv = [path, *rest] + if os.path.isdir(path): + import runpy + + sys.path.insert(0, path) + runpy.run_path(path, run_name="__main__") + return 0 + try: + with open(path, "rb") as f: + source = f.read() + except OSError as e: + sys.stderr.write( + f"python3: can't open file {path!r}: [Errno {e.errno}] {e.strerror}\n" + ) + return 2 + if flags["x"]: + nl = source.find(b"\n") + source = b"" if nl < 0 else b"\n" + source[nl + 1 :] + sys.path.insert(0, os.path.dirname(os.path.abspath(path))) + _run_source(source, path, _fresh_main(path)) + return 0 + + +def main(): + import _bashkit + + _refresh_process_state() + args = _bashkit.argv()[1:] + try: + try: + mode, value, rest, flags = _parse(args) + except _UsageError as e: + sys.stderr.write(f"{e}\n{_USAGE.splitlines()[0]}\n") + sys.stderr.write("Try `python -h' for more information.\n") + return 2 + try: + status = _execute(mode, value, rest, flags) + except SystemExit as e: + status = _exit_code(e) + except KeyboardInterrupt as e: + _report(e) + status = 130 + except BaseException as e: + _report(e) + status = 1 + try: + import atexit + + atexit._run_exitfuncs() + except SystemExit as e: + status = _exit_code(e) + except BaseException: + pass + _release_main() + return status + finally: + _flush() diff --git a/crates/bashkit-cpython-wasm/guest/bashkit_main.c b/crates/bashkit-cpython-wasm/guest/bashkit_main.c new file mode 100644 index 000000000..b8f9aef61 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/bashkit_main.c @@ -0,0 +1,182 @@ +// Bashkit CPython guest entrypoint (wasm32-wasip1, reactor model). +// +// Decisions: +// - Interpreter init runs once, at build time, inside `wizer-initialize` +// (`wasmtime wizer`). The snapshot carries an initialized interpreter with +// common stdlib modules imported, so every `python3` call starts from that +// state copy-on-write instead of paying CPython startup (tens of millions of +// instructions; ~460 ms on Pulley). +// - Per-call inputs (argv, environment, cwd, stdin) only exist at run time, so +// `bashkit_run` reads them through WASI and hands them to the pure-Python +// CLI driver in `_bashkit_boot.py`, which emulates CPython's command line. +// - One instance serves exactly one `python3` call. Nothing a tenant does can +// leak into the next call because the host discards the instance. + +#include +#include +#include +#include + +extern void __wasm_call_ctors(void); + +static int bk_initialized = 0; + +// _bashkit.argv() -> list[str]: WASI argv for this call (argv[0] included). +static PyObject *bk_argv(PyObject *self, PyObject *unused) { + (void)self; + (void)unused; + __wasi_size_t argc = 0, buf_size = 0; + if (__wasi_args_sizes_get(&argc, &buf_size) != 0) { + return PyErr_Format(PyExc_OSError, "args_sizes_get failed"); + } + char **argv = calloc(argc + 1, sizeof(char *)); + char *buf = malloc(buf_size + 1); + if (argv == NULL || buf == NULL) { + free(argv); + free(buf); + return PyErr_NoMemory(); + } + PyObject *list = NULL; + if (__wasi_args_get((uint8_t **)argv, (uint8_t *)buf) != 0) { + PyErr_Format(PyExc_OSError, "args_get failed"); + goto done; + } + list = PyList_New(0); + if (list == NULL) { + goto done; + } + for (__wasi_size_t i = 0; i < argc; i++) { + PyObject *s = PyUnicode_DecodeFSDefault(argv[i]); + if (s == NULL || PyList_Append(list, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(list); + goto done; + } + Py_DECREF(s); + } +done: + free(argv); + free(buf); + return list; +} + +// _bashkit.environ() -> list[str]: WASI environment as "KEY=VALUE" strings. +static PyObject *bk_environ(PyObject *self, PyObject *unused) { + (void)self; + (void)unused; + __wasi_size_t count = 0, buf_size = 0; + if (__wasi_environ_sizes_get(&count, &buf_size) != 0) { + return PyErr_Format(PyExc_OSError, "environ_sizes_get failed"); + } + char **env = calloc(count + 1, sizeof(char *)); + char *buf = malloc(buf_size + 1); + if (env == NULL || buf == NULL) { + free(env); + free(buf); + return PyErr_NoMemory(); + } + PyObject *list = NULL; + if (__wasi_environ_get((uint8_t **)env, (uint8_t *)buf) != 0) { + PyErr_Format(PyExc_OSError, "environ_get failed"); + goto done; + } + list = PyList_New(0); + if (list == NULL) { + goto done; + } + for (__wasi_size_t i = 0; i < count; i++) { + PyObject *s = PyUnicode_DecodeFSDefault(env[i]); + if (s == NULL || PyList_Append(list, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(list); + goto done; + } + Py_DECREF(s); + } +done: + free(env); + free(buf); + return list; +} + +static PyMethodDef bk_methods[] = { + {"argv", bk_argv, METH_NOARGS, "WASI argv for this call."}, + {"environ", bk_environ, METH_NOARGS, "WASI environment for this call."}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef bk_module = { + PyModuleDef_HEAD_INIT, "_bashkit", NULL, -1, bk_methods, +}; + +static PyObject *bk_module_init(void) { return PyModule_Create(&bk_module); } + +static void bk_fatal(PyStatus status) { + // Init failures are build-time bugs; abort loudly so wizer fails. + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } +} + +static void bk_initialize(void) { + if (bk_initialized) { + return; + } + __wasm_call_ctors(); + + PyPreConfig preconfig; + PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + bk_fatal(Py_PreInitialize(&preconfig)); + + if (PyImport_AppendInittab("_bashkit", bk_module_init) < 0) { + abort(); + } + + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + config.site_import = 1; + config.write_bytecode = 0; + config.install_signal_handlers = 0; + config.buffered_stdio = 1; + config.user_site_directory = 0; + config.pathconfig_warnings = 0; + bk_fatal(PyConfig_SetString(&config, &config.home, L"/usr/local")); + bk_fatal(PyConfig_SetString(&config, &config.program_name, L"python3")); + bk_fatal(Py_InitializeFromConfig(&config)); + PyConfig_Clear(&config); + + if (PyRun_SimpleString("import _bashkit_boot\n_bashkit_boot.preload()\n") != 0) { + abort(); + } + bk_initialized = 1; +} + +__attribute__((export_name("wizer-initialize"))) void bk_wizer_initialize(void) { + bk_initialize(); +} + +// Run one `python3` invocation; returns the process exit status. +__attribute__((export_name("bashkit_run"))) int bk_run(void) { + bk_initialize(); + PyObject *boot = PyImport_ImportModule("_bashkit_boot"); + if (boot == NULL) { + PyErr_Print(); + return 1; + } + PyObject *result = PyObject_CallMethod(boot, "main", NULL); + Py_DECREF(boot); + if (result == NULL) { + // main() handles SystemExit and tracebacks itself; reaching here is + // an internal error in the driver. + PyErr_Print(); + return 1; + } + long code = PyLong_AsLong(result); + Py_DECREF(result); + if (code == -1 && PyErr_Occurred()) { + PyErr_Clear(); + code = 1; + } + return (int)code; +} diff --git a/crates/bashkit-cpython-wasm/guest/build.sh b/crates/bashkit-cpython-wasm/guest/build.sh new file mode 100755 index 000000000..f585ec219 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/build.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Rebuild the CPython wasm32-wasip1 guest and stdlib committed under +# crates/bashkit-cpython-wasm/artifacts/. +# +# Decisions (see knowledge/runtimes/cpython-wasm.md): +# - CPython is built with its own Tools/wasm/wasi script and WASI SDK 24 +# (PEP 816 support matrix), plus static zlib and sqlite3. +# - The guest is linked as a reactor with bashkit_main.c instead of +# Programs/python.c, then pre-initialized with `wasmtime wizer` so the +# snapshot already holds a running interpreter with common modules imported. +# - The stdlib ships as one deflate-compressed zip of .py sources. The +# snapshot caches the zip directory, so the runtime must serve the exact +# same zip bytes; both artifacts are produced together by this script. +# - Output is a gzip of the stripped snapshot plus the zip. The crate's +# build.rs compiles the snapshot to Pulley bytecode ahead of time. +# +# Usage: guest/build.sh [WORK_DIR] +# Requires: curl, tar, unzip, python3 (>=3.11), make, a C compiler for the +# native build Python. Everything else is downloaded into WORK_DIR. +set -euo pipefail + +CPYTHON_VERSION=3.14.8 +WASI_SDK_MAJOR=24 +WASMTIME_VERSION=49.0.1 +ZLIB_VERSION=1.3.1 +SQLITE_AMALGAMATION=sqlite-amalgamation-3500400 +SQLITE_YEAR=2025 + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CRATE="$(dirname "$HERE")" +WORK="${1:-${TMPDIR:-/tmp}/bashkit-cpython-wasm-build}" +mkdir -p "$WORK" +cd "$WORK" + +arch="$(uname -m)" +case "$arch" in +x86_64) sdk_arch=x86_64 ;; +aarch64 | arm64) sdk_arch=arm64 ;; +*) echo "unsupported build host: $arch" >&2 && exit 1 ;; +esac + +fetch() { # url dest + [ -s "$2" ] || curl -fsSL -o "$2" "$1" +} + +SDK="$WORK/wasi-sdk-${WASI_SDK_MAJOR}.0-${sdk_arch}-linux" +if [ ! -d "$SDK" ]; then + fetch "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_MAJOR}/wasi-sdk-${WASI_SDK_MAJOR}.0-${sdk_arch}-linux.tar.gz" wasi-sdk.tar.gz + tar xzf wasi-sdk.tar.gz +fi +WT_DIR="$WORK/wasmtime-v${WASMTIME_VERSION}-${arch}-linux" +if [ ! -x "$WT_DIR/wasmtime" ]; then + fetch "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${arch}-linux.tar.xz" wasmtime.tar.xz + tar xJf wasmtime.tar.xz +fi +WASMTIME="$WT_DIR/wasmtime" +CC_WASI="$SDK/bin/clang --target=wasm32-wasip1 --sysroot=$SDK/share/wasi-sysroot" + +# --- static deps: zlib (no gz* file API needed) and sqlite3 --------------- +DEPS="$WORK/deps" +if [ ! -f "$DEPS/lib/libz.a" ]; then + mkdir -p "$DEPS/lib" "$DEPS/include" + fetch "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz" zlib.tar.gz + tar xzf zlib.tar.gz + ( + cd "zlib-${ZLIB_VERSION}" + for f in adler32 crc32 deflate infback inffast inflate inftrees trees zutil compress uncompr; do + $CC_WASI -O2 -c "$f.c" -o "$f.o" + done + "$SDK/bin/llvm-ar" rcs "$DEPS/lib/libz.a" ./*.o + cp zlib.h zconf.h "$DEPS/include/" + ) +fi +if [ ! -f "$DEPS/lib/libsqlite3.a" ]; then + fetch "https://www.sqlite.org/${SQLITE_YEAR}/${SQLITE_AMALGAMATION}.zip" sqlite.zip + unzip -oq sqlite.zip + ( + cd "$SQLITE_AMALGAMATION" + $CC_WASI -O2 -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_WAL \ + -DSQLITE_OMIT_SHARED_CACHE -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID \ + -c sqlite3.c -o sqlite3.o + "$SDK/bin/llvm-ar" rcs "$DEPS/lib/libsqlite3.a" sqlite3.o + cp sqlite3.h "$DEPS/include/" + ) +fi + +# --- CPython --------------------------------------------------------------- +SRC="$WORK/Python-${CPYTHON_VERSION}" +if [ ! -d "$SRC" ]; then + fetch "https://www.python.org/ftp/python/${CPYTHON_VERSION}/Python-${CPYTHON_VERSION}.tar.xz" cpython.tar.xz + tar xJf cpython.tar.xz +fi +HOST_BUILD="$SRC/cross-build/wasm32-wasip1" +BUILD_PY="$SRC/cross-build/$(uname -m)-pc-linux-gnu/python" +if [ ! -f "$HOST_BUILD/libpython3.14.a" ]; then + ( + cd "$SRC" + export PATH="$WT_DIR:$PATH" WASI_SDK_PATH="$SDK" + export ZLIB_CFLAGS="-I$DEPS/include" ZLIB_LIBS="-L$DEPS/lib -lz" + export LIBSQLITE3_CFLAGS="-I$DEPS/include" LIBSQLITE3_LIBS="-L$DEPS/lib -lsqlite3" + python3 Tools/wasm/wasi build --quiet -- --disable-test-modules + ) +fi +if [ ! -x "$BUILD_PY" ]; then + BUILD_PY="$(ls "$SRC"/cross-build/*-linux-gnu/python | head -1)" +fi + +# --- guest link: bashkit_main.c replaces Programs/python.o ------------------ +$CC_WASI -O2 -I"$SRC/Include" -I"$HOST_BUILD" -c "$HERE/bashkit_main.c" -o "$WORK/bashkit_main.o" +( + cd "$HOST_BUILD" + touch Programs/python.o + link="$(make -n python.wasm 2>/dev/null | grep -- '-o python.wasm')" + link="${link/Programs\/python.o/$WORK/bashkit_main.o}" + link="${link/-o python.wasm/-o $WORK/bashkit_raw.wasm -mexec-model=reactor}" + eval "$link" +) + +# --- stdlib zip (sources only) ---------------------------------------------- +ROOT="$WORK/root" +rm -rf "$ROOT" "$WORK/stdlib" +mkdir -p "$ROOT/usr/local/lib" "$WORK/stdlib" +cp -r "$SRC/Lib/." "$WORK/stdlib/" +cp "$HOST_BUILD"/build/lib.wasi-wasm32-3.14/_sysconfigdata_*.py "$WORK/stdlib/" +cp "$HERE/_bashkit_boot.py" "$WORK/stdlib/" +( + cd "$WORK/stdlib" + # Not usable or not useful in a sandboxed, single-threaded, headless guest. + rm -rf test idlelib tkinter turtledemo ensurepip venv pydoc_data turtle.py \ + _pyrepl/__pycache__ curses dbm/gnu.py dbm/ndbm.py multiprocessing concurrent/futures/process.py + find . -name __pycache__ -prune -exec rm -rf {} + + find . -type d -name tests -prune -exec rm -rf {} + + # Deterministic zip: sorted entries, fixed timestamps. + "$BUILD_PY" -I - "$ROOT/usr/local/lib/python314.zip" <<'PY' +import os, sys, zipfile +out = sys.argv[1] +files = [] +for dirpath, dirnames, filenames in os.walk("."): + dirnames.sort() + for name in sorted(filenames): + if name.endswith(".py"): + files.append(os.path.join(dirpath, name)[2:]) +with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z: + for path in files: + info = zipfile.ZipInfo(path, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + with open(path, "rb") as f: + z.writestr(info, f.read(), compresslevel=9) +PY +) + +# --- snapshot ---------------------------------------------------------------- +"$WASMTIME" wizer -S cli --dir "$ROOT::/" -o "$WORK/bashkit_wizer.wasm" "$WORK/bashkit_raw.wasm" +"$SDK/bin/llvm-strip" -o "$WORK/python.wasm" "$WORK/bashkit_wizer.wasm" + +# Smoke test the snapshot with the reference runtime before committing it. +out="$("$WASMTIME" run -S cli --dir "$ROOT::/" --env PWD=/ --invoke bashkit_run \ + "$WORK/python.wasm" -c 'import json, re, csv, sqlite3, zlib; print(json.dumps({"ok": 1}))' 2>/dev/null)" +case "$out" in +*'{"ok": 1}'*) ;; +*) echo "snapshot smoke test failed: $out" >&2 && exit 1 ;; +esac + +mkdir -p "$CRATE/artifacts" +gzip -9 -n -c "$WORK/python.wasm" >"$CRATE/artifacts/python.wasm.gz" +cp "$ROOT/usr/local/lib/python314.zip" "$CRATE/artifacts/python314.zip" +( + cd "$CRATE/artifacts" + { + echo "cpython=${CPYTHON_VERSION}" + echo "wasi_sdk=${WASI_SDK_MAJOR}" + echo "wasmtime_wizer=${WASMTIME_VERSION}" + echo "zlib=${ZLIB_VERSION}" + echo "sqlite=${SQLITE_AMALGAMATION}" + sha256sum python.wasm.gz python314.zip + } >MANIFEST +) +ls -la "$CRATE/artifacts" diff --git a/crates/bashkit-cpython-wasm/src/config.rs b/crates/bashkit-cpython-wasm/src/config.rs new file mode 100644 index 000000000..200c79c9e --- /dev/null +++ b/crates/bashkit-cpython-wasm/src/config.rs @@ -0,0 +1,27 @@ +// Shared by build.rs (via `include!`) and the runtime so the precompiled +// module and the loading engine always agree on compile-affecting settings. + +/// Wasmtime configuration the embedded module is compiled for. +/// +/// Decisions: +/// - `pulley64`: portable interpreter bytecode, one artifact for every 64-bit +/// little-endian host; no native code generation or executable memory. +/// - Fuel: lets the host meter guest work and yield cooperatively. +/// - Copy-on-write memory init with a dense image: the snapshot's ~40 MB +/// linear memory is mapped, not copied, per instance. +/// - 1 GiB memory reservation, 64 KiB guard: see below. +pub fn engine_config() -> wasmtime::Config { + let mut config = wasmtime::Config::new(); + config + .target("pulley64") + .expect("pulley64 target is always available with the `pulley` feature"); + config.consume_fuel(true); + config.memory_init_cow(true); + config.memory_guaranteed_dense_image_size(128 << 20); + // Pulley checks every memory access explicitly, so large virtual + // reservations and guard regions buy nothing. Small ones keep many + // concurrent instances (and pooled slots) cheap in address space. + config.memory_reservation(1 << 30); + config.memory_guard_size(64 << 10); + config +} diff --git a/crates/bashkit-cpython-wasm/src/lib.rs b/crates/bashkit-cpython-wasm/src/lib.rs new file mode 100644 index 000000000..51d4e911c --- /dev/null +++ b/crates/bashkit-cpython-wasm/src/lib.rs @@ -0,0 +1,102 @@ +//! CPython 3.14 compiled to `wasm32-wasip1`, pre-initialized with Wizer and +//! precompiled to Wasmtime's Pulley bytecode at build time. +//! +//! This crate only ships bytes and the matching engine configuration. The +//! WASI host that maps the guest onto a virtual filesystem lives in +//! `bashkit` (`cpython` feature). See `knowledge/runtimes/cpython-wasm.md`. +//! +//! # Guest contract +//! +//! - Export `bashkit_run() -> i32` runs one `python3` invocation and returns +//! its exit status. Argv, environment (including `PWD`) and stdio come from +//! WASI preview1 imports. `proc_exit` may also end the call. +//! - The snapshot was taken with exactly one preopened directory, `/`, at +//! file descriptor 3. Hosts must present the same preopen. +//! - The stdlib must be readable at [`STDLIB_ZIP_PATH`] with exactly the +//! bytes of [`STDLIB_ZIP`]; the snapshot caches its zip directory. +//! - Each instance must serve a single call; state is never reused. + +include!("config.rs"); + +/// CPython version inside the guest. +pub const PYTHON_VERSION: &str = "3.14.8"; + +/// Guest path the stdlib zip must be served at. +pub const STDLIB_ZIP_PATH: &str = "/usr/local/lib/python314.zip"; + +/// Guest directory that holds the stdlib (read-only to the guest). +pub const STDLIB_PREFIX: &str = "/usr/local/lib"; + +/// Stdlib sources, deflate-compressed zip read by CPython's `zipimport`. +pub static STDLIB_ZIP: &[u8] = include_bytes!("../artifacts/python314.zip"); + +// Wasmtime maps the module's data segments copy-on-write straight from these +// bytes, which requires host-page alignment. 64 KiB covers every supported +// page size (4 KiB x86-64, 16 KiB Apple silicon, 64 KiB some aarch64 Linux). +#[repr(C, align(65536))] +struct Aligned(T); + +static CWASM: &Aligned<[u8]> = &Aligned(*include_bytes!(concat!(env!("OUT_DIR"), "/python.cwasm"))); + +/// Precompiled Pulley module bytes (for diagnostics and size reporting). +pub fn cwasm_bytes() -> &'static [u8] { + &CWASM.0 +} + +/// Create an engine compatible with the embedded module. +pub fn engine() -> wasmtime::Result { + wasmtime::Engine::new(&engine_config()) +} + +/// Load the embedded CPython module into `engine`. +/// +/// `engine` must come from [`engine`] or use [`engine_config`] unchanged +/// (plus runtime-only settings). +/// +/// The first load in a process maps the embedded bytes in place (no copy). +/// Wasmtime registers code by address, so the same bytes cannot back two live +/// modules; every later load copies them instead (tens of milliseconds). +/// Hosts should load once and share the module. +pub fn load_module(engine: &wasmtime::Engine) -> wasmtime::Result { + static MAPPED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + let bytes: &'static [u8] = &CWASM.0; + if MAPPED.swap(true, std::sync::atomic::Ordering::AcqRel) { + // SAFETY: same provenance as below; `deserialize` copies the bytes. + return unsafe { wasmtime::Module::deserialize(engine, bytes) }; + } + // SAFETY: the bytes were produced by `Engine::precompile_module` in this + // crate's build script with the same wasmtime version and `engine_config`, + // live in immutable static memory for the life of the process, and are + // mapped by at most one module (guarded by `MAPPED`). + unsafe { wasmtime::Module::deserialize_raw(engine, std::ptr::NonNull::from(bytes)) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn module_loads_and_exports_entrypoint() { + let engine = engine().unwrap(); + let module = load_module(&engine).unwrap(); + assert!(module.get_export("bashkit_run").is_some()); + assert!(module.get_export("memory").is_some()); + } + + #[test] + fn repeated_loads_coexist() { + let a = load_module(&engine().unwrap()).unwrap(); + let b = load_module(&engine().unwrap()).unwrap(); + assert!(a.get_export("bashkit_run").is_some() && b.get_export("bashkit_run").is_some()); + } + + #[test] + fn module_bytes_are_page_aligned() { + assert_eq!(cwasm_bytes().as_ptr() as usize % 65536, 0); + } + + #[test] + fn stdlib_zip_is_a_zip() { + assert_eq!(&STDLIB_ZIP[..4], b"PK\x03\x04"); + } +} diff --git a/crates/bashkit/Cargo.toml b/crates/bashkit/Cargo.toml index a53823537..d8ad1ca3f 100644 --- a/crates/bashkit/Cargo.toml +++ b/crates/bashkit/Cargo.toml @@ -138,6 +138,11 @@ os_display = "0.1.3" # enforcement during an uninterrupted VM loop. # See knowledge/runtimes/python-builtin.md § Upgrade blocker. monty = { version = "=0.0.19", optional = true } + +# Real CPython 3.14 as a WASI guest, precompiled to Pulley (cpython feature). +# See knowledge/runtimes/cpython-wasm.md. +bashkit-cpython-wasm = { path = "../bashkit-cpython-wasm", version = "0.18.2", optional = true } +wasmtime = { workspace = true, optional = true, features = ["pooling-allocator"] } monty-types = { version = "=0.0.19", optional = true } # Monty's ruff AST uses compact_str 0.9; get-size2 >=0.10.2 only implements # GetSize for compact_str 0.10. Enforce downstream too, not only in Cargo.lock. @@ -256,6 +261,9 @@ ssh = ["russh"] # Enable python/python3 builtins via embedded Monty interpreter # Usage: cargo build --features python python = ["dep:monty", "dep:monty-types", "dep:get-size2"] +# Enable python/python3 builtins via real CPython 3.14 in a WebAssembly sandbox +# Usage: cargo build --features cpython +cpython = ["dep:bashkit-cpython-wasm", "dep:wasmtime"] # Enable ts/node/deno/bun builtins via embedded ZapCode TypeScript interpreter # Usage: cargo build --features typescript typescript = ["dep:zapcode-core"] @@ -331,6 +339,11 @@ name = "sqlite" harness = false required-features = ["sqlite"] +[[bench]] +name = "python" +harness = false +required-features = ["python", "cpython"] + [[example]] name = "terminal_vi" required-features = ["terminal"] @@ -363,6 +376,18 @@ required-features = ["python"] name = "python_external_functions" required-features = ["python"] +[[example]] +name = "python_startup" +required-features = ["python", "cpython"] + +[[example]] +name = "cpython_scripts" +required-features = ["cpython"] + +[[example]] +name = "cpython_load" +required-features = ["cpython"] + [[example]] name = "typescript_scripts" required-features = ["typescript"] diff --git a/crates/bashkit/benches/python.rs b/crates/bashkit/benches/python.rs new file mode 100644 index 000000000..0fcd83001 --- /dev/null +++ b/crates/bashkit/benches/python.rs @@ -0,0 +1,171 @@ +//! Python runtime benchmark: CPython (WASI, `cpython` feature) vs Monty +//! (`python` feature). +//! +//! Dimensions: +//! - **Raw start**: the first `python3` in a fresh OS process (Monty has no +//! load step; CPython creates a wasmtime engine and maps the precompiled +//! module). One-time per process, so it is measured by the `python_startup` +//! example in new processes (`scripts/bench-python.sh`), not here. +//! - **Per call** (`python_call`): one `python3` invocation in a warm +//! process, through the bashkit interpreter (parse, dispatch, instance, +//! run, teardown) — what a script author observes. +//! - **Per session** (`python_session`): a new `Bash` per call, the +//! multi-tenant request shape. +//! - **Compute** (`python_compute`): CPU-bound Python (interpreter speed). +//! - **Parallel** (`python_parallel`): N concurrent sessions, each one +//! `python3` call, on a multi-thread runtime. +//! +//! Run: `cargo bench --bench python --features python,cpython` or +//! `just bench-python` (saves results). + +use bashkit::{Bash, CPython, SessionLimits}; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use tokio::runtime::Runtime; + +const RUNTIMES: &[&str] = &["monty", "cpython"]; +const PARALLEL: &[usize] = &[1, 16, 64]; + +fn make_bash(runtime: &str) -> Bash { + // Warm sessions run thousands of calls; lift the per-session exec cap. + let builder = Bash::builder().session_limits(SessionLimits::unlimited()); + match runtime { + "monty" => builder + .python() + .env("BASHKIT_ALLOW_INPROCESS_PYTHON", "1") + .build(), + "cpython" => builder.cpython().build(), + _ => unreachable!(), + } +} + +/// Scripts both runtimes accept, with identical output. +const CALLS: &[(&str, &str)] = &[ + ("pass", "python3 -c 'pass'"), + ("print", "python3 -c 'print(1)'"), + ( + "json_roundtrip", + "python3 -c 'import json; print(json.dumps(json.loads(\"{\\\"a\\\": [1, 2, 3]}\")))'", + ), + ( + "file_write_read", + "python3 -c 'from pathlib import Path; p = Path(\"/tmp/b.txt\"); p.write_text(\"x\" * 1000); print(len(p.read_text()))'", + ), +]; + +const COMPUTE: &[(&str, &str)] = &[ + ( + "loop_sum_1e5", + "python3 -c 'print(sum(i * i for i in range(100000)))'", + ), + ( + "fib_20", + "python3 -c 'def fib(n):\n return n if n < 2 else fib(n - 1) + fib(n - 2)\nprint(fib(20))'", + ), + ( + "string_build_1e4", + "python3 -c 'print(len(\",\".join(str(i) for i in range(10000))))'", + ), +]; + +fn verify(rt: &Runtime) { + // Fail fast if a runtime cannot run the workloads, instead of timing errors. + for runtime in RUNTIMES { + for (name, script) in CALLS.iter().chain(COMPUTE) { + let r = rt.block_on(async { make_bash(runtime).exec(script).await.unwrap() }); + assert_eq!(r.exit_code, 0, "{runtime}/{name}: {}", r.stderr); + } + } +} + +fn bench_call(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + CPython::warm_up().unwrap(); + verify(&rt); + let mut group = c.benchmark_group("python_call"); + for runtime in RUNTIMES { + // One long-lived session; the lock is uncontended (iterations are sequential). + let bash = tokio::sync::Mutex::new(make_bash(runtime)); + for (name, script) in CALLS { + group.bench_with_input(BenchmarkId::new(*runtime, name), script, |b, script| { + b.to_async(&rt) + .iter(|| async { bash.lock().await.exec(script).await.unwrap() }); + }); + } + } + group.finish(); +} + +fn bench_session(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + CPython::warm_up().unwrap(); + let mut group = c.benchmark_group("python_session"); + for runtime in RUNTIMES { + group.bench_function(BenchmarkId::new(*runtime, "new_bash_print"), |b| { + b.to_async(&rt).iter(|| async { + make_bash(runtime) + .exec("python3 -c 'print(1)'") + .await + .unwrap() + }); + }); + } + group.finish(); +} + +fn bench_compute(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + CPython::warm_up().unwrap(); + let mut group = c.benchmark_group("python_compute"); + group.sample_size(10); + for runtime in RUNTIMES { + for (name, script) in COMPUTE { + group.bench_with_input(BenchmarkId::new(*runtime, name), script, |b, script| { + b.to_async(&rt) + .iter(|| async { make_bash(runtime).exec(script).await.unwrap() }); + }); + } + } + group.finish(); +} + +fn bench_parallel(c: &mut Criterion) { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + CPython::warm_up().unwrap(); + let mut group = c.benchmark_group("python_parallel"); + group.sample_size(10); + for runtime in RUNTIMES { + for &n in PARALLEL { + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::new(*runtime, n), &n, |b, &n| { + b.to_async(&rt).iter(|| async move { + let tasks: Vec<_> = (0..n) + .map(|i| { + let runtime = *runtime; + tokio::spawn(async move { + let script = format!("python3 -c 'print({i} * 2)'"); + let r = make_bash(runtime).exec(&script).await.unwrap(); + assert_eq!(r.stdout, format!("{}\n", i * 2)); + }) + }) + .collect(); + for t in tasks { + t.await.unwrap(); + } + }); + }); + } + } + group.finish(); +} + +criterion_group!( + benches, + bench_call, + bench_session, + bench_compute, + bench_parallel +); +criterion_main!(benches); diff --git a/crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md b/crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md new file mode 100644 index 000000000..61f1ca8d7 --- /dev/null +++ b/crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md @@ -0,0 +1,79 @@ +# Python Runtimes Benchmark: CPython (wasm) vs Monty + +`python3` through the bashkit interpreter, end to end. CPython 3.14 runs as a +WebAssembly guest on wasmtime's Pulley interpreter, from a pre-initialized +snapshot; Monty runs natively in-process. See +knowledge/runtimes/cpython-wasm.md. + +## System Information + +- **Moniker**: `vm-linux-x86_64` +- **Hostname**: vm +- **OS**: linux +- **Architecture**: x86_64 +- **CPUs**: 4 +- **Timestamp**: 1791319327 + +## Raw start (fresh OS process, 10 runs) + +First `python3 -c 'print(1)'` in a new process, timed from `main` +(includes building `Bash` and any one-time runtime load). + +| Runtime | Measure | Min | Median | Max | +|---------|---------|-----|--------|-----| +| monty | first call (fresh process) | 0.330 ms | 0.369 ms | 0.595 ms | +| monty | second call | 0.035 ms | 0.038 ms | 0.069 ms | +| cpython | first call (fresh process) | 20.947 ms | 21.896 ms | 27.749 ms | +| cpython | second call | 5.682 ms | 6.239 ms | 7.849 ms | + +## Per call (warm session) + +| Benchmark | Time | +|-----------|------| +| python_call/monty/pass | 16.872 µs | +| python_call/monty/print | 15.414 µs | +| python_call/monty/json_roundtrip | 26.897 µs | +| python_call/monty/file_write_read | 41.716 µs | +| python_call/cpython/pass | 5.4560 ms | +| python_call/cpython/print | 6.3116 ms | +| python_call/cpython/json_roundtrip | 8.9680 ms | +| python_call/cpython/file_write_read | 13.095 ms | + +## Per session (new Bash per call) + +| Benchmark | Time | +|-----------|------| +| python_session/monty/new_bash_print | 68.699 µs | +| python_session/cpython/new_bash_print | 6.4656 ms | + +## Compute (CPU-bound Python) + +| Benchmark | Time | +|-----------|------| +| python_compute/monty/loop_sum_1e5 | 57.185 ms | +| python_compute/monty/fib_20 | 17.725 ms | +| python_compute/monty/string_build_1e4 | 7.6934 ms | +| python_compute/cpython/loop_sum_1e5 | 1.5575 s | +| python_compute/cpython/fib_20 | 76.010 ms | +| python_compute/cpython/string_build_1e4 | 230.49 ms | + +## Parallel sessions (N concurrent, one call each) + +| Benchmark | Time | +|-----------|------| +| python_parallel/monty/1 | 149.51 µs | +| python_parallel/monty/16 | 457.04 µs | +| python_parallel/monty/64 | 1.3613 ms | +| python_parallel/cpython/1 | 7.3690 ms | +| python_parallel/cpython/16 | 72.834 ms | +| python_parallel/cpython/64 | 290.82 ms | + +## Load (CPython, concurrent tenants, 4 calls each, one process) + +| Tenants | Calls | Wall (s) | Calls/s | p50 (ms) | p99 (ms) | max (ms) | Failures | Peak RSS (MB) | +|---------|-------|----------|---------|----------|----------|----------|----------|---------------| +| 1 | 4 | 0.09 | 42 | 19.66 | 36.68 | 36.68 | 0 | 50 | +| 16 | 64 | 0.74 | 87 | 163.71 | 281.40 | 286.94 | 0 | 182 | +| 64 | 256 | 2.76 | 93 | 548.42 | 935.22 | 937.04 | 0 | 619 | +| 256 | 1024 | 11.52 | 89 | 2517.48 | 4021.28 | 4022.99 | 0 | 2431 | +| 1024 | 4096 | 44.60 | 92 | 10100.95 | 14775.18 | 14933.32 | 0 | 4688 | diff --git a/crates/bashkit/docs/cpython.md b/crates/bashkit/docs/cpython.md new file mode 100644 index 000000000..0763c4944 --- /dev/null +++ b/crates/bashkit/docs/cpython.md @@ -0,0 +1,155 @@ +# Embedded CPython (WebAssembly) + +Bashkit can run real CPython 3.14 as `python`/`python3`. The interpreter is +compiled to WebAssembly (`wasm32-wasip1`), pre-initialized into a snapshot and +shipped inside the binary, so there is nothing to install and no network +access at build or run time. Each `python3` call runs in a fresh, isolated +instance that can only reach the Bashkit virtual filesystem and its own +stdio. + +**See also:** +- [Embedded Python (Monty)](./python.md) - The lighter, Rust-native alternative +- [Threat Model](./threat-model.md) - Security considerations (TM-PY-CPY-*) +- [Compatibility Reference](./compatibility.md) - Bash feature support +- [`knowledge/runtimes/cpython-wasm.md`][spec] - Design, measurements and decisions + +## Quick start + +Enable the `cpython` cargo feature and register the builtins: + +```toml +[dependencies] +bashkit = { version = "0.18.2", features = ["cpython"] } +``` + +```rust +use bashkit::Bash; + +# #[tokio::main] +# async fn main() -> bashkit::Result<()> { +let mut bash = Bash::builder().cpython().build(); + +let r = bash + .exec("python3 -c 'import json, sys; print(json.dumps({\"v\": sys.version_info[:2]}))'") + .await?; +assert_eq!(r.stdout, "{\"v\": [3, 14]}\n"); +# Ok(()) +# } +``` + +No runtime opt-in variable is needed (unlike Monty's +`BASHKIT_ALLOW_INPROCESS_PYTHON`): calling `.cpython()` is the opt-in, because +the guest never runs native code in your process. + +The interpreter loads on the first call of a process: the first `python3` +takes about 22 ms on the reference machine, later calls about 6 ms. To move +that one-time cost out of the first request, call +`bashkit::CPython::warm_up()` at startup. + +## Why CPython instead of Monty + +[Monty](./python.md) is a Python *subset* written in Rust and designed for +code-mode execution: evaluate a snippet, call back into host functions, +return a value. Scripts written by people and agents expect a Python +*command line* and the standard library behind it. CPython provides that: + +| | Monty (`python` feature) | CPython (`cpython` feature) | +|---|---|---| +| Language | Subset (no classes, limited stdlib) | Full Python 3.14 | +| Stdlib | `math`, `pathlib`, `os.getenv`, `sys`, `typing`, ... | Full pure-Python stdlib plus `json`, `re`, `csv`, `sqlite3`, `zlib`, `hashlib`, `decimal`, `datetime`, `asyncio`, ... | +| CLI | `-c`, file, `-` | `-c`, `-m`, file, directory with `__main__.py`, `-`, stdin, `-x`, `-W`, `-V`, `-h` | +| Errors | Monty-specific text | CPython tracebacks, exit codes, `sys.exit` semantics | +| Isolation | In-process Rust interpreter | WebAssembly sandbox (memory-safe boundary), fresh instance per call | +| Start per call | ~15 µs | ~5-6 ms (first call in a process ~22 ms) | +| CPU-bound speed | Native | ~4-30x slower than Monty (interpreted wasm) | +| Host callbacks | Yes (external functions) | Not yet | + +Pick CPython when scripts need real Python behavior; pick Monty when you +need host function callbacks or microsecond start-up for tiny snippets. +Both can be compiled in: the builder method called last owns +`python`/`python3`. + +## What works + +- `python3 -c CODE`, `python3 FILE`, `python3 -m MODULE`, `python3 DIR` + (runs `DIR/__main__.py`), `python3 -` and programs piped on stdin. +- `sys.argv`, `sys.exit`, uncaught exceptions (exit 1, traceback on stderr), + `os._exit`, `atexit`, `input()`, binary stdio via `sys.stdout.buffer`. +- Exported shell variables in `os.environ`; `PWD` becomes the working + directory, so relative paths resolve like in a real shell. +- `open()`, `os`, `pathlib`, `shutil`, `glob`, `tempfile`, `sqlite3` database + files, `gzip`/`zipfile`/`tarfile` on the virtual filesystem. Files the + script leaves open are flushed when it exits. +- Local modules next to the script or in the working directory import as + usual. +- `asyncio` (single-threaded event loop, timers, queues, gather). + +## Limits + +```rust +use bashkit::{Bash, CPythonLimits}; +use std::time::Duration; + +let bash = Bash::builder() + .cpython_with_limits( + CPythonLimits::default() + .max_duration(Duration::from_secs(5)) // wall clock per call (default 30 s) + .max_memory(128 * 1024 * 1024) // guest memory (default 256 MB, max 1 GB) + .max_recursion(500) // sys.getrecursionlimit() (default 1000) + .max_output(1024 * 1024), // stdout + stderr bytes (default 16 MB) + ) + .build(); +``` + +- **Time**: a call past its deadline stops with exit code 124 and + `python3: execution timed out ...`. A tighter Bashkit `ExecutionLimits` + timeout or cancellation wins. +- **Memory**: allocation past the cap raises `MemoryError` inside Python. +- **Output**: bytes past the cap are dropped and stderr ends with + `python3: output truncated at N bytes`. +- **Files**: Bashkit filesystem limits (file size, total bytes, file count) + apply; violations raise `OSError`. +- **Concurrency**: up to 512 CPython calls run at once per process; further + calls wait for a free slot within their own deadline. + +## Limitations + +- **No subprocesses**: `subprocess`, `os.system`, `os.fork`, `os.popen` raise + `OSError`/`AttributeError`. Python cannot call back into the shell yet. +- **No network**: `socket`, `urllib.request`, `http.client` cannot connect. + Use the shell's `curl`/`http` builtins. +- **No threads**: `threading.Thread.start()` raises `RuntimeError`; + `multiprocessing` and `concurrent.futures.ProcessPoolExecutor` are absent. + `asyncio` works. +- **No native extensions or pip**: only the bundled stdlib. `ctypes`, + `numpy`, `requests` and other third-party packages are unavailable; `ssl`, + `_hashlib` (OpenSSL), `tkinter`, `curses`, `readline`, `dbm.gnu` are not + built. `hashlib` still provides md5, sha1, sha2, sha3 and blake2. +- **No interactive mode**: `python3` with no program reads one from stdin; + there is no REPL. +- **Symlinks are not followed**, like everywhere in the Bashkit VFS. +- **`errno` numbers are WASI's** (`ENOENT` is 44, not 2). Exception types + (`FileNotFoundError`, ...) and messages are correct; code comparing + `e.errno == errno.ENOENT` works because the `errno` module matches. +- **Fixed hash seed**: `hash()` of `str`/`bytes` is the same in every call + (the seed is baked into the snapshot). `random` is re-seeded per call. +- **Deep C-level recursion** (for example `repr` of a list nested 100 000 + levels) ends the call with `python3: fatal error: stack overflow in the + interpreter` instead of `RecursionError`. +- **CPU-bound code is slow**: the guest runs on Wasmtime's portable Pulley + interpreter, roughly 4-30x slower than Monty and far slower than native + CPython. Start-up, not throughput, is what this runtime is tuned for. +- **No host callbacks** (Monty's external functions) or `ToolDef` + integration yet. +- Interpreter-start options (`-E`, `-I`, `-s`, `-S`, `-B`, `-u`, `-O`, `-q`, + `-X ...`) are accepted and ignored. + +## Binary size + +The `cpython` feature adds about 45 MB to a binary: the precompiled +interpreter snapshot (~41 MB, mostly the pre-initialized 40 MB heap image so +it can be mapped copy-on-write) and the zipped stdlib (~2.5 MB) are embedded, +plus the Wasmtime runtime. Pages are mapped on demand, so resident memory per +process is far smaller. + +[spec]: https://github.com/everruns/bashkit/blob/main/knowledge/runtimes/cpython-wasm.md diff --git a/crates/bashkit/docs/python.md b/crates/bashkit/docs/python.md index 4742b74f9..efd5f6b3c 100644 --- a/crates/bashkit/docs/python.md +++ b/crates/bashkit/docs/python.md @@ -9,6 +9,7 @@ a pure-Rust implementation of Python 3.12. Python runs entirely in-memory with configurable resource limits and no host access. **See also:** +- [Embedded CPython (WebAssembly)](./cpython.md) - Full CPython 3.14 in a wasm sandbox (`cpython` feature) - [Threat Model](./threat-model.md) - Security considerations (TM-PY-*) - [Custom Builtins](./custom_builtins.md) - Writing your own builtins - [Compatibility Reference](./compatibility.md) - Bash feature support diff --git a/crates/bashkit/examples/cpython_load.rs b/crates/bashkit/examples/cpython_load.rs new file mode 100644 index 000000000..ad696dc89 --- /dev/null +++ b/crates/bashkit/examples/cpython_load.rs @@ -0,0 +1,113 @@ +//! Load test: many concurrent CPython (wasm) sessions in one process. +//! +//! For each concurrency level N, starts N tenants at once on a multi-thread +//! runtime. Every tenant gets its own `Bash` (own VFS) and runs 4 (or +//! `CPYTHON_LOAD_CALLS`) +//! `python3` invocations that write, read back and verify tenant-specific +//! data, so cross-tenant leakage or a wrong answer counts as a failure. +//! Reports throughput, per-call latency percentiles and peak RSS. +//! +//! Run: cargo run --release --example cpython_load --features cpython -- 1 16 64 256 1024 + +use bashkit::{Bash, CPython}; +use std::time::{Duration, Instant}; + +/// Calls per tenant; override with `CPYTHON_LOAD_CALLS`. +fn calls() -> usize { + std::env::var("CPYTHON_LOAD_CALLS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(4) +} + +async fn tenant(id: usize) -> Result, String> { + let mut bash = Bash::builder().cpython().build(); + let mut latencies = Vec::with_capacity(calls()); + for call in 0..calls() { + let script = format!( + "python3 -c 'import json, os\n\ + p = \"/state.json\"\n\ + s = json.load(open(p)) if os.path.exists(p) else {{\"id\": {id}, \"n\": 0}}\n\ + assert s[\"id\"] == {id}, s\n\ + s[\"n\"] += 1\n\ + json.dump(s, open(p, \"w\"))\n\ + print(s[\"id\"], s[\"n\"])'" + ); + let t = Instant::now(); + let r = bash.exec(&script).await.map_err(|e| e.to_string())?; + latencies.push(t.elapsed()); + let want = format!("{id} {}\n", call + 1); + if r.exit_code != 0 || r.stdout != want.as_str() { + return Err(format!( + "tenant {id} call {call}: exit {} stdout {:?} stderr {:?}", + r.exit_code, + r.stdout.to_string(), + r.stderr.to_string() + )); + } + } + Ok(latencies) +} + +fn peak_rss_mb() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|l| l.starts_with("VmHWM:"))?; + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / 1024.0) +} + +fn pct(sorted: &[Duration], p: f64) -> f64 { + let idx = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[idx].as_secs_f64() * 1e3 +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let levels: Vec = std::env::args() + .skip(1) + .map(|a| a.parse().expect("concurrency level")) + .collect(); + let levels = if levels.is_empty() { + vec![1, 16, 64, 256] + } else { + levels + }; + CPython::warm_up().expect("load CPython"); + println!( + "| Tenants | Calls | Wall (s) | Calls/s | p50 (ms) | p99 (ms) | max (ms) | Failures | Peak RSS (MB) |" + ); + println!( + "|---------|-------|----------|---------|----------|----------|----------|----------|---------------|" + ); + let mut failed = false; + for n in levels { + let start = Instant::now(); + let handles: Vec<_> = (0..n).map(|id| tokio::spawn(tenant(id))).collect(); + let mut all = Vec::with_capacity(n * calls()); + let mut failures = 0; + for h in handles { + match h.await.expect("tenant task") { + Ok(l) => all.extend(l), + Err(e) => { + failures += 1; + eprintln!("{e}"); + } + } + } + let wall = start.elapsed().as_secs_f64(); + all.sort(); + failed |= failures > 0 || all.is_empty(); + println!( + "| {n} | {} | {wall:.2} | {:.0} | {:.2} | {:.2} | {:.2} | {failures} | {} |", + all.len(), + all.len() as f64 / wall, + pct(&all, 0.5), + pct(&all, 0.99), + pct(&all, 1.0), + peak_rss_mb().map_or("n/a".to_string(), |m| format!("{m:.0}")), + ); + } + if failed { + std::process::exit(1); + } +} diff --git a/crates/bashkit/examples/cpython_scripts.rs b/crates/bashkit/examples/cpython_scripts.rs new file mode 100644 index 000000000..3357bf754 --- /dev/null +++ b/crates/bashkit/examples/cpython_scripts.rs @@ -0,0 +1,67 @@ +//! CPython Scripts Example +//! +//! Runs real CPython 3.14 (compiled to WebAssembly) as `python3` inside +//! Bashkit. The interpreter only sees the virtual filesystem and captured +//! stdio; every call starts from a fresh pre-initialized snapshot. +//! +//! Run with: cargo run --features cpython --example cpython_scripts + +use bashkit::{Bash, CPython, CPythonLimits}; +use std::time::Duration; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Optional: load the interpreter at startup instead of on the first call. + CPython::warm_up().map_err(anyhow::Error::msg)?; + + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_duration(Duration::from_secs(10))) + .build(); + + // Classes, dataclasses and the stdlib work as in regular CPython. + let r = bash + .exec( + r#"python3 - <<'EOF' +from dataclasses import dataclass +import json, statistics + +@dataclass +class Sample: + name: str + values: list + +s = Sample("latency", [3, 1, 4, 1, 5]) +print(json.dumps({"name": s.name, "median": statistics.median(s.values)})) +EOF"#, + ) + .await?; + print!("{}", r.stdout); + assert_eq!(r.stdout, "{\"name\": \"latency\", \"median\": 3}\n"); + + // Files written by Python are visible to the shell, and vice versa. + let r = bash + .exec( + "printf 'a,b\\n1,2\\n3,4\\n' > /data.csv + python3 -c 'import csv; rows = list(csv.DictReader(open(\"/data.csv\"))); \ + open(\"/sum.txt\", \"w\").write(str(sum(int(r[\"b\"]) for r in rows)))' + cat /sum.txt", + ) + .await?; + println!("{}", r.stdout); + assert_eq!(r.stdout, "6"); + + // Exit codes and stderr behave like a real process. + let r = bash + .exec("python3 -c 'import sys; sys.exit(\"failed\")' || echo \"exit=$?\"") + .await?; + print!("{}", r.stdout); + assert_eq!(r.stdout, "exit=1\n"); + + // No network or subprocesses inside the sandbox. + let r = bash + .exec("python3 -c 'import subprocess; subprocess.run([\"ls\"])' 2>&1 | tail -1") + .await?; + print!("{}", r.stdout); + assert!(r.stdout.contains("OSError")); + Ok(()) +} diff --git a/crates/bashkit/examples/python_startup.rs b/crates/bashkit/examples/python_startup.rs new file mode 100644 index 000000000..865d7af94 --- /dev/null +++ b/crates/bashkit/examples/python_startup.rs @@ -0,0 +1,32 @@ +//! Raw python3 start time in a fresh process: Monty vs CPython (wasm). +//! +//! Prints one line per run with the wall time of the first `python3` call in +//! this process (includes any one-time runtime load) and of a second call. +//! `scripts/bench-python.sh` runs it repeatedly, each run a new OS process. +//! +//! Run: cargo run --release --example python_startup --features python,cpython -- cpython + +use bashkit::Bash; +use std::time::Instant; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let runtime = std::env::args().nth(1).unwrap_or_else(|| "cpython".into()); + let start = Instant::now(); + let mut bash = match runtime.as_str() { + "monty" => Bash::builder() + .python() + .env("BASHKIT_ALLOW_INPROCESS_PYTHON", "1") + .build(), + "cpython" => Bash::builder().cpython().build(), + other => panic!("unknown runtime {other}; use monty or cpython"), + }; + let first = bash.exec("python3 -c 'print(1)'").await.unwrap(); + let first_ms = start.elapsed().as_secs_f64() * 1e3; + assert_eq!(first.stdout, "1\n", "{}", first.stderr); + let t = Instant::now(); + let second = bash.exec("python3 -c 'print(2)'").await.unwrap(); + let second_ms = t.elapsed().as_secs_f64() * 1e3; + assert_eq!(second.stdout, "2\n"); + println!("runtime={runtime} first_call_ms={first_ms:.3} second_call_ms={second_ms:.3}"); +} diff --git a/crates/bashkit/fuzz/Cargo.lock b/crates/bashkit/fuzz/Cargo.lock index 7f80df7d9..45f97fc82 100644 --- a/crates/bashkit/fuzz/Cargo.lock +++ b/crates/bashkit/fuzz/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -17,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.6" @@ -80,6 +95,7 @@ dependencies = [ "anyhow", "async-trait", "base64 0.23.1", + "bashkit-cpython-wasm", "bigdecimal", "bitflags 2.13.1", "bstr", @@ -111,12 +127,21 @@ dependencies = [ "serde_json", "serde_yaml_ng", "sha1", - "sha2", + "sha2 0.11.0", "thiserror", "tokio", "tower", "unit-prefix", "url", + "wasmtime", +] + +[[package]] +name = "bashkit-cpython-wasm" +version = "0.18.2" +dependencies = [ + "flate2", + "wasmtime", ] [[package]] @@ -168,6 +193,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -193,6 +227,9 @@ name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +dependencies = [ + "allocator-api2", +] [[package]] name = "bytes" @@ -292,6 +329,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -304,6 +350,24 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpp_demangle" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -313,6 +377,152 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-assembler-x64" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfbb8f30b72ef5cedf2dbd85ee2f167a33c888d492a40e999250187138663c9" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5715fdd9d897798ba2bb834df5d6c1129319d51e82ddcf81812c23231e9b852b" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a266da98b248dd98dfc8a46d45d57810db04a55b81a6d8c7a9c6cf0bd2db7c3" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67c1887ed2194ef3ecc283a9add4fe8a79d249a293063046f2adedc31ae7200" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90103abeafa22cc578c577999bdf1b51c32b8a59e15a09789841b4e9a854588d" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.17.1", + "libm", + "log", + "postcard", + "pulley-interpreter", + "regalloc2", + "rustc-hash", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d73eaff808bc302e2899e42d822c0e64df23f38facb98ecdf695d1817d2063a3" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de19c4fc55ac3cdeb8e153f35ef9f51db197da91a64c9811d0ddd21ed1b9d18d" + +[[package]] +name = "cranelift-control" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbd919579d74a7212e6e849dc24ff015cb823e70051d01559eea828efaf773f" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af201af93fa8a576ac4b04b901ac3f252545d7b703053c922526fce28f65a7be" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69395ceee43706b93bf869b3a6631ab8d9dcb2bf19f9af4d1d2db0cb9e18af18" +dependencies = [ + "cranelift-codegen", + "hashbrown 0.17.1", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d98a81f71e2ddc61849ca1454ff424fbf42a3ee5529365be8c0859ddf5b9a" + +[[package]] +name = "cranelift-native" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c79e0265b754f67ff76c42d6db4856dabf9101e3bb1b23c83255e2438dd02" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.135.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" + [[package]] name = "crc32fast" version = "1.5.0" @@ -322,6 +532,41 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -371,15 +616,25 @@ dependencies = [ "thiserror", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -400,12 +655,40 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fancy-regex" version = "0.19.0" @@ -433,11 +716,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -448,12 +737,48 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + [[package]] name = "futures-task" version = "0.3.34" @@ -466,12 +791,26 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -494,11 +833,34 @@ dependencies = [ "r-efi", ] +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap", + "stable_deref_trait", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", + "serde", + "serde_core", +] [[package]] name = "heck" @@ -518,7 +880,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -636,6 +998,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.1.0" @@ -664,7 +1032,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", ] [[package]] @@ -775,6 +1154,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libbz2-rs-sys" version = "0.2.5" @@ -803,6 +1188,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -815,6 +1206,12 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "md-5" version = "0.11.0" @@ -822,7 +1219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -831,6 +1228,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memfd" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" +dependencies = [ + "rustix", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -869,6 +1275,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.1", + "indexmap", + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -929,6 +1347,18 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -947,6 +1377,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulley-interpreter" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc9a7073f2530aab972ac0a7ce43624ff59de8ad06b76a4ef4021f7ddd1035a" +dependencies = [ + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", +] + +[[package]] +name = "pulley-macros" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864f92c9538de48325a4f8709ec83e56471446798d48a29073ca094c5447b549" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "quote" version = "1.0.47" @@ -962,6 +1415,41 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regalloc2" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" +dependencies = [ + "allocator-api2", + "bumpalo", + "hashbrown 0.17.1", + "log", + "rustc-hash", + "serde", + "smallvec", +] + [[package]] name = "regex" version = "1.13.1" @@ -1008,7 +1496,32 @@ dependencies = [ "getrandom 0.2.17", "libc", "untrusted", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1064,6 +1577,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde" version = "1.0.229" @@ -1127,8 +1650,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -1138,8 +1672,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -1171,6 +1705,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "stable_deref_trait" @@ -1223,6 +1760,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "2.0.20" @@ -1367,6 +1919,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1418,6 +1976,261 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebaca0b540bdb15d195ff34fb23074255948708b8916a11a339ba4513dc61906" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1880dd9a848aa68f387966256c34ad49abb4607e3cfa35720d2723d0591cd3" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5ff8685b587722894b6f25958fc608d315b93b8d23647bfb567664e7006d757" +dependencies = [ + "bitflags 2.13.1", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wasmprinter" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b03c8f270f3f384f70f42869b5ff7f6253fe4f0aa8590f7edd332c1943e5f51" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser", +] + +[[package]] +name = "wasmtime" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94eda6b1c1d8ade526a67e4ce50cc6234d1c7af68e5d7bf13f40bc07e70daba" +dependencies = [ + "addr2line", + "async-trait", + "bitflags 2.13.1", + "bumpalo", + "cc", + "futures", + "libc", + "log", + "mach2", + "memfd", + "object", + "once_cell", + "postcard", + "pulley-interpreter", + "rayon", + "rustix", + "serde", + "serde_derive", + "smallvec", + "target-lexicon", + "wasmparser", + "wasmtime-environ", + "wasmtime-internal-component-macro", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-environ" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84ca4865ece3a37abb34c2df96c741ff4aed308d35744d84095cff79848752d4" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "hashbrown 0.17.1", + "indexmap", + "log", + "object", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasm-encoder", + "wasmparser", + "wasmprinter", + "wasmtime-internal-component-util", + "wasmtime-internal-core", +] + +[[package]] +name = "wasmtime-internal-component-macro" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae080b05e5e0356a2167c478c264e26f3fe5dbaaf1b285f8f5501301b99facb" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-internal-component-util" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f014d9ad6f6acbc449e9d7c4e7b7757b4b6b1a84a3662f66ea6da2ebed788c" + +[[package]] +name = "wasmtime-internal-core" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc0a0310fc4a3f545427c7880362657f72f838ffcfe4d3b8c864021df3b3488" +dependencies = [ + "hashbrown 0.17.1", + "libm", + "serde", +] + +[[package]] +name = "wasmtime-internal-cranelift" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6c8d2b6f8ed3a784fcd127a1936589db8051febbd228230660175e41b5751d" +dependencies = [ + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools", + "log", + "object", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-fiber" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92b558b5892ca405508103641fa3a3ec2e986b1cc679a083ed47e6bcb08ecf31" +dependencies = [ + "cc", + "libc", + "rustix", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-jit-debug" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0669ce70a8a8c92eb8ad148c1d7b9b771aab37787e9462efd75b2e2b4575406" +dependencies = [ + "cc", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa66405e95fc7b82adb195c14fa8b38f45046c3ad99a273b06262b8eb755e76" +dependencies = [ + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-unwinder" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70555458e519d6fb8790e042b4274132ab58171ae0f2a6917c3dc29ded0184d0" +dependencies = [ + "cranelift-codegen", + "log", + "object", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-internal-versioned-export-macros" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca407e3e2809c0e6568be132065095a16f184c30c6bee4157dfdc497aee10a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "wasmtime-internal-wit-bindgen" +version = "48.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aacdb42fb0b393353e402a1534e0b492a6b0fddf6528bfaec65c5f56b990c43" +dependencies = [ + "anyhow", + "bitflags 2.13.1", + "heck", + "indexmap", + "wit-component", + "wit-parser", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1486,6 +2299,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1550,6 +2372,44 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-component" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c96b3cea6838dbea55173baa56991fcb087896c321b0a770db201e5a0bd3506" +dependencies = [ + "anyhow", + "bitflags 2.13.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22672d61bfb9df46a16b46ce38522293aee1cc4749fc84a3464ce4083b3d5fd6" +dependencies = [ + "anyhow", + "hashbrown 0.17.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-ident", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/crates/bashkit/fuzz/Cargo.toml b/crates/bashkit/fuzz/Cargo.toml index 7036815ef..c683ad0db 100644 --- a/crates/bashkit/fuzz/Cargo.toml +++ b/crates/bashkit/fuzz/Cargo.toml @@ -19,6 +19,11 @@ tokio = { version = "1", features = ["rt"] } path = ".." features = ["jq"] +[features] +# CPython (wasm) python3 target; off by default so `cargo fuzz build` stays +# light. Run with: cargo +nightly fuzz run cpython_fuzz --features cpython +cpython = ["bashkit/cpython"] + # Prevent this from interfering with workspaces [workspace] members = ["."] @@ -151,3 +156,11 @@ path = "fuzz_targets/tomlq_fuzz.rs" test = false doc = false bench = false + +[[bin]] +name = "cpython_fuzz" +path = "fuzz_targets/cpython_fuzz.rs" +required-features = ["cpython"] +test = false +doc = false +bench = false diff --git a/crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs b/crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs new file mode 100644 index 000000000..fd9bb0ff1 --- /dev/null +++ b/crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs @@ -0,0 +1,62 @@ +//! Fuzz target for the CPython (wasm) python3 builtin +//! +//! Feeds arbitrary programs and command lines to `python3` to find: +//! - Host panics or aborts from guest traps, WASI calls or exit handling +//! - Sandbox escapes: host files, host environment (FUZZ_HOST_CANARY) +//! - Limit bypasses: wall clock, memory, output +//! - Internal Debug shapes leaking into stderr (TM-INF-022) +//! +//! Run with: cargo +nightly fuzz run cpython_fuzz --features cpython -- -max_total_time=300 + +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let Ok(input) = std::str::from_utf8(data) else { + return; + }; + // Keep tracebacks (which quote the source) under the stderr cap. + if input.len() > 256 || bashkit::testing::input_echo_would_trip(input) { + return; + } + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + rt.block_on(async { + bashkit::testing::fuzz_init(); + let mut bash = bashkit::Bash::builder() + .cpython_with_limits( + bashkit::CPythonLimits::default() + .max_duration(std::time::Duration::from_millis(500)) + .max_memory(128 * 1024 * 1024) + .max_output(4096), + ) + .limits( + bashkit::ExecutionLimits::new() + .max_commands(50) + .max_stdout_bytes(4096) + .max_stderr_bytes(1024) + .timeout(std::time::Duration::from_secs(2)), + ) + .build(); + + // Program from a VFS file. + let _ = bash + .fs() + .write_file(std::path::Path::new("/fuzz.py"), input.as_bytes()) + .await; + bashkit::testing::fuzz_exec(&mut bash, "python3 /fuzz.py", "cpython_fuzz", &[]).await; + + // Program from stdin. + bashkit::testing::fuzz_exec(&mut bash, "python3 < /fuzz.py", "cpython_fuzz", &[]).await; + + // Input as a command line (CLI option parsing). + let quoted = input.replace('\'', "'\\''"); + let script = format!("python3 '{quoted}' -c 'print(1)'"); + bashkit::testing::fuzz_exec(&mut bash, &script, "cpython_fuzz", &[]).await; + }); +}); diff --git a/crates/bashkit/src/builtins/cpython/mod.rs b/crates/bashkit/src/builtins/cpython/mod.rs new file mode 100644 index 000000000..01a789a6f --- /dev/null +++ b/crates/bashkit/src/builtins/cpython/mod.rs @@ -0,0 +1,444 @@ +//! `python`/`python3` builtins backed by real CPython 3.14 running as a +//! WebAssembly guest (`cpython` feature). +//! +//! Decisions (see `knowledge/runtimes/cpython-wasm.md`): +//! - Isolation comes from the wasm boundary, not from CPython. The guest can +//! only reach what [`wasi`] hands it: the VFS, captured stdio, a host RNG and +//! clocks. No host filesystem, network, process or thread API exists. +//! - No runtime opt-in env var (unlike Monty's `BASHKIT_ALLOW_INPROCESS_PYTHON`): +//! the guest never runs native code in the host, so calling +//! `BashBuilder::cpython()` is the opt-in. +//! - One process-wide engine and pre-linked module, created on first use (or +//! by [`CPython::warm_up`]). Every call gets a fresh store and instance, +//! mapped copy-on-write from the pre-initialized snapshot; nothing survives +//! between calls or tenants. +//! - Guest work is metered with fuel and yields to the async runtime at a fixed +//! interval; each poll re-checks the call deadline and the request's +//! `ExecutionBudget` (cancellation, timeout), so a busy loop cannot hold a +//! worker thread past its deadline. +//! - Instances come from wasmtime's pooling allocator (512 slots per +//! process): slots keep their memory mapped and are reset by discarding only +//! the pages a call dirtied, instead of mmap/munmap per call. Calls beyond +//! 512 wait for a slot within their own deadline. If the pool cannot reserve +//! its address space, instances are allocated on demand instead. +//! - Memory is capped by a store limiter; `memory.grow` past the cap fails and +//! surfaces as `MemoryError` inside Python, not as a host error. + +mod wasi; + +use std::future::Future; +use std::sync::OnceLock; +use std::task::Poll; +use std::time::Duration; + +use async_trait::async_trait; +use wasmtime::{Engine, InstancePre, Linker, Store}; + +use super::runtime_limits::{DEFAULT_MAX_DURATION, RuntimeLimits}; +use super::{Builtin, Context, ExecutionDeadline}; +use crate::error::Result; +use crate::interpreter::ExecResult; + +use wasi::{GuestConfig, GuestState, ProcExit}; + +/// Default memory cap for one call. The snapshot itself starts at ~40 MB of +/// linear memory (mostly untouched, mapped copy-on-write). +const DEFAULT_MAX_MEMORY: usize = 256 * 1024 * 1024; +/// CPython's own default recursion limit. +const DEFAULT_MAX_RECURSION: usize = 1000; +/// Default cap on captured stdout + stderr. +const DEFAULT_MAX_OUTPUT: usize = 16 * 1024 * 1024; +/// Guest instructions between cooperative yields (~1 ms on Pulley). +const YIELD_INTERVAL_FUEL: u64 = 100_000; +/// Request-budget work units charged per call before the guest runs. +const CALL_WORK_UNITS: u64 = 4096; +/// Guest instructions per request-budget work unit (a trivial call executes +/// ~0.5M instructions, so ~500 units on top of [`CALL_WORK_UNITS`]). +const FUEL_PER_WORK_UNIT: u64 = 1024; +/// Wasm call-stack budget. CPython recurses in C for nested containers, +/// `repr`, the compiler and the parser; 512 KiB (wasmtime's default) overflows +/// before Python's own recursion limit triggers. +const MAX_WASM_STACK: usize = 4 * 1024 * 1024; + +/// Resource limits for the CPython builtin. +/// +/// Defaults: 30 s per call, 256 MB guest memory, recursion limit 1000, 16 MB +/// of captured output. A tighter request deadline always wins over +/// `max_duration`. +#[derive(Debug, Clone)] +pub struct CPythonLimits { + /// Shared VM limits: wall clock, guest memory, Python recursion limit. + pub common: RuntimeLimits, + /// Maximum captured stdout + stderr bytes; output past this is dropped + /// and the result is marked truncated. + pub max_output: usize, +} + +impl Default for CPythonLimits { + fn default() -> Self { + Self { + common: RuntimeLimits { + max_duration: DEFAULT_MAX_DURATION, + max_memory: DEFAULT_MAX_MEMORY, + max_call_depth: DEFAULT_MAX_RECURSION, + }, + max_output: DEFAULT_MAX_OUTPUT, + } + } +} + +impl CPythonLimits { + /// Set max wall-clock duration per call. + #[must_use] + pub fn max_duration(mut self, d: Duration) -> Self { + self.common.max_duration = d; + self + } + + /// Set max guest memory in bytes (at most 1 GiB is honored). Values below + /// the snapshot's initial memory (~40 MB) make every call fail to start. + #[must_use] + pub fn max_memory(mut self, bytes: usize) -> Self { + self.common.max_memory = bytes; + self + } + + /// Set Python's recursion limit (`sys.setrecursionlimit`). + #[must_use] + pub fn max_recursion(mut self, depth: usize) -> Self { + self.common.max_call_depth = depth; + self + } + + /// Set the captured output cap in bytes. + #[must_use] + pub fn max_output(mut self, bytes: usize) -> Self { + self.max_output = bytes; + self + } +} + +/// Concurrent guest instances per process (pooled allocator slots). Calls +/// beyond this wait for a free slot, bounded by their own deadline. +const POOL_SLOTS: u32 = 512; +/// Largest guest memory a pooled slot can hold; `max_memory` above this is +/// clamped. Must not exceed the engine's `memory_reservation` (1 GiB). +const POOL_MAX_MEMORY: usize = 1 << 30; +/// Snapshot pages kept mapped in a released slot. Reset cost then scales with +/// what a call dirtied, not with the snapshot size. +const POOL_KEEP_RESIDENT: usize = 64 << 20; + +/// Process-wide compiled guest, shared by every `Bash` instance. +struct Runtime { + engine: Engine, + pre: InstancePre, + /// Pooled slots; `None` when the pool could not be created and every + /// instance is allocated on demand. + slots: Option, +} + +static RUNTIME: OnceLock> = OnceLock::new(); + +fn runtime() -> std::result::Result<&'static Runtime, &'static str> { + RUNTIME + .get_or_init(|| init_runtime().map_err(|e| e.to_string())) + .as_ref() + .map_err(String::as_str) +} + +fn init_runtime() -> wasmtime::Result { + // Pooled slots first: instances reuse pre-mapped memory and stacks, and a + // released slot is reset by discarding only the pages a call dirtied. On + // demand allocation (mmap + munmap per call) serializes many concurrent + // calls on the kernel's address-space lock. Fall back to it when the pool + // cannot reserve its address space (e.g. a tight RLIMIT_AS). + match Engine::new(&runtime_config(true)) { + Ok(engine) => link( + engine, + Some(tokio::sync::Semaphore::new(POOL_SLOTS as usize)), + ), + Err(_) => link(Engine::new(&runtime_config(false))?, None), + } +} + +fn runtime_config(pooled: bool) -> wasmtime::Config { + let mut config = bashkit_cpython_wasm::engine_config(); + // Runtime-only settings; they do not affect module compatibility. + config.max_wasm_stack(MAX_WASM_STACK); + config.async_stack_size(MAX_WASM_STACK + (1 << 20)); + if pooled { + let mut pool = wasmtime::PoolingAllocationConfig::new(); + pool.total_core_instances(POOL_SLOTS) + .total_memories(POOL_SLOTS) + .total_tables(POOL_SLOTS) + .total_stacks(POOL_SLOTS) + .max_memory_size(POOL_MAX_MEMORY) + .table_elements(1 << 16) + .max_core_instance_size(1 << 20) + .linear_memory_keep_resident(POOL_KEEP_RESIDENT) + .pagemap_scan(wasmtime::Enabled::Auto); + config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pool)); + } + config +} + +fn link(engine: Engine, slots: Option) -> wasmtime::Result { + let module = bashkit_cpython_wasm::load_module(&engine)?; + let mut linker = Linker::new(&engine); + wasi::add_to_linker(&mut linker)?; + let pre = linker.instantiate_pre(&module)?; + Ok(Runtime { engine, pre, slots }) +} + +/// CPython `python`/`python3` builtin. +pub struct CPython { + limits: CPythonLimits, +} + +impl CPython { + /// Create the builtin with custom limits. + pub fn with_limits(limits: CPythonLimits) -> Self { + Self { limits } + } + + /// Load the embedded interpreter now instead of on the first `python3` + /// call. Idempotent and process-wide; returns the error text if the + /// embedded module cannot be loaded. + pub fn warm_up() -> std::result::Result<(), String> { + runtime().map(|_| ()).map_err(str::to_string) + } +} + +impl Default for CPython { + fn default() -> Self { + Self::with_limits(CPythonLimits::default()) + } +} + +/// Why a guest call stopped without returning. +enum Stop { + Timeout, + Budget(crate::limits::LimitExceeded), +} + +#[async_trait] +impl Builtin for CPython { + fn llm_hint(&self) -> Option<&'static str> { + Some( + "python/python3: CPython 3.14 in a WebAssembly sandbox. Full pure-Python \ + stdlib plus json, re, csv, sqlite3, zlib, hashlib, decimal, datetime. \ + open()/pathlib/os work on the virtual filesystem. -c, -m, script files and \ + stdin work. No network, no subprocess, no threads, no pip or third-party packages.", + ) + } + + async fn execute(&self, ctx: Context<'_>) -> Result { + let rt = match runtime() { + Ok(rt) => rt, + Err(e) => { + return Ok(ExecResult::err( + format!("python3: embedded interpreter unavailable: {e}\n"), + 1, + )); + } + }; + + let limits = &self.limits.common; + let mut timeout = limits.max_duration; + if let Some(remaining) = ctx + .execution_extension::() + .and_then(|deadline| deadline.try_with(ExecutionDeadline::remaining).ok()) + { + timeout = timeout.min(remaining); + } + let budget = ctx + .execution_budget() + .map(|b| { + b.try_with(Clone::clone) + .map_err(|_| crate::Error::Cancelled) + }) + .transpose()?; + + let stdin = ctx.stdin.map(|s| s.as_bytes().to_vec()).unwrap_or_default(); + ctx.consume_budget_input(stdin.len())?; + // Charge a fixed cost per call up front (instance setup), then the + // guest work actually done (see below), so repeated calls in one + // request draw down one shared budget. + ctx.consume_budget_work(CALL_WORK_UNITS)?; + + // THREAT[TM-INF]: only exported variables reach the guest, never + // shell-local ones. + let mut env: Vec<(String, String)> = ctx + .env + .iter() + .filter(|(k, _)| k.as_str() != "PWD" && !k.starts_with("__BASHKIT_")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + env.sort(); + env.push(("PWD".to_string(), ctx.cwd.to_string_lossy().into_owned())); + env.push(( + "__BASHKIT_RECURSION_LIMIT".to_string(), + limits.max_call_depth.to_string(), + )); + + let mut args = Vec::with_capacity(ctx.args.len() + 1); + args.push("python3".to_string()); + args.extend(ctx.args.iter().cloned()); + + let deadline = crate::time_compat::Instant::now() + timeout; + let state = GuestState::new( + ctx.fs.clone(), + GuestConfig { + args, + env, + stdin, + output_cap: self.limits.max_output, + max_memory: if rt.slots.is_some() { + limits.max_memory.min(POOL_MAX_MEMORY) + } else { + limits.max_memory + }, + stdlib_path: bashkit_cpython_wasm::STDLIB_ZIP_PATH, + stdlib: bashkit_cpython_wasm::STDLIB_ZIP, + deadline, + }, + ); + + // Hold a pooled slot until the store (and its instance) is dropped; + // waiting for one counts against the deadline. + let _slot = match &rt.slots { + Some(slots) => match guarded(slots.acquire(), deadline, budget.as_ref()).await { + Ok(Ok(permit)) => Some(permit), + // The semaphore is never closed. + Ok(Err(_)) => None, + Err(Stop::Timeout) => return Ok(timed_out(timeout, Vec::new())), + Err(Stop::Budget(e)) => return Err(e.into()), + }, + None => None, + }; + let mut store = Store::new(&rt.engine, state); + store.limiter(|s| &mut s.limits); + let setup = store + .set_fuel(u64::MAX) + .and_then(|()| store.fuel_async_yield_interval(Some(YIELD_INTERVAL_FUEL))); + if let Err(e) = setup { + return Ok(ExecResult::err(format!("python3: {e}\n"), 1)); + } + + let outcome = guarded( + async { + let instance = rt.pre.instantiate_async(&mut store).await?; + let run = instance.get_typed_func::<(), i32>(&mut store, "bashkit_run")?; + run.call_async(&mut store, ()).await + }, + deadline, + budget.as_ref(), + ) + .await; + + let fuel_used = u64::MAX - store.get_fuel().unwrap_or(0); + // Files a script wrote but never closed persist, as they would after a + // real process exits or is killed. + store.data_mut().flush_all().await; + let state = store.into_data(); + + let mut stderr = state.output.stderr; + let exit_code = match outcome { + Ok(Ok(code)) => code, + Ok(Err(e)) => { + if let Some(exit) = e.downcast_ref::() { + exit.0 + } else { + stderr.extend_from_slice(trap_message(&e).as_bytes()); + 1 + } + } + Err(Stop::Timeout) => { + stderr.extend_from_slice(timeout_message(timeout).as_bytes()); + 124 + } + Err(Stop::Budget(e)) => return Err(e.into()), + }; + ctx.consume_budget_work(fuel_used / FUEL_PER_WORK_UNIT)?; + let exit_code = if state.flush_error.is_some() { + stderr.extend_from_slice(b"python3: failed to write back open files\n"); + if exit_code == 0 { 1 } else { exit_code } + } else { + exit_code + }; + + if state.output.truncated { + stderr.extend_from_slice( + format!( + "python3: output truncated at {} bytes\n", + self.limits.max_output + ) + .as_bytes(), + ); + } + let mut result = ExecResult::with_code(state.output.stdout, exit_code); + result.stderr = stderr.into(); + if state.output.truncated { + result.stdout_truncated = true; + result.stderr_truncated = true; + } + Ok(result) + } +} + +fn timeout_message(timeout: Duration) -> String { + format!( + "python3: execution timed out after {:.1}s\n", + timeout.as_secs_f64() + ) +} + +fn timed_out(timeout: Duration, stdout: Vec) -> ExecResult { + let mut result = ExecResult::with_code(stdout, 124); + result.stderr = timeout_message(timeout).into(); + result +} + +/// Poll `fut`, stopping at the deadline or when the request budget closes. +/// Fuel yields guarantee the guest returns to this poll regularly. +async fn guarded( + fut: F, + deadline: crate::time_compat::Instant, + budget: Option<&crate::limits::ExecutionBudget>, +) -> std::result::Result { + let mut fut = std::pin::pin!(fut); + std::future::poll_fn(|cx| { + if crate::time_compat::Instant::now() >= deadline { + return Poll::Ready(Err(Stop::Timeout)); + } + if let Some(budget) = budget + && let Err(e) = budget.check() + { + return Poll::Ready(Err(Stop::Budget(e))); + } + fut.as_mut().poll(cx).map(Ok) + }) + .await +} + +/// Describe a guest trap without leaking internal shapes (TM-INF-022). +fn trap_message(e: &wasmtime::Error) -> String { + match e.downcast_ref::() { + Some(wasmtime::Trap::StackOverflow) => { + "python3: fatal error: stack overflow in the interpreter\n".to_string() + } + Some(wasmtime::Trap::OutOfFuel) => "python3: fatal error: out of fuel\n".to_string(), + Some(wasmtime::Trap::UnreachableCodeReached) => { + "python3: fatal error: interpreter aborted\n".to_string() + } + Some(trap) => format!("python3: fatal error: {trap}\n"), + None => { + let mut msg = e.to_string(); + msg.truncate(msg.floor_char_boundary(512)); + format!("python3: fatal error: {msg}\n") + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/bashkit/src/builtins/cpython/tests.rs b/crates/bashkit/src/builtins/cpython/tests.rs new file mode 100644 index 000000000..05ea88f65 --- /dev/null +++ b/crates/bashkit/src/builtins/cpython/tests.rs @@ -0,0 +1,56 @@ +//! Unit tests for the CPython builtin plumbing (limits, trap mapping). + +use super::*; + +#[test] +fn default_limits() { + let l = CPythonLimits::default(); + assert_eq!(l.common.max_duration, Duration::from_secs(30)); + assert_eq!(l.common.max_memory, 256 * 1024 * 1024); + assert_eq!(l.common.max_call_depth, 1000); + assert_eq!(l.max_output, 16 * 1024 * 1024); +} + +#[test] +fn limit_setters() { + let l = CPythonLimits::default() + .max_duration(Duration::from_secs(2)) + .max_memory(64 << 20) + .max_recursion(77) + .max_output(10); + assert_eq!(l.common.max_duration, Duration::from_secs(2)); + assert_eq!(l.common.max_memory, 64 << 20); + assert_eq!(l.common.max_call_depth, 77); + assert_eq!(l.max_output, 10); +} + +#[test] +fn trap_messages_are_display_only() { + let e = wasmtime::Error::from(wasmtime::Trap::StackOverflow); + assert_eq!( + trap_message(&e), + "python3: fatal error: stack overflow in the interpreter\n" + ); + let e = wasmtime::Error::from(wasmtime::Trap::UnreachableCodeReached); + assert!(trap_message(&e).contains("aborted")); + let long = wasmtime::Error::msg("x".repeat(4096)); + assert!(trap_message(&long).len() < 600); +} + +#[test] +fn runtime_loads() { + CPython::warm_up().expect("embedded CPython loads"); +} + +#[test] +fn on_demand_fallback_links() { + // The path taken when the pooled engine cannot reserve address space. + let rt = link(Engine::new(&runtime_config(false)).unwrap(), None).unwrap(); + assert!(rt.slots.is_none()); +} + +#[test] +fn pooled_engine_links() { + let rt = link(Engine::new(&runtime_config(true)).unwrap(), None).unwrap(); + drop(rt); +} diff --git a/crates/bashkit/src/builtins/cpython/wasi.rs b/crates/bashkit/src/builtins/cpython/wasi.rs new file mode 100644 index 000000000..b6610e702 --- /dev/null +++ b/crates/bashkit/src/builtins/cpython/wasi.rs @@ -0,0 +1,1681 @@ +//! WASI preview1 host for the embedded CPython guest, backed by bashkit's VFS. +//! +//! Decisions: +//! - Only `wasi_snapshot_preview1` is implemented, and only against bashkit +//! state: the VFS, the captured stdin bytes, in-memory stdout/stderr, a +//! host RNG and host clocks. No host filesystem, socket or process API is +//! reachable; socket calls return `ENOTSUP`. +//! - One preopen, `/`, at fd 3 (the snapshot cached that exact table). +//! - Files are opened as whole in-memory buffers and written back to the VFS on +//! close/sync/exit. Buffers are capped by the VFS `max_file_size` and the +//! per-call output budget, so a guest cannot grow host memory without bound. +//! - The stdlib zip is a read-only overlay at a fixed path; the guest cannot +//! modify or shadow it. +//! - Paths are normalized lexically and clamped at `/`; `..` cannot escape the +//! VFS root (which is itself the whole sandbox). + +use crate::time_compat::{SystemTime, UNIX_EPOCH}; +use std::borrow::Cow; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use wasmtime::{Caller, Extern, Linker, Memory, StoreLimits}; + +use crate::fs::{FileSystem, FileType}; + +const MODULE: &str = "wasi_snapshot_preview1"; + +// --- WASI preview1 constants ------------------------------------------------- + +pub(crate) type Errno = u16; +pub(crate) const SUCCESS: Errno = 0; +const E2BIG: Errno = 1; +const EACCES: Errno = 2; +const EBADF: Errno = 8; +const EEXIST: Errno = 20; +const EFAULT: Errno = 21; +const EFBIG: Errno = 22; +const EILSEQ: Errno = 25; +const EINVAL: Errno = 28; +const EIO: Errno = 29; +const EISDIR: Errno = 31; +const EMFILE: Errno = 33; +const ENAMETOOLONG: Errno = 37; +const ENOENT: Errno = 44; +const ENOSPC: Errno = 51; +const ENOTDIR: Errno = 54; +const ENOTEMPTY: Errno = 55; +const ENOTSUP: Errno = 58; +const EROFS: Errno = 69; +const ESPIPE: Errno = 70; + +const FILETYPE_UNKNOWN: u8 = 0; +const FILETYPE_DIRECTORY: u8 = 3; +const FILETYPE_REGULAR_FILE: u8 = 4; +const FILETYPE_SYMBOLIC_LINK: u8 = 7; + +const OFLAGS_CREAT: u16 = 1; +const OFLAGS_DIRECTORY: u16 = 2; +const OFLAGS_EXCL: u16 = 4; +const OFLAGS_TRUNC: u16 = 8; + +const FDFLAGS_APPEND: u16 = 1; + +const RIGHTS_FD_READ: u64 = 1 << 1; +const RIGHTS_FD_WRITE: u64 = 1 << 6; +const RIGHTS_FD_READDIR: u64 = 1 << 14; +const RIGHTS_ALL: u64 = (1 << 30) - 1; + +const FSTFLAGS_MTIM: u16 = 1 << 2; +const FSTFLAGS_MTIM_NOW: u16 = 1 << 3; + +const CLOCK_REALTIME: u32 = 0; +const CLOCK_MONOTONIC: u32 = 1; + +const EVENTTYPE_CLOCK: u8 = 0; +const EVENTTYPE_FD_READ: u8 = 1; +const EVENTTYPE_FD_WRITE: u8 = 2; +const SUBCLOCKFLAGS_ABSTIME: u16 = 1; + +const PREOPEN_FD: u32 = 3; +const MAX_FDS: usize = 1024; +const MAX_PATH_LEN: usize = 4096; +/// Longest single `poll_oneoff` sleep; longer sleeps are cut into slices so +/// the call deadline is re-checked. +const MAX_SLEEP_SLICE: Duration = Duration::from_millis(250); + +// --- guest state ------------------------------------------------------------- + +/// Captured guest output with a shared byte cap. +#[derive(Default)] +pub(crate) struct Output { + pub(crate) stdout: Vec, + pub(crate) stderr: Vec, + pub(crate) truncated: bool, + cap: usize, +} + +impl Output { + fn write(&mut self, fd: u32, data: &[u8]) { + let used = self.stdout.len() + self.stderr.len(); + let room = self.cap.saturating_sub(used); + let take = data.len().min(room); + if take < data.len() { + self.truncated = true; + } + let target = if fd == 1 { + &mut self.stdout + } else { + &mut self.stderr + }; + target.extend_from_slice(&data[..take]); + } +} + +struct OpenFile { + path: PathBuf, + data: Cow<'static, [u8]>, + pos: u64, + read: bool, + write: bool, + append: bool, + dirty: bool, + modified: SystemTime, +} + +struct OpenDir { + path: PathBuf, + /// (name, filetype, inode); computed on first `fd_readdir`. + entries: Option>, +} + +enum Fd { + Stdin, + Stdout, + Stderr, + Root, + File(Box), + Dir(OpenDir), +} + +/// Per-call configuration handed to [`GuestState::new`]. +pub(crate) struct GuestConfig { + pub(crate) args: Vec, + pub(crate) env: Vec<(String, String)>, + pub(crate) stdin: Vec, + pub(crate) output_cap: usize, + pub(crate) max_memory: usize, + pub(crate) stdlib_path: &'static str, + pub(crate) stdlib: &'static [u8], + pub(crate) deadline: crate::time_compat::Instant, +} + +/// Everything one `python3` call's wasm instance can reach. +pub(crate) struct GuestState { + fs: Arc, + args: Vec>, + env: Vec>, + stdin: Vec, + stdin_pos: usize, + pub(crate) output: Output, + fds: Vec>, + pub(crate) limits: StoreLimits, + stdlib_path: &'static Path, + stdlib: &'static [u8], + started: crate::time_compat::Instant, + deadline: crate::time_compat::Instant, + /// Bytes held in open file buffers, bounded by `max_file_size` each and + /// `buffer_cap` in total. + buffered: usize, + buffer_cap: usize, + max_file_size: usize, + /// Errno of the first failed write-back at exit, if any. + pub(crate) flush_error: Option, +} + +/// Exit requested through `proc_exit`. +#[derive(Debug)] +pub(crate) struct ProcExit(pub(crate) i32); + +impl std::fmt::Display for ProcExit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "proc_exit({})", self.0) + } +} + +impl std::error::Error for ProcExit {} + +impl GuestState { + pub(crate) fn new(fs: Arc, config: GuestConfig) -> Self { + let mut args: Vec> = config.args.into_iter().map(String::into_bytes).collect(); + for arg in &mut args { + arg.retain(|b| *b != 0); + } + let env = config + .env + .into_iter() + .filter(|(k, v)| !k.is_empty() && !k.contains(['=', '\0']) && !v.contains('\0')) + .map(|(k, v)| format!("{k}={v}").into_bytes()) + .collect(); + let fs_limits = fs.limits(); + let max_file_size = usize::try_from(fs_limits.max_file_size).unwrap_or(usize::MAX); + let limits = wasmtime::StoreLimitsBuilder::new() + .memory_size(config.max_memory) + .instances(1) + .tables(4) + .memories(1) + .build(); + Self { + fs, + args, + env, + stdin: config.stdin, + stdin_pos: 0, + output: Output { + cap: config.output_cap, + ..Output::default() + }, + fds: vec![ + Some(Fd::Stdin), + Some(Fd::Stdout), + Some(Fd::Stderr), + Some(Fd::Root), + ], + limits, + stdlib_path: Path::new(config.stdlib_path), + stdlib: config.stdlib, + started: crate::time_compat::Instant::now(), + deadline: config.deadline, + buffered: 0, + // Open-file buffers share the guest memory budget: a script can + // keep at most as much file data in flight as it may allocate. + buffer_cap: config.max_memory, + max_file_size, + flush_error: None, + } + } + + fn is_stdlib(&self, path: &Path) -> bool { + path == self.stdlib_path + } + + fn is_stdlib_ancestor(&self, path: &Path) -> bool { + path != self.stdlib_path && self.stdlib_path.starts_with(path) + } + + fn alloc_fd(&mut self, fd: Fd) -> Result { + // Fds 0-2 are stdio and PREOPEN_FD is the root; never reuse them. + let first = PREOPEN_FD as usize + 1; + if let Some(slot) = self.fds.iter().skip(first).position(Option::is_none) { + let idx = slot + first; + self.fds[idx] = Some(fd); + return u32::try_from(idx).map_err(|_| EMFILE); + } + if self.fds.len() >= MAX_FDS { + return Err(EMFILE); + } + self.fds.push(Some(fd)); + u32::try_from(self.fds.len() - 1).map_err(|_| EMFILE) + } + + fn fd(&self, fd: u32) -> Result<&Fd, Errno> { + self.fds + .get(fd as usize) + .and_then(Option::as_ref) + .ok_or(EBADF) + } + + fn fd_mut(&mut self, fd: u32) -> Result<&mut Fd, Errno> { + self.fds + .get_mut(fd as usize) + .and_then(Option::as_mut) + .ok_or(EBADF) + } + + fn dir_base(&self, fd: u32) -> Result { + match self.fd(fd)? { + Fd::Root => Ok(PathBuf::from("/")), + Fd::Dir(d) => Ok(d.path.clone()), + _ => Err(ENOTDIR), + } + } + + /// Resolve a guest path relative to directory fd `fd`. + fn resolve(&self, fd: u32, raw: &[u8]) -> Result { + if raw.len() > MAX_PATH_LEN { + return Err(ENAMETOOLONG); + } + if raw.contains(&0) { + return Err(EINVAL); + } + let rel = std::str::from_utf8(raw).map_err(|_| EILSEQ)?; + let base = self.dir_base(fd)?; + Ok(normalize(&base, rel)) + } + + /// Reserve `extra` bytes of open-file buffer space. + fn reserve(&mut self, extra: usize) -> Result<(), Errno> { + let next = self.buffered.checked_add(extra).ok_or(ENOSPC)?; + if next > self.buffer_cap { + return Err(ENOSPC); + } + self.buffered = next; + Ok(()) + } + + fn release(&mut self, bytes: usize) { + self.buffered = self.buffered.saturating_sub(bytes); + } + + /// Write every dirty file buffer back to the VFS (exit-time flush). + pub(crate) async fn flush_all(&mut self) { + let fs = self.fs.clone(); + for slot in &mut self.fds { + if let Some(Fd::File(file)) = slot + && file.dirty + { + if let Err(e) = fs.write_file(&file.path, &file.data).await { + self.flush_error.get_or_insert(map_fs_error(&e)); + } + file.dirty = false; + } + } + } +} + +/// Lexically normalize `rel` against `base`, clamping `..` at `/`. +fn normalize(base: &Path, rel: &str) -> PathBuf { + let mut parts: Vec<&str> = Vec::new(); + let start: &str = if rel.starts_with('/') { + "" + } else { + base.to_str().unwrap_or("/") + }; + for comp in start.split('/').chain(rel.split('/')) { + match comp { + "" | "." => {} + ".." => { + parts.pop(); + } + c => parts.push(c), + } + } + let mut out = String::from("/"); + out.push_str(&parts.join("/")); + PathBuf::from(out) +} + +fn inode(path: &Path) -> u64 { + let mut h = DefaultHasher::new(); + path.hash(&mut h); + h.finish() | 1 +} + +fn filetype_of(t: FileType) -> u8 { + match t { + FileType::File => FILETYPE_REGULAR_FILE, + FileType::Directory => FILETYPE_DIRECTORY, + FileType::Symlink => FILETYPE_SYMBOLIC_LINK, + FileType::Fifo => FILETYPE_UNKNOWN, + } +} + +fn nanos(t: SystemTime) -> u64 { + t.duration_since(UNIX_EPOCH) + .map(|d| u64::try_from(d.as_nanos()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +/// Map a bashkit VFS error to a WASI errno. +fn map_fs_error(e: &crate::Error) -> Errno { + use std::io::ErrorKind; + if let crate::Error::Io(io) = e { + match io.kind() { + ErrorKind::NotFound => return ENOENT, + ErrorKind::AlreadyExists => return EEXIST, + ErrorKind::PermissionDenied => return EACCES, + ErrorKind::InvalidInput => return EINVAL, + _ => {} + } + } + let msg = e.to_string().to_ascii_lowercase(); + if msg.contains("not found") || msg.contains("no such file") { + ENOENT + } else if msg.contains("is a directory") { + EISDIR + } else if msg.contains("not a directory") { + ENOTDIR + } else if msg.contains("already exists") { + EEXIST + } else if msg.contains("not empty") { + ENOTEMPTY + } else if msg.contains("limit") || msg.contains("too large") || msg.contains("exceed") { + ENOSPC + } else if msg.contains("read-only") || msg.contains("readonly") { + EROFS + } else if msg.contains("permission") || msg.contains("denied") { + EACCES + } else { + EIO + } +} + +// --- guest memory helpers ---------------------------------------------------- + +fn memory(caller: &mut Caller<'_, T>) -> Result { + match caller.get_export("memory") { + Some(Extern::Memory(m)) => Ok(m), + _ => Err(EFAULT), + } +} + +fn range(ptr: i32, len: u32, mem_len: usize) -> Result, Errno> { + let start = ptr as u32 as usize; + let end = start.checked_add(len as usize).ok_or(EFAULT)?; + if end > mem_len { + return Err(EFAULT); + } + Ok(start..end) +} + +fn read_bytes(caller: &mut Caller<'_, GuestState>, ptr: i32, len: i32) -> Result, Errno> { + let mem = memory(caller)?; + let data = mem.data(&*caller); + let r = range(ptr, len as u32, data.len())?; + Ok(data[r].to_vec()) +} + +fn write_bytes(caller: &mut Caller<'_, GuestState>, ptr: i32, bytes: &[u8]) -> Result<(), Errno> { + let mem = memory(caller)?; + let data = mem.data_mut(&mut *caller); + let r = range( + ptr, + u32::try_from(bytes.len()).map_err(|_| EFAULT)?, + data.len(), + )?; + data[r].copy_from_slice(bytes); + Ok(()) +} + +fn write_u32(caller: &mut Caller<'_, GuestState>, ptr: i32, v: u32) -> Result<(), Errno> { + write_bytes(caller, ptr, &v.to_le_bytes()) +} + +fn write_u64(caller: &mut Caller<'_, GuestState>, ptr: i32, v: u64) -> Result<(), Errno> { + write_bytes(caller, ptr, &v.to_le_bytes()) +} + +/// Read an iovec array: `(buf_ptr, buf_len)` pairs. +fn read_iovs( + caller: &mut Caller<'_, GuestState>, + iovs: i32, + count: i32, +) -> Result, Errno> { + let count = u32::try_from(count).map_err(|_| EINVAL)?; + if count > 1024 { + return Err(EINVAL); + } + let raw = read_bytes(caller, iovs, i32::try_from(count * 8).map_err(|_| EINVAL)?)?; + Ok(raw + .chunks_exact(8) + .map(|c| { + let ptr = i32::from_le_bytes([c[0], c[1], c[2], c[3]]); + let len = u32::from_le_bytes([c[4], c[5], c[6], c[7]]); + (ptr, len) + }) + .collect()) +} + +fn gather(caller: &mut Caller<'_, GuestState>, iovs: &[(i32, u32)]) -> Result, Errno> { + let mem = memory(caller)?; + let data = mem.data(&*caller); + let mut out = Vec::new(); + for &(ptr, len) in iovs { + let r = range(ptr, len, data.len())?; + out.extend_from_slice(&data[r]); + } + Ok(out) +} + +fn scatter( + caller: &mut Caller<'_, GuestState>, + iovs: &[(i32, u32)], + src: &[u8], +) -> Result { + let mem = memory(caller)?; + let data = mem.data_mut(&mut *caller); + let mut done = 0; + for &(ptr, len) in iovs { + if done >= src.len() { + break; + } + let r = range(ptr, len, data.len())?; + let n = (src.len() - done).min(r.len()); + data[r.start..r.start + n].copy_from_slice(&src[done..done + n]); + done += n; + } + Ok(done) +} + +fn total_len(iovs: &[(i32, u32)]) -> usize { + iovs.iter().map(|&(_, l)| l as usize).sum() +} + +fn ret(r: Result<(), Errno>) -> i32 { + i32::from(match r { + Ok(()) => SUCCESS, + Err(e) => e, + }) +} + +// --- filestat ---------------------------------------------------------------- + +struct Stat { + filetype: u8, + size: u64, + mtime: u64, + ino: u64, +} + +fn encode_filestat(s: &Stat) -> [u8; 64] { + let mut b = [0u8; 64]; + b[0..8].copy_from_slice(&1u64.to_le_bytes()); // dev + b[8..16].copy_from_slice(&s.ino.to_le_bytes()); + b[16] = s.filetype; + b[24..32].copy_from_slice(&1u64.to_le_bytes()); // nlink + b[32..40].copy_from_slice(&s.size.to_le_bytes()); + b[40..48].copy_from_slice(&s.mtime.to_le_bytes()); // atim + b[48..56].copy_from_slice(&s.mtime.to_le_bytes()); // mtim + b[56..64].copy_from_slice(&s.mtime.to_le_bytes()); // ctim + b +} + +async fn stat_path(state: &GuestState, path: &Path) -> Result { + if state.is_stdlib(path) { + return Ok(Stat { + filetype: FILETYPE_REGULAR_FILE, + size: state.stdlib.len() as u64, + mtime: 0, + ino: inode(path), + }); + } + match state.fs.stat(path).await { + Ok(m) => Ok(Stat { + filetype: filetype_of(m.file_type), + size: m.size, + mtime: nanos(m.modified), + ino: inode(path), + }), + Err(e) if state.is_stdlib_ancestor(path) => { + let _ = e; + Ok(Stat { + filetype: FILETYPE_DIRECTORY, + size: 0, + mtime: 0, + ino: inode(path), + }) + } + Err(e) => Err(map_fs_error(&e)), + } +} + +// --- linker ------------------------------------------------------------------ + +/// Register every `wasi_snapshot_preview1` import the guest uses. +pub(crate) fn add_to_linker(linker: &mut Linker) -> wasmtime::Result<()> { + // args / environ + linker.func_wrap( + MODULE, + "args_sizes_get", + |mut c: Caller<'_, GuestState>, argc: i32, size: i32| { + let n = c.data().args.len() as u32; + let bytes: usize = c.data().args.iter().map(|a| a.len() + 1).sum(); + ret(write_u32(&mut c, argc, n).and_then(|()| write_u32(&mut c, size, bytes as u32))) + }, + )?; + linker.func_wrap( + MODULE, + "args_get", + |mut c: Caller<'_, GuestState>, argv: i32, buf: i32| { + let items = c.data().args.clone(); + ret(write_string_table(&mut c, &items, argv, buf)) + }, + )?; + linker.func_wrap( + MODULE, + "environ_sizes_get", + |mut c: Caller<'_, GuestState>, count: i32, size: i32| { + let n = c.data().env.len() as u32; + let bytes: usize = c.data().env.iter().map(|a| a.len() + 1).sum(); + ret(write_u32(&mut c, count, n).and_then(|()| write_u32(&mut c, size, bytes as u32))) + }, + )?; + linker.func_wrap( + MODULE, + "environ_get", + |mut c: Caller<'_, GuestState>, environ: i32, buf: i32| { + let items = c.data().env.clone(); + ret(write_string_table(&mut c, &items, environ, buf)) + }, + )?; + + // clocks / random / scheduling + linker.func_wrap( + MODULE, + "clock_res_get", + |mut c: Caller<'_, GuestState>, _id: i32, out: i32| ret(write_u64(&mut c, out, 1_000)), + )?; + linker.func_wrap( + MODULE, + "clock_time_get", + |mut c: Caller<'_, GuestState>, id: i32, _precision: i64, out: i32| { + let now = match id as u32 { + CLOCK_REALTIME => nanos(SystemTime::now()), + CLOCK_MONOTONIC => monotonic_nanos(), + // Process/thread CPU clocks: time since this call started. + _ => u64::try_from(c.data().started.elapsed().as_nanos()).unwrap_or(u64::MAX), + }; + ret(write_u64(&mut c, out, now)) + }, + )?; + linker.func_wrap( + MODULE, + "random_get", + |mut c: Caller<'_, GuestState>, buf: i32, len: i32| { + let r = (|| { + let len = usize::try_from(len).map_err(|_| EINVAL)?; + if len > 1 << 20 { + return Err(EINVAL); + } + let mut bytes = vec![0u8; len]; + getrandom::fill(&mut bytes).map_err(|_| EIO)?; + write_bytes(&mut c, buf, &bytes) + })(); + ret(r) + }, + )?; + linker.func_wrap(MODULE, "sched_yield", |_c: Caller<'_, GuestState>| -> i32 { + 0 + })?; + linker.func_wrap( + MODULE, + "proc_exit", + |_c: Caller<'_, GuestState>, code: i32| -> wasmtime::Result<()> { + Err(wasmtime::Error::new(ProcExit(code))) + }, + )?; + linker.func_wrap_async( + MODULE, + "poll_oneoff", + |mut c: Caller<'_, GuestState>, (subs, events, n, nevents): (i32, i32, i32, i32)| { + Box::new(async move { ret(poll_oneoff(&mut c, subs, events, n, nevents).await) }) + }, + )?; + + // sockets: never available + for (name, arity) in [ + ("sock_accept", 3usize), + ("sock_recv", 6), + ("sock_send", 5), + ("sock_shutdown", 2), + ] { + match arity { + 2 => linker.func_wrap( + MODULE, + name, + |_c: Caller<'_, GuestState>, _: i32, _: i32| -> i32 { i32::from(ENOTSUP) }, + )?, + 3 => linker.func_wrap( + MODULE, + name, + |_c: Caller<'_, GuestState>, _: i32, _: i32, _: i32| -> i32 { i32::from(ENOTSUP) }, + )?, + 5 => linker.func_wrap( + MODULE, + name, + |_c: Caller<'_, GuestState>, _: i32, _: i32, _: i32, _: i32, _: i32| -> i32 { + i32::from(ENOTSUP) + }, + )?, + _ => linker.func_wrap( + MODULE, + name, + |_c: Caller<'_, GuestState>, + _: i32, + _: i32, + _: i32, + _: i32, + _: i32, + _: i32| + -> i32 { i32::from(ENOTSUP) }, + )?, + }; + } + + // fd: metadata + linker.func_wrap( + MODULE, + "fd_prestat_get", + |mut c: Caller<'_, GuestState>, fd: i32, buf: i32| { + let r = match c.data().fd(fd as u32) { + Ok(Fd::Root) => { + let mut b = [0u8; 8]; + b[4..8].copy_from_slice(&1u32.to_le_bytes()); + write_bytes(&mut c, buf, &b) + } + Ok(_) => Err(EBADF), + Err(e) => Err(e), + }; + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_prestat_dir_name", + |mut c: Caller<'_, GuestState>, fd: i32, buf: i32, len: i32| { + let r = match c.data().fd(fd as u32) { + Ok(Fd::Root) if len >= 1 => write_bytes(&mut c, buf, b"/"), + Ok(Fd::Root) => Err(EINVAL), + Ok(_) => Err(EBADF), + Err(e) => Err(e), + }; + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_fdstat_get", + |mut c: Caller<'_, GuestState>, fd: i32, buf: i32| { + let r = (|| { + let (filetype, flags, rights) = match c.data().fd(fd as u32)? { + // Pipes, not ttys: `isatty()` is false, like stdio in a bash pipeline. + Fd::Stdin => (FILETYPE_UNKNOWN, 0u16, RIGHTS_FD_READ), + Fd::Stdout | Fd::Stderr => (FILETYPE_UNKNOWN, 0, RIGHTS_FD_WRITE), + Fd::Root | Fd::Dir(_) => (FILETYPE_DIRECTORY, 0, RIGHTS_ALL), + Fd::File(f) => ( + FILETYPE_REGULAR_FILE, + if f.append { FDFLAGS_APPEND } else { 0 }, + RIGHTS_ALL, + ), + }; + let mut b = [0u8; 24]; + b[0] = filetype; + b[2..4].copy_from_slice(&flags.to_le_bytes()); + b[8..16].copy_from_slice(&rights.to_le_bytes()); + b[16..24].copy_from_slice(&rights.to_le_bytes()); + write_bytes(&mut c, buf, &b) + })(); + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_fdstat_set_flags", + |mut c: Caller<'_, GuestState>, fd: i32, flags: i32| { + let r = match c.data_mut().fd_mut(fd as u32) { + Ok(Fd::File(f)) => { + f.append = (flags as u16) & FDFLAGS_APPEND != 0; + Ok(()) + } + Ok(_) => Ok(()), + Err(e) => Err(e), + }; + ret(r) + }, + )?; + linker.func_wrap_async( + MODULE, + "fd_filestat_get", + |mut c: Caller<'_, GuestState>, (fd, buf): (i32, i32)| { + Box::new(async move { + let r = async { + let stat = match c.data().fd(fd as u32)? { + Fd::File(f) => Stat { + filetype: FILETYPE_REGULAR_FILE, + size: f.data.len() as u64, + mtime: nanos(f.modified), + ino: inode(&f.path), + }, + Fd::Root => Stat { + filetype: FILETYPE_DIRECTORY, + size: 0, + mtime: 0, + ino: inode(Path::new("/")), + }, + Fd::Dir(d) => { + let path = d.path.clone(); + stat_path(c.data(), &path).await? + } + Fd::Stdin | Fd::Stdout | Fd::Stderr => Stat { + filetype: FILETYPE_UNKNOWN, + size: 0, + mtime: 0, + ino: u64::from(fd as u32) + 1, + }, + }; + write_bytes(&mut c, buf, &encode_filestat(&stat)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap( + MODULE, + "fd_filestat_set_size", + |mut c: Caller<'_, GuestState>, fd: i32, size: i64| { + let r = (|| { + let size = usize::try_from(size).map_err(|_| EINVAL)?; + let state = c.data_mut(); + let max = state.max_file_size; + let current = match state.fd(fd as u32)? { + Fd::File(f) if f.write => f.data.len(), + Fd::File(_) => return Err(EBADF), + _ => return Err(EINVAL), + }; + if size > max { + return Err(EFBIG); + } + if size > current { + state.reserve(size - current)?; + } else { + state.release(current - size); + } + if let Fd::File(f) = state.fd_mut(fd as u32)? { + f.data.to_mut().resize(size, 0); + f.dirty = true; + } + Ok(()) + })(); + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_filestat_set_times", + |_c: Caller<'_, GuestState>, _fd: i32, _atim: i64, _mtim: i64, _flags: i32| -> i32 { 0 }, + )?; + linker.func_wrap( + MODULE, + "fd_advise", + |c: Caller<'_, GuestState>, fd: i32, _off: i64, _len: i64, _advice: i32| -> i32 { + ret(c.data().fd(fd as u32).map(|_| ())) + }, + )?; + + // fd: data + linker.func_wrap( + MODULE, + "fd_read", + |mut c: Caller<'_, GuestState>, fd: i32, iovs: i32, n: i32, nread: i32| { + ret(fd_read(&mut c, fd as u32, iovs, n, None) + .and_then(|k| write_u32(&mut c, nread, k as u32))) + }, + )?; + linker.func_wrap( + MODULE, + "fd_pread", + |mut c: Caller<'_, GuestState>, fd: i32, iovs: i32, n: i32, offset: i64, nread: i32| { + let r = u64::try_from(offset) + .map_err(|_| EINVAL) + .and_then(|off| fd_read(&mut c, fd as u32, iovs, n, Some(off))) + .and_then(|k| write_u32(&mut c, nread, k as u32)); + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_write", + |mut c: Caller<'_, GuestState>, fd: i32, iovs: i32, n: i32, nwritten: i32| { + ret(fd_write(&mut c, fd as u32, iovs, n, None) + .and_then(|k| write_u32(&mut c, nwritten, k as u32))) + }, + )?; + linker.func_wrap( + MODULE, + "fd_pwrite", + |mut c: Caller<'_, GuestState>, fd: i32, iovs: i32, n: i32, offset: i64, nwritten: i32| { + let r = u64::try_from(offset) + .map_err(|_| EINVAL) + .and_then(|off| fd_write(&mut c, fd as u32, iovs, n, Some(off))) + .and_then(|k| write_u32(&mut c, nwritten, k as u32)); + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_seek", + |mut c: Caller<'_, GuestState>, fd: i32, offset: i64, whence: i32, out: i32| { + let r = (|| { + let file = match c.data_mut().fd_mut(fd as u32)? { + Fd::File(f) => f, + Fd::Stdin | Fd::Stdout | Fd::Stderr => return Err(ESPIPE), + _ => return Err(EBADF), + }; + let base: i128 = match whence { + 0 => 0, + 1 => i128::from(file.pos), + 2 => file.data.len() as i128, + _ => return Err(EINVAL), + }; + let next = base + i128::from(offset); + if next < 0 { + return Err(EINVAL); + } + file.pos = u64::try_from(next).map_err(|_| EINVAL)?; + let pos = file.pos; + write_u64(&mut c, out, pos) + })(); + ret(r) + }, + )?; + linker.func_wrap( + MODULE, + "fd_tell", + |mut c: Caller<'_, GuestState>, fd: i32, out: i32| { + let r = match c.data().fd(fd as u32) { + Ok(Fd::File(f)) => { + let pos = f.pos; + write_u64(&mut c, out, pos) + } + Ok(Fd::Stdin | Fd::Stdout | Fd::Stderr) => Err(ESPIPE), + Ok(_) => Err(EBADF), + Err(e) => Err(e), + }; + ret(r) + }, + )?; + linker.func_wrap_async( + MODULE, + "fd_close", + |mut c: Caller<'_, GuestState>, (fd,): (i32,)| { + Box::new(async move { ret(fd_close(&mut c, fd as u32).await) }) + }, + )?; + linker.func_wrap_async( + MODULE, + "fd_sync", + |mut c: Caller<'_, GuestState>, (fd,): (i32,)| { + Box::new(async move { ret(fd_sync(&mut c, fd as u32).await) }) + }, + )?; + linker.func_wrap_async( + MODULE, + "fd_datasync", + |mut c: Caller<'_, GuestState>, (fd,): (i32,)| { + Box::new(async move { ret(fd_sync(&mut c, fd as u32).await) }) + }, + )?; + linker.func_wrap_async( + MODULE, + "fd_readdir", + |mut c: Caller<'_, GuestState>, (fd, buf, len, cookie, used): (i32, i32, i32, i64, i32)| { + Box::new( + async move { ret(fd_readdir(&mut c, fd as u32, buf, len, cookie, used).await) }, + ) + }, + )?; + + // paths + linker.func_wrap_async( + MODULE, + "path_open", + |mut c: Caller<'_, GuestState>, + (dirfd, _dirflags, path, path_len, oflags, rights, _inherit, fdflags, out): ( + i32, + i32, + i32, + i32, + i32, + i64, + i64, + i32, + i32, + )| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let fd = path_open( + &mut c, + dirfd as u32, + &raw, + oflags as u16, + rights as u64, + fdflags as u16, + ) + .await?; + write_u32(&mut c, out, fd) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_filestat_get", + |mut c: Caller<'_, GuestState>, + (dirfd, _flags, path, path_len, buf): (i32, i32, i32, i32, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + let stat = stat_path(c.data(), &p).await?; + write_bytes(&mut c, buf, &encode_filestat(&stat)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_filestat_set_times", + |mut c: Caller<'_, GuestState>, + (dirfd, _flags, path, path_len, _atim, mtim, fst): (i32, i32, i32, i32, i64, i64, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + let fst = fst as u16; + if c.data().is_stdlib(&p) { + return Err(EROFS); + } + let when = if fst & FSTFLAGS_MTIM_NOW != 0 { + Some(SystemTime::now()) + } else if fst & FSTFLAGS_MTIM != 0 { + Some(UNIX_EPOCH + Duration::from_nanos(mtim as u64)) + } else { + None + }; + let fs = c.data().fs.clone(); + fs.stat(&p).await.map_err(|e| map_fs_error(&e))?; + if let Some(t) = when { + fs.set_modified_time(&p, t).await.map_err(|e| map_fs_error(&e))?; + } + Ok(()) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_create_directory", + |mut c: Caller<'_, GuestState>, (dirfd, path, path_len): (i32, i32, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + if c.data().is_stdlib(&p) || c.data().is_stdlib_ancestor(&p) { + let fs = c.data().fs.clone(); + if fs.stat(&p).await.is_ok() || c.data().is_stdlib(&p) { + return Err(EEXIST); + } + } + let fs = c.data().fs.clone(); + if fs.stat(&p).await.is_ok() { + return Err(EEXIST); + } + fs.mkdir(&p, false).await.map_err(|e| map_fs_error(&e)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_remove_directory", + |mut c: Caller<'_, GuestState>, (dirfd, path, path_len): (i32, i32, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + if p == Path::new("/") || c.data().is_stdlib_ancestor(&p) { + return Err(EACCES); + } + let fs = c.data().fs.clone(); + let meta = fs.stat(&p).await.map_err(|e| map_fs_error(&e))?; + if !meta.file_type.is_dir() { + return Err(ENOTDIR); + } + let entries = fs.read_dir(&p).await.map_err(|e| map_fs_error(&e))?; + if !entries.is_empty() { + return Err(ENOTEMPTY); + } + fs.remove(&p, false).await.map_err(|e| map_fs_error(&e)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_unlink_file", + |mut c: Caller<'_, GuestState>, (dirfd, path, path_len): (i32, i32, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + if c.data().is_stdlib(&p) { + return Err(EROFS); + } + let fs = c.data().fs.clone(); + let meta = fs.stat(&p).await.map_err(|e| map_fs_error(&e))?; + if meta.file_type.is_dir() { + return Err(EISDIR); + } + fs.remove(&p, false).await.map_err(|e| map_fs_error(&e)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_rename", + |mut c: Caller<'_, GuestState>, + (fd1, old, old_len, fd2, new, new_len): (i32, i32, i32, i32, i32, i32)| { + Box::new(async move { + let r = async { + let old_raw = read_bytes(&mut c, old, old_len)?; + let new_raw = read_bytes(&mut c, new, new_len)?; + let from = c.data().resolve(fd1 as u32, &old_raw)?; + let to = c.data().resolve(fd2 as u32, &new_raw)?; + if c.data().is_stdlib(&from) || c.data().is_stdlib(&to) { + return Err(EROFS); + } + let fs = c.data().fs.clone(); + fs.rename(&from, &to).await.map_err(|e| map_fs_error(&e)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_symlink", + |mut c: Caller<'_, GuestState>, + (old, old_len, dirfd, new, new_len): (i32, i32, i32, i32, i32)| { + Box::new(async move { + let r = async { + let target = read_bytes(&mut c, old, old_len)?; + let target = String::from_utf8(target).map_err(|_| EILSEQ)?; + let new_raw = read_bytes(&mut c, new, new_len)?; + let link = c.data().resolve(dirfd as u32, &new_raw)?; + if c.data().is_stdlib(&link) { + return Err(EROFS); + } + let fs = c.data().fs.clone(); + fs.symlink(Path::new(&target), &link) + .await + .map_err(|e| map_fs_error(&e)) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap_async( + MODULE, + "path_readlink", + |mut c: Caller<'_, GuestState>, + (dirfd, path, path_len, buf, buf_len, used): (i32, i32, i32, i32, i32, i32)| { + Box::new(async move { + let r = async { + let raw = read_bytes(&mut c, path, path_len)?; + let p = c.data().resolve(dirfd as u32, &raw)?; + let fs = c.data().fs.clone(); + let target = fs.read_link(&p).await.map_err(|e| map_fs_error(&e))?; + let bytes = target.to_string_lossy().into_owned().into_bytes(); + let n = bytes.len().min(buf_len.max(0) as usize); + write_bytes(&mut c, buf, &bytes[..n])?; + write_u32(&mut c, used, n as u32) + } + .await; + ret(r) + }) + }, + )?; + linker.func_wrap( + MODULE, + "path_link", + |_c: Caller<'_, GuestState>, + _: i32, + _: i32, + _: i32, + _: i32, + _: i32, + _: i32, + _: i32| + -> i32 { i32::from(ENOTSUP) }, + )?; + Ok(()) +} + +fn monotonic_nanos() -> u64 { + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = BASE.get_or_init(crate::time_compat::Instant::now); + u64::try_from(base.elapsed().as_nanos()).unwrap_or(u64::MAX) +} + +fn write_string_table( + c: &mut Caller<'_, GuestState>, + items: &[Vec], + ptrs: i32, + buf: i32, +) -> Result<(), Errno> { + let mut offset = buf as u32; + let mut pointers = Vec::with_capacity(items.len() * 4); + let mut blob = Vec::new(); + for item in items { + pointers.extend_from_slice(&offset.to_le_bytes()); + blob.extend_from_slice(item); + blob.push(0); + offset = offset.checked_add(item.len() as u32 + 1).ok_or(E2BIG)?; + } + write_bytes(c, ptrs, &pointers)?; + write_bytes(c, buf, &blob) +} + +fn fd_read( + c: &mut Caller<'_, GuestState>, + fd: u32, + iovs: i32, + n: i32, + offset: Option, +) -> Result { + let iovs = read_iovs(c, iovs, n)?; + let want = total_len(&iovs); + let chunk: Vec = { + let state = c.data_mut(); + match state.fd_mut(fd)? { + Fd::Stdin => { + if offset.is_some() { + return Err(ESPIPE); + } + let start = state.stdin_pos.min(state.stdin.len()); + let end = start.saturating_add(want).min(state.stdin.len()); + state.stdin_pos = end; + state.stdin[start..end].to_vec() + } + Fd::File(f) => { + if !f.read { + return Err(EBADF); + } + let pos = offset.unwrap_or(f.pos); + let start = usize::try_from(pos).unwrap_or(usize::MAX).min(f.data.len()); + let end = start.saturating_add(want).min(f.data.len()); + if offset.is_none() { + f.pos = end as u64; + } + f.data[start..end].to_vec() + } + Fd::Root | Fd::Dir(_) => return Err(EISDIR), + Fd::Stdout | Fd::Stderr => return Err(EBADF), + } + }; + scatter(c, &iovs, &chunk) +} + +fn fd_write( + c: &mut Caller<'_, GuestState>, + fd: u32, + iovs: i32, + n: i32, + offset: Option, +) -> Result { + let iovs = read_iovs(c, iovs, n)?; + let data = gather(c, &iovs)?; + let state = c.data_mut(); + let max = state.max_file_size; + match state.fd(fd)? { + Fd::Stdout | Fd::Stderr => { + if offset.is_some() { + return Err(ESPIPE); + } + state.output.write(fd, &data); + return Ok(data.len()); + } + Fd::File(f) if f.write => { + let pos = if f.append { + f.data.len() as u64 + } else { + offset.unwrap_or(f.pos) + }; + let start = usize::try_from(pos).map_err(|_| EFBIG)?; + let end = start.checked_add(data.len()).ok_or(EFBIG)?; + if end > max { + return Err(EFBIG); + } + let grow = end.saturating_sub(f.data.len()); + state.reserve(grow)?; + } + Fd::File(_) => return Err(EBADF), + Fd::Stdin => return Err(EBADF), + Fd::Root | Fd::Dir(_) => return Err(EISDIR), + } + let Fd::File(f) = state.fd_mut(fd)? else { + return Err(EBADF); + }; + let pos = if f.append { + f.data.len() + } else { + usize::try_from(offset.unwrap_or(f.pos)).map_err(|_| EFBIG)? + }; + let end = pos + data.len(); + let buf = f.data.to_mut(); + if buf.len() < end { + buf.resize(end, 0); + } + buf[pos..end].copy_from_slice(&data); + f.dirty = true; + f.modified = SystemTime::now(); + if offset.is_none() { + f.pos = end as u64; + } + Ok(data.len()) +} + +async fn fd_close(c: &mut Caller<'_, GuestState>, fd: u32) -> Result<(), Errno> { + let state = c.data_mut(); + let slot = state.fds.get_mut(fd as usize).ok_or(EBADF)?; + let entry = slot.take().ok_or(EBADF)?; + if let Fd::File(file) = entry { + let len = match &file.data { + Cow::Owned(v) => v.len(), + Cow::Borrowed(_) => 0, + }; + state.release(len); + if file.dirty { + let fs = state.fs.clone(); + fs.write_file(&file.path, &file.data) + .await + .map_err(|e| map_fs_error(&e))?; + } + } + Ok(()) +} + +async fn fd_sync(c: &mut Caller<'_, GuestState>, fd: u32) -> Result<(), Errno> { + let state = c.data_mut(); + let fs = state.fs.clone(); + if let Fd::File(f) = state.fd_mut(fd)? + && f.dirty + { + fs.write_file(&f.path, &f.data) + .await + .map_err(|e| map_fs_error(&e))?; + f.dirty = false; + } + Ok(()) +} + +async fn fd_readdir( + c: &mut Caller<'_, GuestState>, + fd: u32, + buf: i32, + len: i32, + cookie: i64, + used: i32, +) -> Result<(), Errno> { + let (path, cached) = match c.data().fd(fd)? { + Fd::Root => (PathBuf::from("/"), None), + Fd::Dir(d) => (d.path.clone(), d.entries.clone()), + _ => return Err(ENOTDIR), + }; + let entries = match cached { + Some(e) => e, + None => { + let state = c.data(); + let mut list = vec![ + (".".to_string(), FILETYPE_DIRECTORY, inode(&path)), + ( + "..".to_string(), + FILETYPE_DIRECTORY, + inode(path.parent().unwrap_or(&path)), + ), + ]; + let mut names: Vec<(String, u8, u64)> = match state.fs.read_dir(&path).await { + Ok(items) => items + .into_iter() + .map(|e| { + let child = crate::fs::vfs_join(&path, &e.name); + (e.name, filetype_of(e.metadata.file_type), inode(&child)) + }) + .collect(), + Err(e) if state.is_stdlib_ancestor(&path) => { + let _ = e; + Vec::new() + } + Err(e) => return Err(map_fs_error(&e)), + }; + // Surface the stdlib overlay in its parent listing. + if state.stdlib_path.parent() == Some(path.as_path()) + && let Some(name) = state.stdlib_path.file_name().and_then(|n| n.to_str()) + && !names.iter().any(|(n, _, _)| n == name) + { + names.push(( + name.to_string(), + FILETYPE_REGULAR_FILE, + inode(state.stdlib_path), + )); + } + names.sort_by(|a, b| a.0.cmp(&b.0)); + list.extend(names); + if let Ok(Fd::Dir(d)) = c.data_mut().fd_mut(fd) { + d.entries = Some(list.clone()); + } + list + } + }; + let cap = usize::try_from(len).map_err(|_| EINVAL)?; + let start = usize::try_from(cookie).map_err(|_| EINVAL)?; + let mut out = Vec::with_capacity(cap.min(64 * 1024)); + for (i, (name, ty, ino)) in entries.iter().enumerate().skip(start) { + if out.len() >= cap { + break; + } + let mut header = [0u8; 24]; + header[0..8].copy_from_slice(&((i + 1) as u64).to_le_bytes()); + header[8..16].copy_from_slice(&ino.to_le_bytes()); + header[16..20].copy_from_slice(&(name.len() as u32).to_le_bytes()); + header[20] = *ty; + out.extend_from_slice(&header); + out.extend_from_slice(name.as_bytes()); + } + out.truncate(cap); + write_bytes(c, buf, &out)?; + write_u32(c, used, out.len() as u32) +} + +async fn path_open( + c: &mut Caller<'_, GuestState>, + dirfd: u32, + raw: &[u8], + oflags: u16, + rights: u64, + fdflags: u16, +) -> Result { + let path = c.data().resolve(dirfd, raw)?; + let want_dir = oflags & OFLAGS_DIRECTORY != 0; + let creat = oflags & OFLAGS_CREAT != 0; + let excl = oflags & OFLAGS_EXCL != 0; + let trunc = oflags & OFLAGS_TRUNC != 0; + let append = fdflags & FDFLAGS_APPEND != 0; + let write = rights & RIGHTS_FD_WRITE != 0 || trunc || append; + let read = rights & (RIGHTS_FD_READ | RIGHTS_FD_READDIR) != 0 || !write; + + if c.data().is_stdlib(&path) { + if write || creat || trunc { + return Err(EROFS); + } + if want_dir { + return Err(ENOTDIR); + } + if creat && excl { + return Err(EEXIST); + } + let stdlib = c.data().stdlib; + return c.data_mut().alloc_fd(Fd::File(Box::new(OpenFile { + path, + data: Cow::Borrowed(stdlib), + pos: 0, + read: true, + write: false, + append: false, + dirty: false, + modified: UNIX_EPOCH, + }))); + } + + let fs = c.data().fs.clone(); + let meta = fs.stat(&path).await; + match meta { + Ok(m) if m.file_type.is_dir() => { + if creat && excl { + return Err(EEXIST); + } + if write { + return Err(EISDIR); + } + c.data_mut().alloc_fd(Fd::Dir(OpenDir { + path, + entries: None, + })) + } + Ok(m) => { + if want_dir { + return Err(ENOTDIR); + } + if creat && excl { + return Err(EEXIST); + } + let data = if trunc { + Vec::new() + } else { + let bytes = fs.read_file(&path).await.map_err(|e| map_fs_error(&e))?; + c.data_mut().reserve(bytes.len())?; + bytes + }; + c.data_mut().alloc_fd(Fd::File(Box::new(OpenFile { + path, + data: Cow::Owned(data), + pos: 0, + read, + write, + append, + dirty: trunc && m.size > 0, + modified: m.modified, + }))) + } + Err(e) => { + if c.data().is_stdlib_ancestor(&path) { + if want_dir || !write { + return c.data_mut().alloc_fd(Fd::Dir(OpenDir { + path, + entries: None, + })); + } + return Err(EISDIR); + } + let errno = map_fs_error(&e); + if errno != ENOENT || !creat || want_dir { + return Err(errno); + } + // O_CREAT: the parent must be an existing directory. + let parent = path.parent().unwrap_or(Path::new("/")); + match fs.stat(parent).await { + Ok(p) if p.file_type.is_dir() => {} + Ok(_) => return Err(ENOTDIR), + Err(_) => return Err(ENOENT), + } + fs.write_file(&path, b"") + .await + .map_err(|e| map_fs_error(&e))?; + c.data_mut().alloc_fd(Fd::File(Box::new(OpenFile { + path, + data: Cow::Owned(Vec::new()), + pos: 0, + read, + write, + append, + dirty: false, + modified: SystemTime::now(), + }))) + } + } +} + +async fn poll_oneoff( + c: &mut Caller<'_, GuestState>, + subs: i32, + events: i32, + n: i32, + nevents: i32, +) -> Result<(), Errno> { + let n = u32::try_from(n).map_err(|_| EINVAL)?; + if n == 0 || n > 64 { + return Err(EINVAL); + } + let raw = read_bytes(c, subs, i32::try_from(n * 48).map_err(|_| EINVAL)?)?; + let mut ready = Vec::new(); + let mut sleep: Option<(Duration, u64)> = None; + for sub in raw.chunks_exact(48) { + let userdata = u64::from_le_bytes(sub[0..8].try_into().map_err(|_| EINVAL)?); + match sub[8] { + EVENTTYPE_CLOCK => { + let clock = u32::from_le_bytes(sub[16..20].try_into().map_err(|_| EINVAL)?); + let timeout = u64::from_le_bytes(sub[24..32].try_into().map_err(|_| EINVAL)?); + let flags = u16::from_le_bytes(sub[40..42].try_into().map_err(|_| EINVAL)?); + let wait = if flags & SUBCLOCKFLAGS_ABSTIME != 0 { + let now = if clock == CLOCK_REALTIME { + nanos(SystemTime::now()) + } else { + monotonic_nanos() + }; + timeout.saturating_sub(now) + } else { + timeout + }; + let wait = Duration::from_nanos(wait); + if sleep.is_none_or(|(d, _)| wait < d) { + sleep = Some((wait, userdata)); + } + } + kind @ (EVENTTYPE_FD_READ | EVENTTYPE_FD_WRITE) => { + let fd = u32::from_le_bytes(sub[16..20].try_into().map_err(|_| EINVAL)?); + let (error, nbytes) = match c.data().fd(fd) { + Ok(Fd::Stdin) => (SUCCESS, (c.data().stdin.len() - c.data().stdin_pos) as u64), + Ok(Fd::File(f)) => { + (SUCCESS, f.data.len().saturating_sub(f.pos as usize) as u64) + } + Ok(_) => (SUCCESS, 0), + Err(e) => (e, 0), + }; + ready.push((userdata, error, kind, nbytes)); + } + _ => return Err(EINVAL), + } + } + if ready.is_empty() + && let Some((wait, userdata)) = sleep + { + // Sleep in slices, never past the call deadline; the caller's + // timeout/budget wrapper observes every wake-up. + let target = crate::time_compat::Instant::now() + wait; + loop { + let now = crate::time_compat::Instant::now(); + if now >= target || now >= c.data().deadline { + break; + } + let slice = (target - now).min(MAX_SLEEP_SLICE); + tokio::time::sleep(slice).await; + } + ready.push((userdata, SUCCESS, EVENTTYPE_CLOCK, 0)); + } + let mut out = Vec::with_capacity(ready.len() * 32); + for (userdata, error, kind, nbytes) in &ready { + let mut e = [0u8; 32]; + e[0..8].copy_from_slice(&userdata.to_le_bytes()); + e[8..10].copy_from_slice(&error.to_le_bytes()); + e[10] = *kind; + e[16..24].copy_from_slice(&nbytes.to_le_bytes()); + out.extend_from_slice(&e); + } + write_bytes(c, events, &out)?; + write_u32(c, nevents, ready.len() as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_clamps_parent_traversal() { + assert_eq!( + normalize(Path::new("/"), "../../etc/passwd"), + PathBuf::from("/etc/passwd") + ); + assert_eq!(normalize(Path::new("/a/b"), "../c"), PathBuf::from("/a/c")); + assert_eq!( + normalize(Path::new("/a"), "./x/./y/.."), + PathBuf::from("/a/x") + ); + assert_eq!(normalize(Path::new("/a"), "/abs"), PathBuf::from("/abs")); + assert_eq!(normalize(Path::new("/"), ""), PathBuf::from("/")); + } + + #[test] + fn output_cap_truncates_and_flags() { + let mut out = Output { + cap: 4, + ..Output::default() + }; + out.write(1, b"abc"); + out.write(2, b"def"); + assert_eq!(out.stdout, b"abc"); + assert_eq!(out.stderr, b"d"); + assert!(out.truncated); + } + + #[test] + fn fs_errors_map_to_errno() { + let nf: crate::Error = std::io::Error::new(std::io::ErrorKind::NotFound, "x").into(); + assert_eq!(map_fs_error(&nf), ENOENT); + let ex: crate::Error = std::io::Error::new(std::io::ErrorKind::AlreadyExists, "x").into(); + assert_eq!(map_fs_error(&ex), EEXIST); + let dir: crate::Error = std::io::Error::other("is a directory").into(); + assert_eq!(map_fs_error(&dir), EISDIR); + let other: crate::Error = std::io::Error::other("boom").into(); + assert_eq!(map_fs_error(&other), EIO); + } +} diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index b9b2ba030..86510091d 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -130,9 +130,11 @@ pub(crate) mod git; pub(crate) mod ssh; -#[cfg(any(feature = "python", feature = "typescript"))] +#[cfg(any(feature = "python", feature = "typescript", feature = "cpython"))] mod runtime_limits; +#[cfg(feature = "cpython")] +mod cpython; #[cfg(feature = "python")] mod python; @@ -241,9 +243,11 @@ pub use git::Git; #[cfg(feature = "ssh")] pub use ssh::{Scp, Sftp, Ssh}; -#[cfg(any(feature = "python", feature = "typescript"))] +#[cfg(any(feature = "python", feature = "typescript", feature = "cpython"))] pub use runtime_limits::RuntimeLimits; +#[cfg(feature = "cpython")] +pub use cpython::{CPython, CPythonLimits}; #[cfg(feature = "python")] pub(crate) use python::PythonInprocessOptIn; #[cfg(feature = "python")] diff --git a/crates/bashkit/src/lib.rs b/crates/bashkit/src/lib.rs index fc82b6ded..8a15dbe64 100644 --- a/crates/bashkit/src/lib.rs +++ b/crates/bashkit/src/lib.rs @@ -16,6 +16,7 @@ //! - **Async-first** - Built on tokio //! - **Experimental: Git** - Virtual git operations on the VFS (`git` feature) //! - **Experimental: Python** - Embedded Python via [Monty](https://github.com/pydantic/monty) (`python` feature) +//! - **Python (CPython)** - Real CPython 3.14 in a WebAssembly sandbox (`cpython` feature) //! - **Experimental: SQLite** - Embedded SQLite-compatible engine via [Turso](https://github.com/tursodatabase/turso) (`sqlite` feature) //! //! # Built-in Commands (164) @@ -40,7 +41,7 @@ //! | Structured data | `json`, `csv`, `tomlq`, `semver` | //! | Network | `curl`, `wget`, `http` (requires [`NetworkAllowlist`]) //! | Arithmetic | `bc` | -//! | Experimental | `python`, `python3` (requires `python` feature), `git` (requires `git` feature), `ts`, `typescript`, `node`, `deno`, `bun` (requires `typescript` feature), `ssh`, `scp`, `sftp` (requires `ssh` feature), `sqlite`, `sqlite3` (requires `sqlite` feature) +//! | Experimental | `python`, `python3` (requires `python` or `cpython` feature), `git` (requires `git` feature), `ts`, `typescript`, `node`, `deno`, `bun` (requires `typescript` feature), `ssh`, `scp`, `sftp` (requires `ssh` feature), `sqlite`, `sqlite3` (requires `sqlite` feature) //! //! # Shell Features //! @@ -406,6 +407,7 @@ //! - [`live_mounts_guide`] - Live mount/unmount on running instances //! - [`namespace_filesystems_guide`] - Static namespaces with rebasing and per-mount access //! - `python_guide` - Embedded Python (Monty) guide (requires `python` feature) +//! - `cpython_guide` - Embedded CPython (WebAssembly) guide (requires `cpython` feature) //! - `logging_guide` - Structured logging with security (requires `logging` feature) //! //! # Resources @@ -568,11 +570,14 @@ pub use builtins::ssh::{SshClient, SshHandler, SshOutput, SshTarget}; pub use builtins::{Python, PythonExternalFnHandler, PythonExternalFns, PythonLimits}; // Shared resource-limit core for embedded language VMs (Python, TypeScript). -#[cfg(any(feature = "python", feature = "typescript"))] +#[cfg(any(feature = "python", feature = "typescript", feature = "cpython"))] pub use builtins::RuntimeLimits; #[cfg(any(feature = "python", feature = "typescript"))] pub use runtime_call::RuntimeCallContext; +#[cfg(feature = "cpython")] +pub use builtins::{CPython, CPythonLimits}; + #[cfg(feature = "sqlite")] pub use builtins::{Sqlite, SqliteBackend, SqliteLimits}; // Re-export monty types needed by external handler consumers. @@ -2430,6 +2435,43 @@ impl BashBuilder { self.python_with_limits(limits) } + /// Enable `python`/`python3` backed by real CPython 3.14 running in a + /// WebAssembly sandbox, with default [`CPythonLimits`]. + /// + /// The guest only reaches the virtual filesystem, captured stdio, clocks + /// and a random source; there is no network, subprocess or host + /// filesystem access. Each call runs in a fresh instance mapped from a + /// pre-initialized snapshot, so no state crosses calls or tenants. + /// Requires the `cpython` feature. Replaces the Monty-backed builtins when + /// both features are enabled and this method is called after + /// [`BashBuilder::python`]. + /// + /// The interpreter loads on the first call; call [`CPython::warm_up`] at + /// process start to move that cost out of the first request. + /// + /// # Example + /// + /// ```rust,ignore + /// let bash = Bash::builder().cpython().build(); + /// bash.exec("python3 -c 'import json; print(json.dumps([1, 2]))'").await?; + /// ``` + #[cfg(feature = "cpython")] + pub fn cpython(self) -> Self { + self.cpython_with_limits(builtins::CPythonLimits::default()) + } + + /// Enable CPython `python`/`python3` with custom limits. + /// + /// See [`BashBuilder::cpython`]. + #[cfg(feature = "cpython")] + pub fn cpython_with_limits(self, limits: builtins::CPythonLimits) -> Self { + self.builtin( + "python", + Box::new(builtins::CPython::with_limits(limits.clone())), + ) + .builtin("python3", Box::new(builtins::CPython::with_limits(limits))) + } + /// Enable embedded SQLite (`sqlite`/`sqlite3` builtins) via Turso. /// /// Registers both names with the default [`SqliteLimits`]. The Turso @@ -3878,6 +3920,20 @@ pub mod threat_model {} #[doc = include_str!("../docs/python.md")] pub mod python_guide {} +/// Guide for real CPython 3.14 running as a WebAssembly guest. +/// +/// Topics covered: +/// - Quick start with `Bash::builder().cpython()` +/// - Why CPython instead of Monty +/// - Supported command line, stdio, environment and VFS behavior +/// - Resource limits via [`CPythonLimits`] +/// - Limitations (no subprocess, network, threads or third-party packages) +/// +/// **Related:** [`BashBuilder::cpython`], [`CPythonLimits`], [`CPython`], [`threat_model`] +#[cfg(feature = "cpython")] +#[doc = include_str!("../docs/cpython.md")] +pub mod cpython_guide {} + /// Guide for the embedded SQLite builtin (Turso). /// /// Topics covered: diff --git a/crates/bashkit/tests/integration/cpython_capability_tests.rs b/crates/bashkit/tests/integration/cpython_capability_tests.rs new file mode 100644 index 000000000..ce22b0880 --- /dev/null +++ b/crates/bashkit/tests/integration/cpython_capability_tests.rs @@ -0,0 +1,523 @@ +// Capability tests for the CPython (WASI) python3 builtin (`cpython` feature). +// +// One table of small programs covering the language and the stdlib modules +// scripts reach for. Each case runs through bashkit and must print exactly +// the expected output. `cases_match_host_python` runs the same table through +// the host's real `python3` (when one is installed) so the expectations stay +// honest; it skips cases that only make sense inside the sandbox. + +#![cfg(feature = "cpython")] + +use bashkit::Bash; + +struct Case { + name: &'static str, + code: &'static str, + expected: &'static str, + /// Also compare against the host interpreter (version-stable output). + host: bool, +} + +const fn case(name: &'static str, code: &'static str, expected: &'static str) -> Case { + Case { + name, + code, + expected, + host: true, + } +} + +const fn sandbox_only(name: &'static str, code: &'static str, expected: &'static str) -> Case { + Case { + name, + code, + expected, + host: false, + } +} + +const CASES: &[Case] = &[ + // --- language ------------------------------------------------------------ + case( + "classes_inheritance", + "class A:\n def hi(self): return 'A'\nclass B(A):\n def hi(self): return 'B' + super().hi()\nprint(B().hi())", + "BA\n", + ), + case( + "dunder_methods", + "class V:\n def __init__(s, x): s.x = x\n def __add__(s, o): return V(s.x + o.x)\n def __repr__(s): return f'V({s.x})'\nprint(V(1) + V(2))", + "V(3)\n", + ), + case( + "properties_and_slots", + "class P:\n __slots__ = ('_v',)\n def __init__(s): s._v = 1\n @property\n def v(s): return s._v * 10\nprint(P().v)", + "10\n", + ), + case( + "match_statement", + "def f(x):\n match x:\n case {'k': [a, *rest]}: return a, rest\n case _: return None\nprint(f({'k': [1, 2, 3]}))", + "(1, [2, 3])\n", + ), + case( + "generators_and_yield_from", + "def g():\n yield from range(3)\n yield 9\nprint(list(g()))", + "[0, 1, 2, 9]\n", + ), + case( + "closures_nonlocal", + "def c():\n n = 0\n def inc():\n nonlocal n\n n += 1\n return n\n return inc\ni = c(); i(); print(i())", + "2\n", + ), + case( + "decorators", + "import functools\ndef twice(f):\n @functools.wraps(f)\n def w(*a): return f(*a) * 2\n return w\n@twice\ndef sq(x): return x * x\nprint(sq(3), sq.__name__)", + "18 sq\n", + ), + case( + "exceptions_chaining", + "try:\n try:\n {}['x']\n except KeyError as e:\n raise ValueError('wrapped') from e\nexcept ValueError as e:\n print(type(e.__cause__).__name__, e)", + "KeyError wrapped\n", + ), + case( + "exception_groups", + "try:\n raise ExceptionGroup('g', [ValueError(1), TypeError(2)])\nexcept* ValueError as eg:\n print('v', len(eg.exceptions))\nexcept* TypeError:\n print('t')", + "v 1\nt\n", + ), + case( + "context_managers", + "import contextlib\n@contextlib.contextmanager\ndef cm():\n print('in'); yield 5; print('out')\nwith cm() as v: print(v)", + "in\n5\nout\n", + ), + case( + "comprehensions_walrus", + "print({k: v for k, v in zip('ab', [1, 2])}, [y for x in range(5) if (y := x * x) > 4])", + "{'a': 1, 'b': 2} [9, 16]\n", + ), + case( + "fstrings", + "x = 3.14159; n = 'w'\nprint(f'{x:.2f}|{n!r:>5}|{x=:.1f}|{1000000:,}')", + "3.14| 'w'|x=3.1|1,000,000\n", + ), + case( + "big_ints", + "print(2 ** 100, (10 ** 30) // 7)", + "1267650600228229401496703205376 142857142857142857142857142857\n", + ), + case( + "float_repr", + "print(0.1 + 0.2, 1e308 * 10, round(2.675, 2))", + "0.30000000000000004 inf 2.67\n", + ), + case( + "complex_numbers", + "print((1 + 2j) * (3 - 1j), abs(3 + 4j))", + "(5+5j) 5.0\n", + ), + case( + "bytes_and_bytearray", + "b = bytearray(b'abc'); b[0] = 65; print(bytes(b), b.hex(), bytes.fromhex('ff00'))", + "b'Abc' 416263 b'\\xff\\x00'\n", + ), + case( + "str_methods", + "print('a,b,,c'.split(','), ' x '.strip(), 'abc'.title(), 'ß'.upper(), 'x'.center(5, '*'))", + "['a', 'b', '', 'c'] x Abc SS **x**\n", + ), + case( + "sorting_stable_key", + "d = [('b', 2), ('a', 2), ('c', 1)]\nprint(sorted(d, key=lambda t: t[1]))", + "[('c', 1), ('b', 2), ('a', 2)]\n", + ), + case( + "dict_order_and_merge", + "a = {'x': 1}; b = {'y': 2}\nprint(a | b, list({3: 0, 1: 0, 2: 0}))", + "{'x': 1, 'y': 2} [3, 1, 2]\n", + ), + case( + "sets", + "print(sorted({1, 2, 3} & {2, 3, 4}), sorted({1} ^ {2}), frozenset() == set())", + "[2, 3] [1, 2] True\n", + ), + case( + "type_hints_runtime", + "from typing import get_type_hints\ndef f(a: int, b: 'list[str]') -> None: ...\nprint(get_type_hints(f)['a'].__name__)", + "int\n", + ), + case( + "metaclass_and_init_subclass", + "class R:\n reg = []\n def __init_subclass__(cls, **kw): R.reg.append(cls.__name__)\nclass X(R): pass\nclass Y(R): pass\nprint(R.reg)", + "['X', 'Y']\n", + ), + case( + "async_await", + "import asyncio\nasync def w(n):\n await asyncio.sleep(0)\n return n * 2\nasync def main():\n return await asyncio.gather(w(1), w(2), w(3))\nprint(asyncio.run(main()))", + "[2, 4, 6]\n", + ), + case( + "async_sleep_timers", + "import asyncio\nasync def main():\n out = []\n async def t(n, d):\n await asyncio.sleep(d); out.append(n)\n await asyncio.gather(t('slow', 0.02), t('fast', 0.0))\n return out\nprint(asyncio.run(main()))", + "['fast', 'slow']\n", + ), + case( + "async_queue_tasks", + "import asyncio\nasync def main():\n q = asyncio.Queue()\n async def prod():\n for i in range(3): await q.put(i)\n await q.put(None)\n async def cons():\n s = 0\n while (i := await q.get()) is not None: s += i\n return s\n _, s = await asyncio.gather(prod(), cons())\n return s\nprint(asyncio.run(main()))", + "3\n", + ), + case( + "eval_exec_compile", + "ns = {}\nexec(compile('y = 2 + 3', 'm', 'exec'), ns)\nprint(ns['y'], eval('y * 2', ns))", + "5 10\n", + ), + case( + "recursion_moderate", + "def f(n): return 0 if n == 0 else 1 + f(n - 1)\nprint(f(500))", + "500\n", + ), + // --- stdlib -------------------------------------------------------------- + case( + "json", + "import json\nd = json.loads('{\"a\": [1, 2.5, null, true]}')\nprint(d, json.dumps(d, sort_keys=True, indent=None))", + "{'a': [1, 2.5, None, True]} {\"a\": [1, 2.5, null, true]}\n", + ), + case( + "re", + "import re\nprint(re.findall(r'(\\w)(\\d+)', 'a1 b22'), re.sub(r'(?P\\w+)@', r'<\\g>', 'me@x'))", + "[('a', '1'), ('b', '22')] x\n", + ), + case( + "csv", + "import csv, io\nr = list(csv.DictReader(io.StringIO('a,b\\n1,\"x,y\"\\n')))\nw = io.StringIO(); csv.writer(w).writerow(['q\"', 1])\nprint(r, w.getvalue().strip())", + "[{'a': '1', 'b': 'x,y'}] \"q\"\"\",1\n", + ), + case( + "collections", + "from collections import Counter, defaultdict, deque, OrderedDict, namedtuple\nP = namedtuple('P', 'x y')\nd = deque([1, 2]); d.appendleft(0)\ndd = defaultdict(list); dd['k'].append(1)\nprint(Counter('abca').most_common(1), list(d), dict(dd), P(1, 2))", + "[('a', 2)] [0, 1, 2] {'k': [1]} P(x=1, y=2)\n", + ), + case( + "itertools", + "import itertools as it\nprint(list(it.combinations('abc', 2)), list(it.accumulate([1, 2, 3])), list(it.islice(it.count(5), 3)), list(it.chain.from_iterable([[1], [2]])))", + "[('a', 'b'), ('a', 'c'), ('b', 'c')] [1, 3, 6] [5, 6, 7] [1, 2]\n", + ), + case( + "functools", + "import functools\n@functools.lru_cache\ndef fib(n): return n if n < 2 else fib(n - 1) + fib(n - 2)\nprint(fib(80), functools.reduce(lambda a, b: a * b, range(1, 6)))", + "23416728348467685 120\n", + ), + case( + "dataclasses", + "from dataclasses import dataclass, field, asdict\n@dataclass(order=True)\nclass P:\n x: int\n tags: list = field(default_factory=list)\nprint(asdict(P(1)), P(1) < P(2))", + "{'x': 1, 'tags': []} True\n", + ), + case( + "enum", + "import enum\nclass C(enum.Enum):\n RED = 1\n BLUE = enum.auto()\nprint(C.BLUE, C(1).name, [c.value for c in C])", + "C.BLUE RED [1, 2]\n", + ), + case( + "datetime", + "from datetime import datetime, timedelta, timezone, date\nd = datetime(2020, 1, 31, 12, tzinfo=timezone.utc)\nprint((d + timedelta(days=1)).isoformat(), date(2024, 2, 29).strftime('%a %d %b'), datetime.fromisoformat('2021-05-01T10:00:00').weekday())", + "2020-02-01T12:00:00+00:00 Thu 29 Feb 5\n", + ), + case( + "decimal_fractions", + "from decimal import Decimal, getcontext\nfrom fractions import Fraction\ngetcontext().prec = 30\nprint(Decimal(1) / Decimal(7), Fraction(1, 3) + Fraction(1, 6), Decimal('0.1') + Decimal('0.2'))", + "0.142857142857142857142857142857 1/2 0.3\n", + ), + case( + "math_statistics", + "import math, statistics\nprint(math.factorial(20), math.isqrt(10 ** 10), round(math.pi, 5), statistics.median([3, 1, 2]), statistics.mean([1, 2, 3, 4]))", + "2432902008176640000 100000 3.14159 2 2.5\n", + ), + case( + "hashlib", + "import hashlib\nprint(hashlib.sha256(b'abc').hexdigest()[:16], hashlib.md5(b'x').hexdigest()[:8], hashlib.sha1(b'').hexdigest()[:8], hashlib.blake2b(b'a', digest_size=8).hexdigest())", + "ba7816bf8f01cfea 9dd4e461 da39a3ee 40f89e395b66422f\n", + ), + case( + "hmac_secrets", + "import hmac, hashlib, secrets\nprint(hmac.new(b'k', b'm', hashlib.sha256).hexdigest()[:12], len(secrets.token_hex(8)))", + "b60090e30522 16\n", + ), + case( + "base64_binascii", + "import base64, binascii\nprint(base64.b64encode(b'hi!'), base64.urlsafe_b64decode('aGk_'), binascii.crc32(b'abc'))", + "b'aGkh' b'hi?' 891568578\n", + ), + case( + "zlib_gzip", + "import zlib, gzip\nd = b'x' * 1000\nprint(len(zlib.compress(d)) < 50, zlib.decompress(zlib.compress(d)) == d, gzip.decompress(gzip.compress(b'g')))", + "True True b'g'\n", + ), + case( + "zlib_module_spec", + "import importlib.util\nprint(importlib.util.find_spec('zlib') is not None)", + "True\n", + ), + case( + "struct", + "import struct\nprint(struct.pack('H', b'\\x01\\x02'), struct.calcsize('a')\nprint(html.escape('<&>'), r.find('i').get('n'), r.find('i').text)", + "<&> 1 a\n", + ), + case( + "shlex", + "import shlex\nprint(shlex.split('a \"b c\" d\\\\ e'), shlex.quote(\"it's\"))", + "['a', 'b c', 'd e'] 'it'\"'\"'s'\n", + ), + case( + "argparse", + "import argparse\np = argparse.ArgumentParser(prog='t')\np.add_argument('--n', type=int, default=1)\np.add_argument('files', nargs='*')\nprint(p.parse_args(['--n', '3', 'a', 'b']))", + "Namespace(n=3, files=['a', 'b'])\n", + ), + case( + "logging", + "import logging, sys\nlogging.basicConfig(stream=sys.stdout, format='%(levelname)s:%(message)s', level=logging.INFO)\nlogging.getLogger('x').info('hi')\nlogging.debug('hidden')", + "INFO:hi\n", + ), + case( + "pprint", + "import pprint\npprint.pprint({'b': [1] * 3, 'a': 'x'}, width=20)", + "{'a': 'x',\n 'b': [1, 1, 1]}\n", + ), + case( + "heapq_bisect", + "import heapq, bisect\nh = [5, 1, 3]; heapq.heapify(h)\nprint(heapq.nsmallest(2, h), bisect.bisect([1, 3, 5], 4))", + "[1, 3] 2\n", + ), + case( + "copy_weakref", + "import copy, weakref\nclass O: pass\na = [[1]]; b = copy.deepcopy(a); b[0].append(2)\no = O(); r = weakref.ref(o)\nprint(a, r() is o)", + "[[1]] True\n", + ), + case( + "io_streams", + "import io\ns = io.StringIO(); print('x', file=s, end='')\nb = io.BytesIO(b'ab'); b.seek(1)\nprint(s.getvalue(), b.read())", + "x b'b'\n", + ), + case( + "typing_protocols", + "from typing import Protocol, TypedDict, NamedTuple\nclass T(TypedDict):\n a: int\nclass N(NamedTuple):\n x: int = 1\nprint(T(a=1), N())", + "{'a': 1} N(x=1)\n", + ), + case( + "abc", + "import abc\nclass A(abc.ABC):\n @abc.abstractmethod\n def f(self): ...\ntry:\n A()\nexcept TypeError:\n print('abstract')", + "abstract\n", + ), + case( + "operator_keyword", + "import operator, keyword\nprint(operator.itemgetter(1)('ab'), keyword.iskeyword('match'), keyword.issoftkeyword('match'))", + "b False True\n", + ), + case( + "calendar", + "import calendar\nprint(calendar.isleap(2024), tuple(map(int, calendar.monthrange(2023, 2))))", + "True (2, 28)\n", + ), + case( + "uuid", + "import uuid\nu = uuid.uuid4()\nprint(u.version, len(str(u)), uuid.UUID('12345678123456781234567812345678'))", + "4 36 12345678-1234-5678-1234-567812345678\n", + ), + case( + "random_seeded", + "import random\nr = random.Random(42)\nprint(r.randint(1, 100), r.choice('abc'))", + "82 a\n", + ), + case( + "tokenize_ast", + "import ast\nt = ast.parse('x = 1 + 2')\nprint(type(t.body[0]).__name__, ast.literal_eval('[1, (2, 3)]'))", + "Assign [1, (2, 3)]\n", + ), + case( + "string_io_csv_sniffer", + "import csv\nprint(csv.Sniffer().sniff('a;b\\n1;2\\n').delimiter)", + ";\n", + ), + case( + "sqlite_memory", + "import sqlite3\nc = sqlite3.connect(':memory:')\nc.execute('create table t(a, b)')\nc.executemany('insert into t values (?, ?)', [(1, 'x'), (2, 'y')])\nprint(c.execute('select sum(a), group_concat(b) from t').fetchone())", + "(3, 'x,y')\n", + ), + case( + "sqlite_json1", + "import sqlite3\nprint(sqlite3.connect(':memory:').execute(\"select json_extract('{\\\"a\\\":[1,2]}', '$.a[1]')\").fetchone()[0])", + "2\n", + ), + case( + "zipfile_in_memory", + "import zipfile, io\nb = io.BytesIO()\nwith zipfile.ZipFile(b, 'w', zipfile.ZIP_DEFLATED) as z: z.writestr('a.txt', 'hello')\nprint(zipfile.ZipFile(b).read('a.txt'))", + "b'hello'\n", + ), + case( + "tarfile_in_memory", + "import tarfile, io\nb = io.BytesIO()\nwith tarfile.open(fileobj=b, mode='w:gz') as t:\n d = b'data'; i = tarfile.TarInfo('f'); i.size = len(d); t.addfile(i, io.BytesIO(d))\nb.seek(0)\nprint(tarfile.open(fileobj=b).extractfile('f').read())", + "b'data'\n", + ), + case( + "time_basics", + "import time\nt0 = time.monotonic(); time.sleep(0.01)\nprint(time.monotonic() - t0 >= 0.01, time.time() > 1.7e9, time.strftime('%Y', time.gmtime(0)))", + "True True 1970\n", + ), + case( + "warnings_capture", + "import warnings\nwith warnings.catch_warnings(record=True) as w:\n warnings.simplefilter('always'); warnings.warn('x', DeprecationWarning)\nprint(len(w), w[0].category.__name__)", + "1 DeprecationWarning\n", + ), + case( + "traceback_format", + "import traceback\ntry:\n 1 / 0\nexcept ZeroDivisionError:\n print(traceback.format_exc().splitlines()[-1])", + "ZeroDivisionError: division by zero\n", + ), + case( + "inspect", + "import inspect\ndef f(a, b=2, *c, d): pass\nprint(inspect.signature(f))", + "(a, b=2, *c, d)\n", + ), + case( + "fnmatch_glob_patterns", + "import fnmatch\nprint(fnmatch.filter(['a.py', 'b.txt', 'c.py'], '*.py'))", + "['a.py', 'c.py']\n", + ), + case( + "os_path", + "import os.path as p\nprint(p.join('a', 'b'), p.splitext('x.tar.gz'), p.normpath('/a/./b/../c'), p.basename('/q/w.e'))", + "a/b ('x.tar', '.gz') /a/c w.e\n", + ), + case( + "email_message", + "from email.message import EmailMessage\nm = EmailMessage(); m['Subject'] = 'hi'; m.set_content('body')\nprint(m['Subject'], m.get_content().strip())", + "hi body\n", + ), + case( + "graphlib_contextvars", + "import graphlib, contextvars\nprint(list(graphlib.TopologicalSorter({'b': {'a'}}).static_order()), contextvars.ContextVar('v', default=3).get())", + "['a', 'b'] 3\n", + ), + case( + "array_memoryview", + "import array\na = array.array('i', [1, 2, 3])\nprint(a.tolist(), bytes(memoryview(b'abc')[1:]))", + "[1, 2, 3] b'bc'\n", + ), + case( + "locale_codecs", + "import codecs\nprint(codecs.encode('abc', 'rot13'), 'é'.encode('latin-1'), b'\\xe9'.decode('cp1252'))", + "nop b'\\xe9' é\n", + ), + // --- sandbox-specific behavior ------------------------------------------------ + sandbox_only( + "version_info", + "import sys\nprint(sys.version_info[:2], sys.platform, sys.implementation.name)", + "(3, 14) wasi cpython\n", + ), + sandbox_only( + "sqlite_version_bundled", + "import sqlite3\nprint(sqlite3.sqlite_version)", + "3.50.4\n", + ), + sandbox_only( + "subprocess_unavailable", + "import subprocess\ntry:\n subprocess.run(['ls'])\nexcept OSError as e:\n print('OSError')", + "OSError\n", + ), + sandbox_only( + "threads_unavailable", + "import threading\ntry:\n threading.Thread(target=print).start()\nexcept RuntimeError:\n print('no threads')", + "no threads\n", + ), + sandbox_only( + "ctypes_unavailable", + "try:\n import ctypes\nexcept ImportError:\n print('no ctypes')", + "no ctypes\n", + ), + sandbox_only( + "third_party_unavailable", + "try:\n import requests\nexcept ImportError:\n print('no requests')", + "no requests\n", + ), +]; + +async fn run_case(case: &Case) -> (String, String, i32) { + let mut bash = Bash::builder().cpython().build(); + bash.fs() + .write_file(std::path::Path::new("/case.py"), case.code.as_bytes()) + .await + .unwrap(); + let r = bash.exec("python3 /case.py").await.unwrap(); + (r.stdout.to_string(), r.stderr.to_string(), r.exit_code) +} + +#[tokio::test] +async fn capability_cases() { + let mut failures = Vec::new(); + for case in CASES { + let (stdout, stderr, code) = run_case(case).await; + if stdout != case.expected || code != 0 { + failures.push(format!( + "{}: exit {code}\n expected: {:?}\n actual: {:?}\n stderr: {}", + case.name, case.expected, stdout, stderr + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// Keep the table honest: the same programs must print the same output under +/// the host's real CPython. Skipped when no `python3` >= 3.11 is installed. +#[test] +fn cases_match_host_python() { + let Ok(probe) = std::process::Command::new("python3") + .args(["-c", "import sys; print(sys.version_info >= (3, 11))"]) + .output() + else { + eprintln!("skip: no host python3"); + return; + }; + if String::from_utf8_lossy(&probe.stdout).trim() != "True" { + eprintln!("skip: host python3 older than 3.11"); + return; + } + let mut failures = Vec::new(); + for case in CASES.iter().filter(|c| c.host) { + let out = std::process::Command::new("python3") + .args(["-I", "-c", case.code]) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + if stdout != case.expected { + failures.push(format!( + "{}: host printed {:?}, table expects {:?}\n stderr: {}", + case.name, + stdout, + case.expected, + String::from_utf8_lossy(&out.stderr) + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/bashkit/tests/integration/cpython_integration_tests.rs b/crates/bashkit/tests/integration/cpython_integration_tests.rs new file mode 100644 index 000000000..ba5cbe92d --- /dev/null +++ b/crates/bashkit/tests/integration/cpython_integration_tests.rs @@ -0,0 +1,607 @@ +// Integration tests for the CPython (WASI) python3 builtin (`cpython` feature). +// +// Bash -> CPython pipeline: CLI modes, sys.argv, stdio, exit codes, env and +// cwd, VFS bridging, per-call isolation, and interop with pipelines, +// substitutions, heredocs and conditionals. Stdlib coverage lives in +// `cpython_capability_tests.rs`; sandbox and limits in +// `cpython_security_tests.rs`. + +#![cfg(feature = "cpython")] + +use bashkit::{Bash, CPythonLimits, FileSystem, InMemoryFs}; +use std::path::Path; +use std::sync::Arc; + +fn bash() -> Bash { + Bash::builder().cpython().build() +} + +async fn run(script: &str) -> bashkit::ExecResult { + bash().exec(script).await.expect("exec") +} + +async fn ok(script: &str) -> String { + let r = run(script).await; + assert_eq!(r.exit_code, 0, "script: {script}\nstderr: {}", r.stderr); + r.stdout.to_string() +} + +// --- CLI modes --------------------------------------------------------------- + +#[tokio::test] +async fn dash_c_prints() { + assert_eq!(ok("python3 -c 'print(1 + 1)'").await, "2\n"); +} + +#[tokio::test] +async fn python_alias_registered() { + assert_eq!(ok("python -c 'print(\"alias\")'").await, "alias\n"); +} + +#[tokio::test] +async fn dash_c_argv() { + assert_eq!( + ok("python3 -c 'import sys; print(sys.argv)' x 'y z'").await, + "['-c', 'x', 'y z']\n" + ); +} + +#[tokio::test] +async fn dash_c_attached_value() { + assert_eq!(ok("python3 -c'print(7)'").await, "7\n"); +} + +#[tokio::test] +async fn script_file_and_argv() { + let out = ok("echo 'import sys; print(sys.argv)' > /s.py; python3 /s.py a b").await; + assert_eq!(out, "['/s.py', 'a', 'b']\n"); +} + +#[tokio::test] +async fn script_relative_to_cwd() { + let out = ok("mkdir -p /proj; cd /proj; echo 'print(__file__)' > run.py; python3 run.py").await; + assert_eq!(out, "run.py\n"); +} + +#[tokio::test] +async fn script_dir_on_sys_path() { + let out = ok("mkdir -p /pkg; echo 'X = 5' > /pkg/helper.py; \ + printf 'import helper\\nprint(helper.X)\\n' > /pkg/main.py; python3 /pkg/main.py") + .await; + assert_eq!(out, "5\n"); +} + +#[tokio::test] +async fn main_guard_runs() { + let out = + ok("printf 'if __name__ == \"__main__\":\\n print(\"main\")\\n' > /m.py; python3 /m.py") + .await; + assert_eq!(out, "main\n"); +} + +#[tokio::test] +async fn stdin_program_when_no_args() { + assert_eq!(ok("echo 'print(40 + 2)' | python3").await, "42\n"); +} + +#[tokio::test] +async fn dash_reads_program_from_stdin_with_args() { + let out = ok("python3 - a b <<'EOF'\nimport sys\nprint(sys.argv)\nEOF").await; + assert_eq!(out, "['-', 'a', 'b']\n"); +} + +#[tokio::test] +async fn dash_m_stdlib_module() { + let out = ok("echo '{\"b\":1,\"a\":2}' | python3 -m json.tool --sort-keys --compact").await; + assert_eq!(out, "{\"a\":2,\"b\":1}\n"); +} + +#[tokio::test] +async fn dash_m_module_from_cwd() { + let out = ok("mkdir -p /mm; cd /mm; echo 'import sys; print(\"mod\", sys.argv[1:])' > tool.py; python3 -m tool q").await; + assert_eq!(out, "mod ['q']\n"); +} + +#[tokio::test] +async fn directory_with_dunder_main() { + let out = + ok("mkdir -p /app; echo 'print(\"from main\")' > /app/__main__.py; python3 /app").await; + assert_eq!(out, "from main\n"); +} + +#[tokio::test] +async fn dash_x_skips_first_line() { + let out = ok("printf 'not python\\nprint(\"ok\")\\n' > /x.py; python3 -x /x.py").await; + assert_eq!(out, "ok\n"); +} + +#[tokio::test] +async fn ignored_compat_flags() { + assert_eq!(ok("python3 -u -B -E -s -I -c 'print(1)'").await, "1\n"); +} + +#[tokio::test] +async fn version_flags() { + assert_eq!(ok("python3 -V").await, "Python 3.14.8\n"); + assert_eq!(ok("python3 --version").await, "Python 3.14.8\n"); + assert!(ok("python3 -VV").await.starts_with("Python 3.14.8 ")); +} + +#[tokio::test] +async fn help_flag() { + assert!(ok("python3 -h").await.starts_with("usage: python3")); +} + +#[tokio::test] +async fn unknown_option_exits_2() { + let r = run("python3 -Z").await; + assert_eq!(r.exit_code, 2); + assert!(r.stderr.to_string().contains("Unknown option: -Z")); +} + +#[tokio::test] +async fn dash_c_without_value_exits_2() { + let r = run("python3 -c").await; + assert_eq!(r.exit_code, 2); + assert!( + r.stderr + .to_string() + .contains("Argument expected for the -c option") + ); +} + +#[tokio::test] +async fn missing_script_exits_2() { + let r = run("python3 /missing.py").await; + assert_eq!(r.exit_code, 2); + assert!( + r.stderr + .to_string() + .starts_with("python3: can't open file '/missing.py'") + ); +} + +#[tokio::test] +async fn warning_option_turns_warning_into_error() { + let r = run("python3 -W error -c 'import warnings; warnings.warn(\"w\")'").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.to_string().contains("UserWarning: w")); +} + +// --- exit status and errors -------------------------------------------------- + +#[tokio::test] +async fn sys_exit_codes() { + assert_eq!( + run("python3 -c 'import sys; sys.exit(3)'").await.exit_code, + 3 + ); + assert_eq!( + run("python3 -c 'import sys; sys.exit()'").await.exit_code, + 0 + ); + assert_eq!( + run("python3 -c 'raise SystemExit(True)'").await.exit_code, + 1 + ); + assert_eq!(run("python3 -c 'exit(300)'").await.exit_code, 300 & 0xff); +} + +#[tokio::test] +async fn sys_exit_message_goes_to_stderr() { + let r = run("python3 -c 'import sys; sys.exit(\"bye\")'").await; + assert_eq!(r.exit_code, 1); + assert_eq!(r.stderr, "bye\n"); +} + +#[tokio::test] +async fn os_exit_keeps_flushed_output() { + let r = run("python3 -c 'import os; print(\"a\", flush=True); os._exit(5)'").await; + assert_eq!(r.exit_code, 5); + assert_eq!(r.stdout, "a\n"); +} + +#[tokio::test] +async fn uncaught_exception_traceback_matches_cpython() { + let r = run("python3 -c 'def f(): 1/0\nf()'").await; + assert_eq!(r.exit_code, 1); + assert_eq!( + r.stderr, + "Traceback (most recent call last):\n File \"\", line 2, in \n \ + File \"\", line 1, in f\nZeroDivisionError: division by zero\n" + ); +} + +#[tokio::test] +async fn traceback_hides_driver_frames_for_scripts() { + let r = run("echo 'raise ValueError(\"boom\")' > /e.py; python3 /e.py").await; + let err = r.stderr.to_string(); + assert!(err.contains("File \"/e.py\", line 1"), "{err}"); + assert!(!err.contains("_bashkit_boot"), "{err}"); + assert!(err.ends_with("ValueError: boom\n")); +} + +#[tokio::test] +async fn syntax_error_exit_1() { + let r = run("python3 -c 'def ('").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.to_string().contains("SyntaxError")); +} + +#[tokio::test] +async fn atexit_handlers_run() { + let out = + ok("python3 -c 'import atexit; atexit.register(print, \"bye\"); print(\"hi\")'").await; + assert_eq!(out, "hi\nbye\n"); +} + +// --- stdio --------------------------------------------------------------------- + +#[tokio::test] +async fn stdin_read_and_iterate() { + assert_eq!( + ok("echo hello | python3 -c 'import sys; print(sys.stdin.read().upper(), end=\"\")'").await, + "HELLO\n" + ); + assert_eq!( + ok( + "printf '1\\n2\\n3\\n' | python3 -c 'import sys; print(sum(int(l) for l in sys.stdin))'" + ) + .await, + "6\n" + ); +} + +#[tokio::test] +async fn input_builtin() { + assert_eq!( + ok("printf 'bob\\n' | python3 -c 'print(\"hi\", input())'").await, + "hi bob\n" + ); +} + +#[tokio::test] +async fn input_eof_raises() { + let r = run("python3 -c 'input()' < /dev/null").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.to_string().contains("EOFError")); +} + +#[tokio::test] +async fn stderr_separate_from_stdout() { + let r = run("python3 -c 'import sys; print(\"o\"); print(\"e\", file=sys.stderr)'").await; + assert_eq!(r.stdout, "o\n"); + assert_eq!(r.stderr, "e\n"); +} + +#[tokio::test] +async fn binary_stdout_roundtrip() { + let r = run("python3 -c 'import sys; sys.stdout.buffer.write(bytes(range(256)))'").await; + assert_eq!( + r.stdout.as_bytes(), + (0u8..=255).collect::>().as_slice() + ); +} + +#[tokio::test] +async fn binary_stdin() { + let out = ok( + "python3 -c 'open(\"/b.bin\",\"wb\").write(b\"\\x00\\xff\\x10\")'; \ + python3 -c 'import sys; print(list(sys.stdin.buffer.read()))' < /b.bin", + ) + .await; + assert_eq!(out, "[0, 255, 16]\n"); +} + +#[tokio::test] +async fn unicode_output_is_utf8() { + assert_eq!( + ok("python3 -c 'print(\"héllo ✓ 日本\")'").await, + "héllo ✓ 日本\n" + ); + assert_eq!( + ok("python3 -c 'import sys; print(sys.stdout.encoding, sys.getfilesystemencoding())'") + .await, + "utf-8 utf-8\n" + ); +} + +// --- environment and cwd ------------------------------------------------------- + +#[tokio::test] +async fn only_exported_variables_visible() { + let out = ok("FOO=1; export BAR=2; python3 -c 'import os; print(os.environ.get(\"FOO\"), os.environ.get(\"BAR\"))'").await; + assert_eq!(out, "None 2\n"); +} + +#[tokio::test] +async fn prefix_assignment_visible() { + let out = ok("GREETING=hey python3 -c 'import os; print(os.environ[\"GREETING\"])'").await; + assert_eq!(out, "hey\n"); +} + +#[tokio::test] +async fn internal_variables_not_visible() { + let out = ok("python3 -c 'import os; print(sorted(k for k in os.environ if k.startswith(\"__BASHKIT\")))'").await; + assert_eq!(out, "[]\n"); +} + +#[tokio::test] +async fn environ_changes_do_not_leak_to_shell() { + let out = ok("export A=1; python3 -c 'import os; os.environ[\"A\"]=\"2\"; os.environ[\"B\"]=\"3\"'; echo \"$A-${B:-unset}\"").await; + assert_eq!(out, "1-unset\n"); +} + +#[tokio::test] +async fn cwd_follows_shell_cd() { + let out = ok("mkdir -p /w/sub; cd /w/sub; python3 -c 'import os; print(os.getcwd())'").await; + assert_eq!(out, "/w/sub\n"); +} + +#[tokio::test] +async fn python_chdir_does_not_move_shell() { + let out = ok("mkdir -p /a /b; cd /a; python3 -c 'import os; os.chdir(\"/b\")'; pwd").await; + assert_eq!(out, "/a\n"); +} + +// --- VFS bridging -------------------------------------------------------------- + +#[tokio::test] +async fn python_writes_shell_reads() { + let out = + ok("python3 -c 'open(\"/out.txt\", \"w\").write(\"from py\\n\")'; cat /out.txt").await; + assert_eq!(out, "from py\n"); +} + +#[tokio::test] +async fn shell_writes_python_reads() { + let out = + ok("echo 'from sh' > /in.txt; python3 -c 'print(open(\"/in.txt\").read().strip())'").await; + assert_eq!(out, "from sh\n"); +} + +#[tokio::test] +async fn relative_write_lands_in_cwd() { + let out = + ok("mkdir -p /w; cd /w; python3 -c 'open(\"rel.txt\",\"w\").write(\"r\")'; cat /w/rel.txt") + .await; + assert_eq!(out, "r"); +} + +#[tokio::test] +async fn unclosed_file_is_flushed_at_exit() { + let out = ok("python3 -c 'f = open(\"/u.txt\", \"w\"); f.write(\"data\")'; cat /u.txt").await; + assert_eq!(out, "data"); +} + +#[tokio::test] +async fn append_mode() { + let out = ok( + "echo one > /ap.txt; python3 -c 'open(\"/ap.txt\",\"a\").write(\"two\\n\")'; cat /ap.txt", + ) + .await; + assert_eq!(out, "one\ntwo\n"); +} + +#[tokio::test] +async fn read_write_seek() { + let out = ok("python3 -c ' +with open(\"/rw.bin\", \"w+b\") as f: + f.write(b\"hello world\") + f.seek(6) + print(f.read()) + f.seek(0) + f.write(b\"J\") +print(open(\"/rw.bin\",\"rb\").read()) +'") + .await; + assert_eq!(out, "b'world'\nb'Jello world'\n"); +} + +#[tokio::test] +async fn binary_file_roundtrip() { + let out = + ok("python3 -c 'open(\"/b.bin\",\"wb\").write(bytes(range(256)))'; wc -c < /b.bin").await; + assert_eq!(out.trim(), "256"); +} + +#[tokio::test] +async fn directory_operations() { + let out = ok("python3 -c ' +import os, shutil +os.makedirs(\"/d/x/y\", exist_ok=True) +open(\"/d/x/y/f.txt\",\"w\").write(\"1\") +os.rename(\"/d/x/y/f.txt\", \"/d/x/g.txt\") +print(sorted(os.listdir(\"/d/x\"))) +shutil.copy(\"/d/x/g.txt\", \"/d/copy.txt\") +os.remove(\"/d/x/g.txt\") +os.rmdir(\"/d/x/y\") +print(sorted(os.listdir(\"/d\")), sorted(os.listdir(\"/d/x\"))) +'; cat /d/copy.txt") + .await; + assert_eq!(out, "['g.txt', 'y']\n['copy.txt', 'x'] []\n1"); +} + +#[tokio::test] +async fn walk_glob_and_stat() { + let out = ok( + "mkdir -p /t/a /t/b; echo 12345 > /t/a/1.txt; echo x > /t/b/2.log; python3 -c ' +import os, glob +print(sorted(glob.glob(\"/t/*/*.txt\"))) +print(sum(len(fs) for _, _, fs in os.walk(\"/t\"))) +st = os.stat(\"/t/a/1.txt\") +print(st.st_size, os.path.isfile(\"/t/a/1.txt\"), os.path.isdir(\"/t/a\")) +'", + ) + .await; + assert_eq!(out, "['/t/a/1.txt']\n2\n6 True True\n"); +} + +#[tokio::test] +async fn pathlib_operations() { + let out = ok("python3 -c ' +from pathlib import Path +p = Path(\"/pl/dir\") +p.mkdir(parents=True) +(p / \"a.txt\").write_text(\"alpha\") +print((p / \"a.txt\").read_text(), [c.name for c in p.iterdir()], (p / \"a.txt\").exists()) +'") + .await; + assert_eq!(out, "alpha ['a.txt'] True\n"); +} + +#[tokio::test] +async fn missing_file_raises_file_not_found() { + let r = run("python3 -c 'open(\"/nope.txt\")'").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.to_string().contains("FileNotFoundError")); +} + +#[tokio::test] +async fn unicode_file_names() { + let out = ok("python3 -c 'open(\"/données.txt\",\"w\").write(\"ü\")'; cat /données.txt").await; + assert_eq!(out, "ü"); +} + +#[tokio::test] +async fn sqlite_database_on_vfs_persists() { + let out = ok("python3 -c ' +import sqlite3 +c = sqlite3.connect(\"/db.sqlite\") +c.execute(\"create table t(x)\") +c.execute(\"insert into t values (42)\") +c.commit(); c.close() +'; python3 -c 'import sqlite3; print(sqlite3.connect(\"/db.sqlite\").execute(\"select x from t\").fetchone())'") + .await; + assert_eq!(out, "(42,)\n"); +} + +#[tokio::test] +async fn custom_filesystem_is_used() { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(Path::new("/seed.txt"), b"seeded") + .await + .unwrap(); + let mut bash = Bash::builder().fs(fs.clone()).cpython().build(); + let r = bash + .exec("python3 -c 'print(open(\"/seed.txt\").read()); open(\"/made.txt\",\"w\").write(\"m\")'") + .await + .unwrap(); + assert_eq!(r.stdout, "seeded\n"); + assert_eq!(fs.read_file(Path::new("/made.txt")).await.unwrap(), b"m"); +} + +#[tokio::test] +async fn stdlib_zip_is_not_in_the_vfs() { + let out = ok("python3 -c 'import os; print(os.listdir(\"/usr/local/lib\"))'; ls /usr/local/lib 2>/dev/null || echo absent").await; + assert_eq!(out, "['python314.zip']\nabsent\n"); +} + +// --- per-call isolation ---------------------------------------------------------- + +#[tokio::test] +async fn interpreter_state_does_not_persist_between_calls() { + let out = ok("python3 -c 'import builtins, json, sys; builtins.LEAK = 1; json.LEAK = 2; sys.modules[\"evil\"] = 1'; \ + python3 -c 'import builtins, json, sys; print(hasattr(builtins, \"LEAK\"), hasattr(json, \"LEAK\"), \"evil\" in sys.modules)'") + .await; + assert_eq!(out, "False False False\n"); +} + +#[tokio::test] +async fn local_modules_are_reimported_each_call() { + let out = ok( + "mkdir -p /m; cd /m; echo 'V = 1' > ver.py; python3 -c 'import ver; print(ver.V)'; \ + echo 'V = 2' > ver.py; python3 -c 'import ver; print(ver.V)'", + ) + .await; + assert_eq!(out, "1\n2\n"); +} + +#[tokio::test] +async fn random_is_reseeded_per_call() { + let out = ok("a=$(python3 -c 'import random; print(random.random())'); \ + b=$(python3 -c 'import random; print(random.random())'); [ \"$a\" != \"$b\" ] && echo differ") + .await; + assert_eq!(out, "differ\n"); +} + +#[tokio::test] +async fn separate_bash_instances_share_nothing() { + let mut a = bash(); + let mut b = bash(); + a.exec("python3 -c 'open(\"/only-a.txt\",\"w\").write(\"a\")'") + .await + .unwrap(); + let r = b + .exec("python3 -c 'import os; print(os.path.exists(\"/only-a.txt\"))'") + .await + .unwrap(); + assert_eq!(r.stdout, "False\n"); +} + +// --- bash interop ---------------------------------------------------------------- + +#[tokio::test] +async fn pipeline_into_shell_tools() { + let out = ok("python3 -c 'for i in range(5): print(i)' | tail -n 2 | tr '\\n' ,").await; + assert_eq!(out, "3,4,"); +} + +#[tokio::test] +async fn pipeline_between_python_calls() { + let out = ok("python3 -c 'print(\"a b c\")' | python3 -c 'import sys; print(len(sys.stdin.read().split()))'").await; + assert_eq!(out, "3\n"); +} + +#[tokio::test] +async fn command_substitution() { + let out = ok("n=$(python3 -c 'print(6*7)'); echo \"n=$n\"").await; + assert_eq!(out, "n=42\n"); +} + +#[tokio::test] +async fn conditionals_use_exit_status() { + let out = ok( + "if python3 -c 'import sys; sys.exit(1)'; then echo yes; else echo no; fi; \ + python3 -c 'pass' && echo and; python3 -c 'raise SystemExit(2)' || echo \"or $?\"", + ) + .await; + assert_eq!(out, "no\nand\nor 2\n"); +} + +#[tokio::test] +async fn errexit_stops_on_python_failure() { + let r = run("set -e; python3 -c 'raise SystemExit(4)'; echo unreachable").await; + assert_eq!(r.exit_code, 4); + assert!(!r.stdout.to_string().contains("unreachable")); +} + +#[tokio::test] +async fn heredoc_script() { + let out = ok("python3 <<'EOF'\nimport json\nprint(json.dumps({\"k\": [1, 2]}))\nEOF").await; + assert_eq!(out, "{\"k\": [1, 2]}\n"); +} + +#[tokio::test] +async fn redirect_python_output_to_file() { + let out = ok("python3 -c 'print(\"to file\")' > /r.txt 2>/r.err; cat /r.txt").await; + assert_eq!(out, "to file\n"); +} + +#[tokio::test] +async fn loop_of_calls() { + let out = ok("for i in 1 2 3; do python3 -c \"print($i * 10)\"; done").await; + assert_eq!(out, "10\n20\n30\n"); +} + +#[tokio::test] +async fn custom_limits_builder() { + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_recursion(120)) + .build(); + let r = bash + .exec("python3 -c 'import sys; print(sys.getrecursionlimit())'") + .await + .unwrap(); + assert_eq!(r.stdout, "120\n"); +} diff --git a/crates/bashkit/tests/integration/cpython_security_tests.rs b/crates/bashkit/tests/integration/cpython_security_tests.rs new file mode 100644 index 000000000..5ea792ed5 --- /dev/null +++ b/crates/bashkit/tests/integration/cpython_security_tests.rs @@ -0,0 +1,626 @@ +// Security tests for the CPython (WASI) python3 builtin (`cpython` feature). +// +// Threat IDs refer to knowledge/security/threat-model.md (TM-PY-CPY-*). +// Covers: host filesystem/network/process isolation, stdlib integrity, +// wall-clock, memory, output, fd and file-size limits, cancellation, crash +// containment, per-call and per-tenant isolation, error-message hygiene +// (TM-INF-022) and bounded property-based fuzzing. + +#![cfg(feature = "cpython")] + +use bashkit::testing::{assert_no_leak, input_echo_would_trip}; +use bashkit::{Bash, CPythonLimits, ExecutionLimits, FsLimits, InMemoryFs}; +use proptest::prelude::*; +use std::sync::atomic::Ordering; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +fn bash() -> Bash { + Bash::builder().cpython().build() +} + +fn bash_limits(limits: CPythonLimits) -> Bash { + Bash::builder().cpython_with_limits(limits).build() +} + +async fn run(script: &str) -> bashkit::ExecResult { + let r = bash().exec(script).await.expect("exec"); + assert_no_leak(&r, script, &[]); + r +} + +fn stderr(r: &bashkit::ExecResult) -> String { + r.stderr.to_string() +} + +// --- TM-PY-CPY-001: host filesystem is unreachable ----------------------------- + +#[tokio::test] +async fn host_files_are_not_reachable() { + // A file that certainly exists on the host, by absolute path. + let host = std::env::current_exe().unwrap(); + let script = format!( + "python3 -c 'import os, sys; p = sys.argv[1]; print(os.path.exists(p))' '{}'", + host.display() + ); + let r = run(&script).await; + assert_eq!(r.stdout, "False\n"); +} + +#[tokio::test] +async fn parent_traversal_is_clamped_at_vfs_root() { + let r = run("echo vfs > /inside.txt; mkdir -p /a/b; cd /a/b; \ + python3 -c 'print(open(\"../../../../../../inside.txt\").read().strip()); import os; print(os.path.realpath(\"../../../..\"))'") + .await; + assert_eq!(r.stdout, "vfs\n/\n"); +} + +#[tokio::test] +async fn host_system_paths_absent() { + let r = run("python3 -c ' +import os +for p in [\"/etc/passwd\", \"/proc/self/environ\", \"/proc/self/mem\", \"/sys\", \"/root\"]: + print(p, os.path.exists(p)) +'") + .await; + assert_eq!( + r.stdout, + "/etc/passwd False\n/proc/self/environ False\n/proc/self/mem False\n/sys False\n/root False\n" + ); +} + +#[tokio::test] +async fn symlinks_are_not_followed() { + // L-FS-001 / TM-ESC-002: the VFS stores symlinks but never follows them. + let r = run("echo secret > /target; ln -s /target /link; \ + python3 -c 'import os; print(os.path.islink(\"/link\"), os.readlink(\"/link\")) +try: + open(\"/link\").read() +except OSError: + print(\"blocked\")'") + .await; + assert_eq!(r.stdout, "True /target\nblocked\n"); +} + +#[tokio::test] +async fn host_environment_not_visible() { + // Only exported shell variables (plus PWD) reach the guest; the host + // process environment (HOME, PATH, CARGO_*, ...) never does. + let r = run("python3 -c 'import os; print(sorted(os.environ))'").await; + assert_eq!(r.stdout, "['PWD']\n"); +} + +// --- TM-PY-CPY-002: stdlib integrity ----------------------------------------------- + +#[tokio::test] +async fn stdlib_zip_is_read_only() { + let r = run("python3 -c ' +import os +z = \"/usr/local/lib/python314.zip\" +for op in (lambda: open(z, \"w\"), lambda: open(z, \"ab\"), lambda: os.remove(z), + lambda: os.rename(z, \"/z\"), lambda: os.truncate(z, 0), + lambda: open(\"/usr/local/lib/x.py\", \"w\"), lambda: os.mkdir(\"/usr/local/lib/m\"), + lambda: os.rmdir(\"/usr/local/lib\")): + try: + op(); print(\"ALLOWED\") + except OSError: + print(\"denied\") +print(len(open(z, \"rb\").read()) > 1000000) +'") + .await; + assert_eq!(r.stdout, "denied\n".repeat(8) + "True\n"); +} + +#[tokio::test] +async fn vfs_files_cannot_shadow_stdlib() { + // A tenant can put files under /usr/local/lib in its own VFS, but the zip + // path always serves the embedded stdlib, files next to it are not on + // sys.path, and preloaded modules come from the snapshot. + let r = run("mkdir -p /usr/local/lib; echo 'print(\"pwned\")' > /usr/local/lib/json.py; \ + echo 'print(\"pwned\")' > /usr/local/lib/evilmod.py; \ + python3 -c 'import json, os; print(json.dumps(1), open(\"/usr/local/lib/python314.zip\", \"rb\").read(2)) +try: + import evilmod +except ImportError: + print(\"not importable\")'") + .await; + assert_eq!(r.stdout, "1 b'PK'\nnot importable\n"); +} + +// --- TM-PY-CPY-003: no network, processes, threads or native code --------------- + +#[tokio::test] +async fn network_unavailable() { + let r = run("python3 -c ' +import socket +for f in (lambda: socket.create_connection((\"127.0.0.1\", 80), timeout=1), + lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM), + lambda: socket.getaddrinfo(\"example.com\", 80)): + try: + f(); print(\"ALLOWED\") + except (OSError, AttributeError): + print(\"denied\") +'") + .await; + assert_eq!(r.stdout, "denied\ndenied\ndenied\n"); +} + +#[tokio::test] +async fn urllib_cannot_fetch() { + // Real CPython prints a long traceback here; only the outcome matters. + let r = bash() + .exec("python3 -c 'import urllib.request; urllib.request.urlopen(\"http://example.com\")'") + .await + .unwrap(); + assert_eq!(r.exit_code, 1); + assert!(r.stdout.is_empty()); +} + +#[tokio::test] +async fn processes_unavailable() { + let r = run("python3 -c ' +import os, subprocess +for f in (lambda: subprocess.run([\"ls\"]), lambda: os.system(\"ls\"), lambda: os.fork(), + lambda: os.execv(\"/bin/sh\", [\"sh\"]), lambda: os.popen(\"ls\")): + try: + f(); print(\"ALLOWED\") + except (OSError, AttributeError): + print(\"denied\") +'") + .await; + assert_eq!(r.stdout, "denied\n".repeat(5)); +} + +#[tokio::test] +async fn threads_and_native_code_unavailable() { + let r = run("python3 -c ' +import threading +try: + threading.Thread(target=print).start(); print(\"ALLOWED\") +except RuntimeError: + print(\"denied\") +for m in (\"ctypes\", \"_ctypes\", \"mmap\", \"multiprocessing\", \"_posixsubprocess\"): + try: + __import__(m); print(m, \"ALLOWED\") + except ImportError: + print(\"denied\") +'") + .await; + assert_eq!(r.stdout, "denied\n".repeat(6)); +} + +// --- TM-PY-CPY-004: CPU time ---------------------------------------------------------- + +#[tokio::test] +async fn infinite_loop_times_out() { + let mut bash = bash_limits(CPythonLimits::default().max_duration(Duration::from_millis(300))); + let start = Instant::now(); + let r = bash + .exec("python3 -c 'while True: pass'; echo \"after $?\"") + .await + .unwrap(); + assert!( + start.elapsed() < Duration::from_secs(10), + "{:?}", + start.elapsed() + ); + assert_eq!(r.stdout, "after 124\n"); + assert!(stderr(&r).contains("python3: execution timed out")); + assert_no_leak(&r, "timeout", &[]); +} + +#[tokio::test] +async fn sleep_is_bounded_by_deadline() { + let mut bash = bash_limits(CPythonLimits::default().max_duration(Duration::from_millis(300))); + let start = Instant::now(); + let r = bash + .exec("python3 -c 'import time; time.sleep(3600)'") + .await + .unwrap(); + assert!(start.elapsed() < Duration::from_secs(10)); + assert_eq!(r.exit_code, 124); +} + +#[tokio::test] +async fn swallowing_exceptions_cannot_escape_timeout() { + let mut bash = bash_limits(CPythonLimits::default().max_duration(Duration::from_millis(300))); + let r = bash + .exec( + "python3 -c ' +while True: + try: + while True: pass + except BaseException: + pass'", + ) + .await + .unwrap(); + assert_eq!(r.exit_code, 124); +} + +#[tokio::test] +async fn shell_timeout_tighter_than_python_limit_wins() { + let mut bash = Bash::builder() + .cpython() + .limits(ExecutionLimits::new().timeout(Duration::from_millis(400))) + .build(); + let start = Instant::now(); + let _ = bash.exec("python3 -c 'while True: pass'").await; + assert!( + start.elapsed() < Duration::from_secs(10), + "{:?}", + start.elapsed() + ); +} + +#[tokio::test] +async fn cancellation_stops_busy_guest() { + let mut bash = bash(); + let token = bash.cancellation_token(); + let cancel = token.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(300)).await; + cancel.store(true, Ordering::Relaxed); + }); + let start = Instant::now(); + let r = bash.exec("python3 -c 'while True: pass'").await; + assert!( + start.elapsed() < Duration::from_secs(10), + "{:?}", + start.elapsed() + ); + assert_eq!(r.unwrap_err().to_string(), "execution cancelled"); +} + +// --- TM-PY-CPY-005: memory -------------------------------------------------------------- + +#[tokio::test] +async fn huge_allocation_raises_memory_error() { + let r = run("python3 -c 'b = bytearray(1 << 30)'").await; + assert_eq!(r.exit_code, 1); + assert!(stderr(&r).ends_with("MemoryError\n"), "{}", stderr(&r)); +} + +#[tokio::test] +async fn incremental_growth_raises_memory_error() { + let r = run("python3 -c ' +l = [] +while True: + l.append(\"x\" * 100000)'") + .await; + assert_eq!(r.exit_code, 1); + assert!(stderr(&r).contains("MemoryError"), "{}", stderr(&r)); +} + +#[tokio::test] +async fn memory_error_is_catchable_and_call_continues() { + let r = run("python3 -c ' +try: + b = bytearray(1 << 30) +except MemoryError: + print(\"caught\") +print(sum(range(10)))'") + .await; + assert_eq!(r.stdout, "caught\n45\n"); +} + +#[tokio::test] +async fn memory_cap_below_snapshot_fails_cleanly() { + let mut bash = bash_limits(CPythonLimits::default().max_memory(1 << 20)); + let r = bash + .exec("python3 -c 'print(1)'; echo after") + .await + .unwrap(); + assert_eq!(r.stdout, "after\n"); + assert!( + stderr(&r).starts_with("python3: fatal error:"), + "{}", + stderr(&r) + ); + assert_no_leak(&r, "tiny memory", &[]); +} + +#[tokio::test] +async fn file_buffers_share_the_memory_budget() { + // Open files are host-side buffers; they are capped so a guest cannot + // grow host memory past its own budget. + let mut bash = Bash::builder() + .fs(Arc::new(InMemoryFs::with_limits( + FsLimits::new() + .max_file_size(1 << 30) + .max_total_bytes(1 << 31), + ))) + .cpython_with_limits(CPythonLimits::default().max_memory(96 << 20)) + .build(); + let r = bash + .exec( + "python3 -c ' +f = open(\"/big\", \"wb\") +chunk = b\"x\" * (1 << 20) +try: + for _ in range(4096): + f.write(chunk); f.flush() + print(\"ALLOWED\") +except OSError: + print(\"capped\")'", + ) + .await + .unwrap(); + assert_eq!(r.stdout, "capped\n", "{}", r.stderr); +} + +// --- TM-PY-CPY-006: output, files, descriptors ---------------------------------------------- + +#[tokio::test] +async fn output_is_capped_and_marked_truncated() { + let mut bash = bash_limits(CPythonLimits::default().max_output(1000)); + let r = bash + .exec("python3 -c 'import sys; sys.stdout.write(\"x\" * 100000); sys.stderr.write(\"e\" * 100000)'") + .await + .unwrap(); + let note = "python3: output truncated at 1000 bytes\n"; + assert!(r.stdout.len() + r.stderr.len() <= 1000 + note.len()); + assert!(stderr(&r).ends_with(note), "{}", stderr(&r)); +} + +#[tokio::test] +async fn output_flood_loop_is_bounded() { + let mut bash = bash_limits( + CPythonLimits::default() + .max_output(4096) + .max_duration(Duration::from_millis(500)), + ); + let r = bash + .exec("python3 -c 'while True: print(\"y\" * 1000)'") + .await + .unwrap(); + assert!(r.stdout.len() <= 4096); + assert_eq!(r.exit_code, 124); +} + +#[tokio::test] +async fn vfs_file_size_limit_applies() { + let mut bash = Bash::builder() + .fs(Arc::new(InMemoryFs::with_limits( + FsLimits::new().max_file_size(1 << 20), + ))) + .cpython() + .build(); + let r = bash + .exec( + "python3 -c ' +try: + with open(\"/f\", \"wb\") as f: f.write(b\"x\" * (2 << 20)) + print(\"ALLOWED\") +except OSError: + print(\"too large\")'", + ) + .await + .unwrap(); + assert_eq!(r.stdout, "too large\n", "{}", r.stderr); +} + +#[tokio::test] +async fn descriptor_table_is_bounded() { + let r = run("python3 -c ' +fs = [] +try: + for i in range(5000): fs.append(open(\"/fd%d\" % i, \"w\")) +except OSError as e: + print(\"EMFILE\", len(fs) < 1100)'") + .await; + assert_eq!(r.stdout, "EMFILE True\n"); +} + +// --- TM-PY-CPY-007: crash containment --------------------------------------------------------- + +#[tokio::test] +async fn deep_c_recursion_is_contained() { + let r = run("python3 -c ' +l = [] +for _ in range(200000): l = [l] +print(repr(l))'; echo \"after $?\"") + .await; + assert!(r.stdout.to_string().ends_with("after 1\n"), "{}", r.stdout); + assert!( + stderr(&r).contains("python3: fatal error:"), + "{}", + stderr(&r) + ); +} + +#[tokio::test] +async fn parser_bomb_is_contained() { + let r = run("python3 -c 'eval(\"(\" * 100000 + \")\" * 100000)'; echo \"after $?\"").await; + assert_eq!(r.stdout, "after 1\n"); +} + +#[tokio::test] +async fn python_recursion_limit_enforced() { + let mut bash = bash_limits(CPythonLimits::default().max_recursion(100)); + let r = bash + .exec( + "python3 -c ' +import sys +def f(n): return f(n + 1) +try: + f(0) +except RecursionError: + print(\"limited\", sys.getrecursionlimit())'", + ) + .await + .unwrap(); + assert_eq!(r.stdout, "limited 100\n"); +} + +#[tokio::test] +async fn abort_is_contained() { + let r = run("python3 -c 'import os; os.abort()'; echo \"after $?\"").await; + assert_eq!(r.stdout, "after 1\n"); + assert_eq!(stderr(&r), "python3: fatal error: interpreter aborted\n"); +} + +#[tokio::test] +async fn shell_continues_after_guest_failures() { + let r = run("python3 -c 'import os; os.abort()'; \ + python3 -c 'b = bytearray(1 << 30)' 2>/dev/null; \ + python3 -c 'print(\"healthy\")'") + .await; + assert_eq!(r.stdout, "healthy\n"); +} + +// --- TM-PY-CPY-008: isolation across calls and tenants ----------------------------------------- + +#[tokio::test] +async fn no_state_crosses_calls() { + let r = run("python3 -c ' +import builtins, sys, os, json +builtins.open = None +sys.path.insert(0, \"/evil\") +json.dumps = lambda *a, **k: \"tampered\" +os.environ[\"X\"] = \"1\"'; \ + python3 -c 'import builtins, sys, json, os; print(builtins.open is not None, \"/evil\" in sys.path, json.dumps(1), \"X\" in os.environ)'") + .await; + assert_eq!(r.stdout, "True False 1 False\n"); +} + +#[tokio::test] +async fn concurrent_tenants_are_isolated() { + let mut handles = Vec::new(); + for i in 0..8 { + handles.push(tokio::spawn(async move { + let mut bash = bash(); + let script = format!( + "python3 -c 'open(\"/id\", \"w\").write(\"{i}\")'; \ + python3 -c 'import builtins; builtins.T = {i}; print(open(\"/id\").read())'; \ + python3 -c 'import builtins; print(hasattr(builtins, \"T\"))'" + ); + let r = bash.exec(&script).await.unwrap(); + assert_eq!(r.stdout, format!("{i}\nFalse\n")); + })); + } + for h in handles { + h.await.unwrap(); + } +} + +// --- TM-INF-022: error-message hygiene --------------------------------------------------------- + +#[tokio::test] +async fn error_paths_do_not_leak_internals() { + for script in [ + "python3 -Z", + "python3 /missing.py", + "python3 -c 'import os; os.abort()'", + "python3 -c 'raise ValueError(1)'", + "python3 -c 'def ('", + "python3 -c 'import nope'", + "python3 -c 'open(\"/usr/local/lib/python314.zip\", \"w\")'", + ] { + let r = run(script).await; + let err = stderr(&r); + for banned in [ + "wasmtime", + "wasm backtrace", + "Caller", + "GuestState", + "_bashkit_boot", + "bashkit_main", + ] { + assert!(!err.contains(banned), "{script}: leaked {banned}: {err}"); + } + } +} + +// --- bounded property-based fuzzing -------------------------------------------------------------- + +fn runtime() -> &'static tokio::runtime::Runtime { + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime") + }) +} + +fn fuzz_run(code: &str, args: &[&str]) -> bashkit::ExecResult { + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_duration(Duration::from_secs(2))) + .limits(ExecutionLimits::new().max_stderr_bytes(1024)) + .build(); + runtime().block_on(async { + bash.fs() + .write_file(std::path::Path::new("/fuzz.py"), code.as_bytes()) + .await + .unwrap(); + let mut script = String::from("python3"); + for a in args { + script.push_str(" '"); + script.push_str(&a.replace('\'', "'\\''")); + script.push('\''); + } + bash.exec(&script).await.expect("exec") + }) +} + +const FRAGMENTS: &[&str] = &[ + "import os", + "import sys", + "open('/x','w').write('a')", + "print(open('/x').read())", + "os.listdir('/')", + "os.chdir('..')", + "os.makedirs('/a/b', exist_ok=True)", + "sys.exit(3)", + "raise ValueError('v')", + "x = [1] * 1000", + "def f(n): return f(n+1)", + "f(0)", + "while False: pass", + "import json; json.loads('{')", + "print('\\x00\\xff')", + "os.remove('/x')", + "open('../../etc/passwd')", + "import socket", + "sys.stdout.buffer.write(b'\\xff')", + "os.rename('/x', '/usr/local/lib/python314.zip')", + "1/0", + "import time; time.sleep(0.001)", + "print(*range(100))", + "sys.setrecursionlimit(10**6)", + "import random; random.random()", +]; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(24))] + + /// Arbitrary text as a program never panics the host, never leaks + /// internals and always yields a well-formed exit status. + #[test] + fn arbitrary_source_is_contained(code in "\\PC{0,200}") { + prop_assume!(!input_echo_would_trip(&code)); + let r = fuzz_run(&code, &["/fuzz.py"]); + assert_no_leak(&r, "cpython arbitrary_source", &[]); + prop_assert!((0..=255).contains(&r.exit_code) || r.exit_code == 124); + } + + /// Programs stitched from filesystem/os/sys fragments stay inside the + /// VFS and the limits. + #[test] + fn fragment_programs_are_contained(idx in proptest::collection::vec(0..FRAGMENTS.len(), 1..8)) { + let code: Vec<&str> = idx.iter().map(|i| FRAGMENTS[*i]).collect(); + let code = code.join("\n"); + let r = fuzz_run(&code, &["/fuzz.py"]); + assert_no_leak(&r, "cpython fragments", &[]); + prop_assert!(!r.stdout.to_string().contains("root:")); + } + + /// Arbitrary command-line options never panic the CLI emulation. + #[test] + fn arbitrary_cli_args_are_contained(args in proptest::collection::vec("[-a-zA-Z0-9=/.]{0,6}", 0..5)) { + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let r = fuzz_run("print(1)", &refs); + assert_no_leak(&r, "cpython args", &[]); + } +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 99e474761..9640fe053 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -48,6 +48,9 @@ pub mod custom_fs_tests; pub mod memory_growth_security_tests; // Both assert named-IANA-zone behavior, which only exists with `tzdata`. // `date_timezone_no_tzdata_tests` covers the closed-to-UTC side. +pub mod cpython_capability_tests; +pub mod cpython_integration_tests; +pub mod cpython_security_tests; #[cfg(feature = "tzdata")] pub mod date_timezone_differential_tests; #[cfg(not(feature = "tzdata"))] diff --git a/deny.toml b/deny.toml index 1d6862fa8..59d41bd9c 100644 --- a/deny.toml +++ b/deny.toml @@ -17,6 +17,7 @@ allow = [ "CDLA-Permissive-2.0", # Community Data License Agreement (webpki-root-certs) "Unicode-DFS-2016", # Unicode Data Files and Software License (unicode_names2 via monty/ruff, --all-features only) "BSL-1.0", # Boost Software License (error-code via clipboard-win via rustyline, Windows-only) + "Python-2.0", # CPython license (bashkit-cpython-wasm embeds CPython built to WASI) ] # Review unclear/dual licenses diff --git a/docs/cli.md b/docs/cli.md index 572196d11..87896039a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -60,6 +60,7 @@ cargo binstall bashkit-cli |---------|---------|--------| | `interactive` | on | Interactive REPL (rustyline, signal-hook, terminal_size) | | `python` | on | `python`/`python3` builtin via Monty | +| `cpython` | off | `python`/`python3` as real CPython 3.14 in a WebAssembly sandbox (replaces Monty when both are on) | | `sqlite` | on | `sqlite`/`sqlite3` builtin via Turso (BETA upstream) | | `realfs` | off | `--mount-ro` / `--mount-rw` host filesystem mounts | diff --git a/docs/start-rust.md b/docs/start-rust.md index 6cdeb3f50..08421419c 100644 --- a/docs/start-rust.md +++ b/docs/start-rust.md @@ -33,7 +33,9 @@ cargo add bashkit --no-default-features --features http_client,aws-lc-rs `python` embeds the [Monty](https://github.com/pydantic/monty) interpreter, see the [Python builtin](../crates/bashkit/docs/python.md) guide to run Python inside the shell, and [Get -started in Python](start-python.md) to embed Bashkit *in* a Python app. +started in Python](start-python.md) to embed Bashkit *in* a Python app. `cpython` runs real +CPython 3.14 in a WebAssembly sandbox instead, see the +[CPython guide](../crates/bashkit/docs/cpython.md). ## First script diff --git a/justfile b/justfile index 77d6482a6..b2d5c225f 100644 --- a/justfile +++ b/justfile @@ -260,6 +260,11 @@ bench-sqlite: ./scripts/bench-sqlite.sh pnpm --dir site run data:performance +# Python runtimes (CPython wasm vs Monty): raw start, per call, compute, load +bench-python: + ./scripts/bench-python.sh + pnpm --dir site run data:performance + # === Eval (mira study) === # Evals run on the mira framework (github.com/everruns/mira). The crate is a # study binary the `mira` host CLI spawns over stdio; mira owns the model diff --git a/knowledge/log.md b/knowledge/log.md index 7284a9aff..9dd01f465 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -6,6 +6,7 @@ * **Decision**: Step 2 of the extraction landed. `crates/bashkit-scripted-tool` (published, lockstep version) now holds `ToolDef`, `ToolRegistry`, `ScriptedTool`, `ScriptingToolSet` and the logic-only shell; the core `scripted_tool` feature is removed (it was an alias for `bash_tool`, so the core dependency graph is unchanged). `BashBuilder::tool_registry` became `ToolRegistry::install(builder)`. Two more generic core public APIs made that possible without crate-private access: `bashkit::time` (the wasm-safe `Instant`/`SystemTime`/`timeout`/`sleep` shim) and `BashBuilder::execution_profile()` (read the Python/TypeScript limits a builder carries). Publish order is now `bashkit` → `bashkit-scripted-tool` + `bashkit-cli`; `just release-check` dry-runs the new crate against the latest published core. Release prep owes a CHANGELOG Breaking Changes note linking `docs/migrating-scripted-tool.md`. Recorded in [Scripted Tool Orchestration](integrations/scripted-tool-orchestration.md), [Release Process](operations/release-process.md), [Architecture](foundations/architecture.md). * **Decision**: Everything built on `ToolDef` (`ToolDef`/`ToolArgs`/`ToolImpl`, `ToolDefExtension`, `ToolRegistry`, `ScriptedTool`, `ScriptingToolSet`) leaves the core crate for a published `bashkit-scripted-tool` crate; Python/JS move the class to `bashkit.scripted` / `@everruns/bashkit/scripted`, with a public migration guide. Step 1 adds only generic public extension points to core: `RuntimeCallContext` (request extensions, budget, lease and deadline for Python/TypeScript host functions, replacing the registry's private task-local), public `Python`/`TypeScriptExtension::with_external_handler_and_prelude`, `ExecutionCapability::derive`, `ExecutionBudget::run`, `BuiltinContext::{run_budgeted, remaining_deadline}`, generic restricted-shell switches `BashBuilder::shell_features(ShellFeatures)` (file redirects, process substitution, script execution) and `BashBuilder::builtin_filter`, which replace the interpreter-level `ShellProfile::LogicOnly`; the logic-only shell itself (allowlist + rejecting filesystem) now lives beside `ScriptedTool` and moves with it, public `ExecutionPlan`/`SubCommand` re-exports, and the `Tool` contract helpers. Recorded in [Scripted Tool Orchestration](integrations/scripted-tool-orchestration.md), [Python Builtin](runtimes/python-builtin.md), [ZapCode Runtime](runtimes/zapcode-runtime.md), [Tool Contract](integrations/tool-contract.md). * **Feature**: [In-Process Terminal](integrations/in-process-terminal.md) behind the `terminal` feature (off by default): `Terminal` drives a `Bash` session from raw keystrokes, reports `Idle` when the session waits for input, and exposes the screen as text through the `vt100` crate. libghostty was considered and rejected because it needs Zig and C FFI, which would break the wasm and binding builds. The new `vi` builtin reads the device in raw mode and edits VFS files. Time blocked on terminal input is excluded from the execution timeout so an editor session is not killed while an agent thinks; busy work still times out (TM-DOS-120). Buffers are bounded (TM-DOS-119). Gaps recorded as L-TERM-001..003: vi is a subset, `read` does not wait for typed input, Ctrl-C lands at the next command boundary. `less` and `more` page interactively only inside a `Terminal` and stay cat-like for `exec()`/`BashTool`/CLI callers, so no existing caller can block on a pager (L-TERM-004 covers redirected stdout inside a session). +* **Runtime**: Real CPython 3.14 can now back `python`/`python3` (`cpython` feature, off by default). Monty stays: it is a code-mode subset, and scripts written by people and agents expect CPython's language, stdlib and command line. CPython is not trusted to isolate tenants; it runs as a `wasm32-wasip1` guest and the WASI host over the VFS is the boundary. Startup is the constraint that shaped everything: a Wizer snapshot with the interpreter already initialized, Pulley bytecode precompiled in `build.rs`, the module mapped in place from the binary, and copy-on-write instances. One finding worth keeping: exit-time `gc.collect()` walked the whole 40 MB snapshot heap and made every call ~100 ms; `gc.freeze()` at snapshot time brought it to ~5 ms. Second: the same static `.cwasm` bytes cannot back two live wasmtime modules (code is registered by address), so only the first load maps in place. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md), `L-CPY-*` and `TM-PY-CPY-*`. ## 2026-10-04 diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 116fc6695..ebd6bd48e 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -114,6 +114,22 @@ builtins are currently tracked; partial boundaries follow. |----|------|------------|----------| | L-DATE-001 | date | `TZ` accepts bundled IANA identifiers/aliases only. POSIX rule strings and `:zoneinfo` paths are unsupported and intentionally resolve to UTC because the sandbox has no trusted host zoneinfo filesystem. GNU nanosecond formatting supports the useful `%N`/`%3N`/`%6N`/`%9N` forms, not other widths | `date_timezone_tests` | +### CPython runtime (`cpython` feature) + +Boundaries of the WebAssembly CPython guest; see +[CPython WebAssembly Runtime](../runtimes/cpython-wasm.md). + +| ID | Limitation | Why | Evidence | +|----|------------|-----|----------| +| L-CPY-001 | No subprocesses from Python (`subprocess`, `os.system`, `os.fork`, `os.popen` fail); Python cannot call back into the shell | The guest has no process API; a shell bridge is a deliberate follow-up, not an accident | `processes_unavailable` (cpython_security_tests) | +| L-CPY-002 | No network from Python (`socket`, `urllib.request` cannot connect) | No socket imports in the WASI host; egress stays with the allowlisted shell builtins | TM-PY-CPY-003, `network_unavailable` | +| L-CPY-003 | No threads, `multiprocessing`, `ctypes`, native extensions or third-party packages | Single-threaded wasm guest with only the bundled pure-Python stdlib | `threads_and_native_code_unavailable` | +| L-CPY-004 | `errno` values are WASI's (`ENOENT` is 44); exception types and the `errno` module agree | wasi-libc numbering; rewriting it would desync the guest's `errno` module | `cpython_integration_tests::missing_script_exits_2`, stance | +| L-CPY-005 | `hash()` of `str`/`bytes` uses one seed baked into the snapshot | The seed is chosen during snapshot initialization; per-call re-seeding would require re-hashing every interned object | stance | +| L-CPY-006 | Deep C-level recursion ends the call with `python3: fatal error: stack overflow` instead of `RecursionError` | The interpreter's wasm stack is bounded (4 MiB); the trap is contained | `deep_c_recursion_is_contained` | +| L-CPY-007 | No interactive REPL; `python3` with no program reads one from stdin | No TTY inside the sandbox | stance | +| L-CPY-008 | Guest memory per call is capped at 1 GiB even if `max_memory` is higher | Pooled instance slots have a fixed maximum size | [CPython WebAssembly Runtime](../runtimes/cpython-wasm.md) | + ## Text Processing What each tool does is covered by its spec tests (all unskipped tests diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index b9f41f70a..247055857 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -80,6 +80,11 @@ silently failed. --all-targets --all-features -- -D warnings`, `cargo test`. 5. **Verify publish-readiness** (catches what local tests don't, the `cargo publish` packaging step, missing files, version drift): + - `cargo publish --dry-run -p bashkit-cpython-wasm` must succeed and the + `.crate` must stay under crates.io's 10 MiB limit (it embeds the CPython + snapshot; ~9.5 MiB as of 0.18). `bashkit` depends on the same version + of it, so `publish.yml` publishes it first and the `bashkit` dry-run only + resolves once that version is live. - `cargo publish --dry-run -p bashkit` must succeed. Package `bashkit-cli` and `bashkit-scripted-tool` in a disposable copy against the latest published registry core version (remove the local path in the copy) as a structural @@ -214,6 +219,7 @@ Use the latest entries in `CHANGELOG.md` as the template. Rules: ## Package Names and Registries +- `bashkit-cpython-wasm` on crates.io (embedded CPython for the `cpython` feature; published first) - `bashkit` on crates.io (core library) - `bashkit-scripted-tool` on crates.io (`ScriptedTool` / `ToolDef` / `ToolRegistry`) - `bashkit-cli` on crates.io (CLI tool) diff --git a/knowledge/runtimes/cpython-wasm.md b/knowledge/runtimes/cpython-wasm.md new file mode 100644 index 000000000..1db599d9c --- /dev/null +++ b/knowledge/runtimes/cpython-wasm.md @@ -0,0 +1,172 @@ +--- +type: Subsystem Design +title: CPython WebAssembly Runtime +description: Real CPython 3.14 as python3, compiled to WASI, snapshotted with Wizer and run on Wasmtime's Pulley interpreter over the bashkit VFS. +tags: + - bashkit + - python + - runtime + - sandbox + - wasm +--- + +# CPython WebAssembly Runtime + +## Status +Implemented (`cpython` feature, off by default) + +## Decision + +`python`/`python3` can run real CPython 3.14 instead of Monty. Monty is a +code-mode Python subset (no classes, small stdlib, its own CLI); scripts +written by people and agents expect CPython's language, stdlib and command +line. Bashkit's main target is many tenants in one process, so CPython's own +memory safety cannot be the isolation boundary: CPython runs as a WebAssembly +guest, and the wasm boundary is the sandbox. + +Public guide: `crates/bashkit/docs/cpython.md` (rustdoc `cpython_guide`). + +### Shape + +| Piece | Where | Role | +|-------|-------|------| +| `bashkit-cpython-wasm` crate | `crates/bashkit-cpython-wasm/` | Ships bytes only: precompiled module, stdlib zip, `engine_config()` | +| Guest build | `crates/bashkit-cpython-wasm/guest/build.sh` | Reproducible: CPython 3.14.8 + WASI SDK 24 + static zlib/sqlite3, Wizer snapshot | +| Guest entry | `guest/bashkit_main.c`, `guest/_bashkit_boot.py` | Reactor exports `wizer-initialize`, `bashkit_run`; Python CLI emulation | +| WASI host | `crates/bashkit/src/builtins/cpython/wasi.rs` | All 42 `wasi_snapshot_preview1` imports over the bashkit `FileSystem` | +| Builtin | `crates/bashkit/src/builtins/cpython/mod.rs` | Engine, pooled instances, limits, deadline/budget polling, exit mapping | + +### Startup is the product constraint + +Nothing is compiled at run time, and CPython initialization never runs per +call: + +1. **Wizer snapshot**: `wizer-initialize` runs `Py_InitializeFromConfig`, + imports ~70 common stdlib modules, patches asyncio, then `gc.freeze()`s the + heap. The resulting module starts with a running interpreter. +2. **Pulley AOT**: `build.rs` precompiles the snapshot to Wasmtime's portable + Pulley bytecode (`pulley64`) with Cranelift as a build-dependency only. One + artifact serves every 64-bit little-endian host; no executable memory is + needed at run time. Native AOT (`.cwasm` per target) is the documented + fallback if guest CPU speed ever matters more than portability. +3. **Zero-copy load**: the `.cwasm` is embedded 64 KiB-aligned and loaded with + `Module::deserialize_raw`; wasmtime registers code by address, so only the + first load in a process maps in place, later loads copy (`load_module`). +4. **Copy-on-write instances**: `memory_init_cow` + + `memory_guaranteed_dense_image_size(128 MB)` map the ~40 MB snapshot heap + instead of copying it (without the dense image every instantiation copied + 9.4 MB). +5. **`gc.freeze()` in the snapshot**: without it every call's exit-time + `gc.collect()` walked the whole snapshot heap, dirtying its pages; calls + cost ~100 ms instead of ~5 ms. + +Measured on a 4-vCPU x86-64 VM (see `criterion-python-*` results under +`crates/bashkit/benches/results/`): first `python3` in a fresh process +~22 ms (one-time engine/module load and first-touch page faults), then +`python3 -c pass` ~5.5 ms per call warm; Monty ~15 µs (first call ~0.4 ms). +CPU-bound Python is ~4-30x slower than Monty. 1024 concurrent tenants × 4 +calls: 0 failures, ~90 calls/s on 4 vCPUs, 4.7 GB peak RSS. + +### Guest contract (snapshot invariants) + +- The snapshot cached the preopen table: the host must present exactly one + preopen, `/`, at fd 3 (`PREOPEN_FD`). +- The snapshot cached the stdlib zip directory: the host must serve the exact + bytes of `STDLIB_ZIP` at `/usr/local/lib/python314.zip`. Both artifacts come + from one `build.sh` run. +- One instance serves one call; state is never reused. + +### WASI host decisions + +- Only `wasi_snapshot_preview1`, only against bashkit state: VFS, captured + stdin bytes, in-memory stdout/stderr, host RNG and clocks. Socket calls + return `ENOTSUP`; there is no process API. +- Files are whole in-memory buffers written back on close/sync/exit. Buffers + are bounded by `max_file_size` and, in total, by the call's memory budget. +- Paths normalize lexically and clamp at `/`. The VFS does not follow + symlinks (L-FS-001); neither does the host. +- The stdlib zip is a read-only overlay at a fixed path. Writes, renames, + truncation, unlink and directory changes there fail (`EROFS`/`ENOENT`/ + `EACCES`); tenant files placed next to it are not on `sys.path`. +- `poll_oneoff` sleeps in 250 ms slices and never past the call deadline. + +### Builtin decisions + +- **No runtime opt-in env var**: unlike Monty, no native interpreter runs in + the host, so `BashBuilder::cpython()` is the opt-in. +- **Only exported variables** reach `os.environ`, plus `PWD`; `__BASHKIT_*` + names are filtered (the recursion limit is passed as one and removed by the + guest). +- **Fuel** is set effectively unlimited with a 100K-instruction async yield + interval. Each poll re-checks the call deadline and the request's + `ExecutionBudget` (cancellation, request timeout), so a busy loop cannot + pin a worker thread. Budget work units: 4096 per call plus one per 1024 + guest instructions. +- **Pooling allocator**: 512 slots per process, 1 GiB max memory per slot + (`max_memory` above that is clamped), 64 MiB kept resident, `pagemap_scan` + on Linux. A semaphore makes call 513 wait for a slot inside its deadline. + If the pool cannot reserve address space, the engine falls back to on-demand + allocation. Engine reservation is 1 GiB with a 64 KiB guard: Pulley + bounds-checks every access, so large virtual reservations buy nothing. +- **Exit mapping**: `bashkit_run` return value or `proc_exit` code; timeout + 124; traps become `python3: fatal error: ...` (Display only, TM-INF-022) + with exit 1; open files are flushed even after a trap or timeout. +- **Output cap** drops bytes past `max_output` and appends + `python3: output truncated at N bytes` to stderr. + +### Guest driver (`_bashkit_boot.py`) + +Emulates CPython's command line on an already-initialized interpreter: +option parsing (`-c`, `-m`, file, directory, `-`, `-x`, `-W`, `-V`/`-VV`, +`-h`, ignored init-only flags), fresh `__main__` per call, `sys.argv` and +`sys.path[0]` rules, tracebacks with driver frames stripped, `SystemExit` +semantics, `atexit`, then clearing `__main__` + `gc.collect()` so unclosed +files flush. `random` is re-seeded per call. asyncio's self-pipe is disabled +(no sockets, threads or signals exist to need it). + +## Testing + +| Layer | Location | +|-------|----------| +| Unit | `builtins/cpython/tests.rs`, `wasi.rs` tests, `bashkit-cpython-wasm` lib tests | +| Integration (CLI, stdio, env, VFS, isolation, bash interop) | `tests/integration/cpython_integration_tests.rs` | +| Capability (~85 language/stdlib programs) | `tests/integration/cpython_capability_tests.rs`; the same table is diffed against the host `python3` when present | +| Security (threats below, limits, crash containment, proptest fuzz) | `tests/integration/cpython_security_tests.rs` | +| Fuzz | `crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs` (`--features cpython`, nightly job) | +| Bench / load | `benches/python.rs`, `examples/python_startup.rs`, `examples/cpython_load.rs`, `just bench-python` | + +CI: the Test job runs `cargo test -p bashkit --features cpython --lib --test +integration -- cpython`; the examples job runs `cpython_scripts`; fuzz-check +builds `cpython_fuzz`. + +## Rebuilding the guest + +`crates/bashkit-cpython-wasm/guest/build.sh [WORK_DIR]` downloads pinned +sources (CPython, WASI SDK, wasmtime CLI for `wizer`, zlib, sqlite), builds, +snapshots, smoke-tests with the reference runtime and rewrites `artifacts/` +(`python.wasm.gz`, `python314.zip`, `MANIFEST` with checksums). Commit all +three together. Changing `engine_config()` or the wasmtime version requires +no guest rebuild; `build.rs` recompiles the `.cwasm`. + +wasmtime is pinned (`=48.0.5`): a `.cwasm` loads only into the version and +compile-affecting configuration that produced it. 49.x needs rustc 1.96. + +## Known gaps + +- Python cannot call back into the shell (subprocess bridge) or host + functions (`ToolDef`). TODO: design a `bashkit` module over a host import. +- ~650 page faults per call (host Pulley stack + guest CoW writes); kernel + fault cost grows under concurrency, so 4 vCPUs reach ~2x, not 4x, single + thread throughput. TODO: profile which pages fault; consider pooling the + Pulley stack upstream. +- macOS has no `memfd`; CoW falls back to copying the image per instance. +- See [Limitations](../operations/limitations.md) `L-CPY-*` and + [Threat Model](../security/threat-model.md) `TM-PY-CPY-*`. + +## See also + +- [Python Builtin](python-builtin.md), the Monty runtime this complements +- [ZapCode Runtime](zapcode-runtime.md), the other embedded language VM +- [Builtins](../foundations/builtins.md), builtin trait and registration +- [Threat Model](../security/threat-model.md) +- [Performance Results](../operations/performance-results.md) diff --git a/knowledge/runtimes/index.md b/knowledge/runtimes/index.md index 96a206bf3..1408ac4bf 100644 --- a/knowledge/runtimes/index.md +++ b/knowledge/runtimes/index.md @@ -1,6 +1,7 @@ # Runtimes and packages * [Python Builtin](python-builtin.md) - Embedded Python execution through Monty with security and resource controls. +* [CPython WebAssembly Runtime](cpython-wasm.md) - Real CPython 3.14 as python3, compiled to WASI, snapshotted with Wizer and run on Wasmtime's Pulley interpreter over the bashkit VFS. * [ZapCode Runtime](zapcode-runtime.md) - Embedded TypeScript runtime, external functions, VFS bridging, and resource limits. * [SQLite Builtin](sqlite-builtin.md) - Embedded SQLite through Turso with memory and virtual filesystem backends. * [Coreutils Argument Port](coreutils-args-port.md) - Code generation design for porting uutils clap arguments and uucore modules. diff --git a/knowledge/runtimes/python-builtin.md b/knowledge/runtimes/python-builtin.md index 2281fe9a8..2310c69e0 100644 --- a/knowledge/runtimes/python-builtin.md +++ b/knowledge/runtimes/python-builtin.md @@ -19,6 +19,9 @@ tags: ## Status Implemented (experimental) +For full CPython semantics in a WebAssembly sandbox, see +[CPython WebAssembly Runtime](cpython-wasm.md) (`cpython` feature). + ## Decision Bashkit provides sandboxed Python execution via `python` and `python3` builtins, diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 758e952d5..c5ffc7b80 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -1785,6 +1785,37 @@ The following components are fuzz-tested for robustness: --- +## CPython WebAssembly Security (TM-PY-CPY) + +The `cpython` feature runs real CPython 3.14 as a `wasm32-wasip1` guest on +Wasmtime's Pulley interpreter ([CPython WebAssembly Runtime](../runtimes/cpython-wasm.md)). +CPython itself is *not* trusted: a guest bug, a malicious script or a crafted +input may corrupt the guest's own linear memory. The boundary is the wasm +sandbox plus the WASI host in `builtins/cpython/wasi.rs`, which is the only +code that touches bashkit state. + +``` +python code -> CPython (wasm guest) -> wasi_snapshot_preview1 imports -> WASI host -> bashkit VFS / captured stdio +``` + +| ID | Threat | Severity | Mitigation | Test | +|----|--------|----------|------------|------| +| TM-PY-CPY-001 | Host filesystem access (absolute paths, `..`, `/proc`, symlinks) | Critical | Only the VFS is reachable; paths clamp at `/`; symlinks are not followed | `host_files_are_not_reachable`, `parent_traversal_is_clamped_at_vfs_root`, `host_system_paths_absent`, `symlinks_are_not_followed` | +| TM-PY-CPY-002 | Tampering with or shadowing the stdlib | High | Stdlib zip is a read-only host overlay; tenant files next to it are not on `sys.path`; preloaded modules come from the snapshot | `stdlib_zip_is_read_only`, `vfs_files_cannot_shadow_stdlib` | +| TM-PY-CPY-003 | Network, process, thread or native-code escape | Critical | No socket or process imports; sockets return `ENOTSUP`; no threads, `ctypes` or dynamic loading in the guest | `network_unavailable`, `urllib_cannot_fetch`, `processes_unavailable`, `threads_and_native_code_unavailable` | +| TM-PY-CPY-004 | CPU exhaustion (busy loops, swallowed exceptions, sleeps) | High | Fuel-driven async yields; every poll checks the call deadline and the request `ExecutionBudget`; sleeps never pass the deadline | `infinite_loop_times_out`, `sleep_is_bounded_by_deadline`, `swallowing_exceptions_cannot_escape_timeout`, `shell_timeout_tighter_than_python_limit_wins`, `cancellation_stops_busy_guest` | +| TM-PY-CPY-005 | Memory exhaustion (guest heap, host-side file buffers) | High | Store limiter caps linear memory (MemoryError in Python); open-file buffers share the same budget; VFS limits apply | `huge_allocation_raises_memory_error`, `incremental_growth_raises_memory_error`, `file_buffers_share_the_memory_budget`, `vfs_file_size_limit_applies` | +| TM-PY-CPY-006 | Output and descriptor floods | Medium | `max_output` cap with truncation note; 1024 fds per call | `output_is_capped_and_marked_truncated`, `output_flood_loop_is_bounded`, `descriptor_table_is_bounded` | +| TM-PY-CPY-007 | Guest crash (abort, stack overflow, parser bombs) takes down the host | Critical | Traps end only that call (exit 1, Display-only message); instance discarded; shell continues | `deep_c_recursion_is_contained`, `parser_bomb_is_contained`, `abort_is_contained`, `shell_continues_after_guest_failures` | +| TM-PY-CPY-008 | State leaking across calls or tenants | Critical | Fresh instance per call from a copy-on-write snapshot; per-`Bash` VFS; only exported variables reach `os.environ`; `random` re-seeded per call | `no_state_crosses_calls`, `concurrent_tenants_are_isolated`, `host_environment_not_visible`, `interpreter_state_does_not_persist_between_calls` | +| TM-PY-CPY-009 | Internal shapes leaking through errors (TM-INF-022) | Medium | Trap and host errors formatted via Display, capped at 512 bytes; driver frames stripped from tracebacks | `error_paths_do_not_leak_internals`, `trap_messages_are_display_only`, `cpython_fuzz` | +| TM-PY-CPY-010 | Predictable `hash()` within a call | Low | Accepted: hash flooding is bounded by the call's CPU and memory limits and affects only that call (L-CPY-005) | stance | + +Fuzzing: `cpython_security_tests` runs bounded proptest cases (arbitrary +source, stitched os/sys/file fragments, arbitrary CLI arguments) through +`assert_no_leak`; `fuzz/fuzz_targets/cpython_fuzz.rs` runs nightly under +cargo-fuzz with the host-environment canary. + ## Python / Monty Security (TM-PY) > **Experimental.** Monty is an early-stage Python interpreter that may have diff --git a/knowledge/status/builtins.json b/knowledge/status/builtins.json index d510ea3c4..843c8371c 100644 --- a/knowledge/status/builtins.json +++ b/knowledge/status/builtins.json @@ -333,6 +333,10 @@ "feature": null, "name": "mktemp" }, + { + "feature": null, + "name": "more" + }, { "feature": null, "name": "mv" @@ -625,6 +629,10 @@ "feature": null, "name": "verify" }, + { + "feature": null, + "name": "vi" + }, { "feature": null, "name": "wait" @@ -670,5 +678,5 @@ "name": "zip" } ], - "count": 167 + "count": 169 } diff --git a/scripts/bench-python.sh b/scripts/bench-python.sh new file mode 100755 index 000000000..a498040b3 --- /dev/null +++ b/scripts/bench-python.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Run the Python runtime benchmarks (CPython wasm vs Monty) and save results to +# crates/bashkit/benches/results/criterion-python--.md. +# +# Three parts: +# 1. Raw start: `python_startup` example in fresh OS processes (first call in +# a process, including one-time runtime load), RUNS times per runtime. +# 2. Criterion `python` bench: per call, per session, compute, +# parallel sessions. +# 3. Load: `cpython_load` example, many concurrent tenants in one process. +# +# Usage: +# ./scripts/bench-python.sh # run + save +# RUNS=20 LOAD_LEVELS="1 64 1024" ./scripts/bench-python.sh +set -euo pipefail + +RESULTS_DIR="crates/bashkit/benches/results" +HOSTNAME=$(hostname 2>/dev/null || echo "unknown") +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m) +CPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "?") +TIMESTAMP=$(date +%s) +MONIKER="${HOSTNAME}-${OS}-${ARCH}" +RUNS="${RUNS:-10}" +LOAD_LEVELS="${LOAD_LEVELS:-1 16 64 256 1024}" + +# Security: private temp file so tee cannot follow a pre-created /tmp symlink. +BENCH_OUT=$(mktemp "${TMPDIR:-/tmp}/criterion-python-output.XXXXXX") +trap 'rm -f "$BENCH_OUT"' EXIT + +echo "Building examples..." +cargo build --release -p bashkit --example python_startup --example cpython_load \ + --features python,cpython + +# summarize RUNTIME LABEL < values -> one markdown row (min / median / max) +summarize() { + sort -n | awk -v r="$1" -v k="$2" \ + '{a[NR]=$1} END {printf "| %s | %s | %.3f ms | %.3f ms | %.3f ms |\n", r, k, a[1], a[int((NR+1)/2)], a[NR]}' +} + +startup_rows() { + local runtime="$1" lines + lines=$(for _ in $(seq "$RUNS"); do ./target/release/examples/python_startup "$runtime"; done) + sed -E 's/.*first_call_ms=([0-9.]+).*/\1/' <<<"$lines" | summarize "$runtime" "first call (fresh process)" + sed -E 's/.*second_call_ms=([0-9.]+).*/\1/' <<<"$lines" | summarize "$runtime" "second call" +} + +echo "Measuring raw start ($RUNS fresh processes per runtime)..." +STARTUP=$(startup_rows monty; startup_rows cpython) + +echo "Running criterion python bench..." +cargo bench -p bashkit --bench python --features python,cpython 2>&1 | tee "$BENCH_OUT" + +echo "Running load test ($LOAD_LEVELS)..." +# shellcheck disable=SC2086 # word-split levels on purpose +LOAD=$(./target/release/examples/cpython_load $LOAD_LEVELS) + +extract_times() { + local pattern="$1" + grep -A2 "$pattern" "$BENCH_OUT" | + awk -v pat="$pattern" ' + $0 ~ pat {name=$1} + /time:/ { + match($0, /\[.*\]/) + bracket = substr($0, RSTART+1, RLENGTH-2) + split(bracket, vals, " ") + printf "| %s | %s %s |\n", name, vals[3], vals[4] + }' +} + +section() { # title pattern + printf '\n## %s\n\n| Benchmark | Time |\n|-----------|------|\n' "$1" + extract_times "$2" +} + +MD_PATH="${RESULTS_DIR}/criterion-python-${MONIKER}-${TIMESTAMP}.md" +{ + cat <
"$MD_PATH" + +echo "" +echo "Saved: $MD_PATH" diff --git a/site/astro.config.mjs b/site/astro.config.mjs index bfbc923f0..3cf69090b 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -15,6 +15,7 @@ const DOC_LINKS = new Map([ ["cli.md", "/docs/cli/"], ["compatibility.md", "/docs/compatibility/"], ["configuration.md", "/docs/configuration/"], + ["cpython.md", "/docs/cpython/"], ["credential-injection.md", "/docs/credential-injection/"], ["custom_builtins.md", "/docs/custom-builtins/"], ["custom_builtins_js.md", "/docs/custom-builtins-js/"], diff --git a/site/src/data/performance-timeline.json b/site/src/data/performance-timeline.json index 11283d3b8..95cfdc636 100644 --- a/site/src/data/performance-timeline.json +++ b/site/src/data/performance-timeline.json @@ -1,5 +1,5 @@ { - "generatedAt": "2026-09-05T17:37:13.000Z", + "generatedAt": "2026-10-06T20:42:07.000Z", "sources": { "bench": "crates/bashkit-bench/results/*.json", "criterion": "crates/bashkit/benches/results/*.md", @@ -7,7 +7,7 @@ }, "summary": { "benchRuns": 10, - "criterionRuns": 15, + "criterionRuns": 16, "evalRuns": 38, "latestBench": { "id": "bench-mykhailosmac-macos-aarch64-1788629833", @@ -1831,6 +1831,27 @@ "us": 370.97 }, "bestImprovement": null + }, + { + "id": "criterion-python-vm-linux-x86_64-1791319327", + "kind": "criterion", + "family": "python", + "label": "Python Runtimes Benchmark: CPython (wasm) vs Monty", + "date": "2026-10-06", + "timestamp": "2026-10-06T20:42:07.000Z", + "source": "crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md", + "reportSource": "crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md", + "cases": 22, + "medianUs": 6917.3, + "p95Us": 287803.5, + "medianChangePct": null, + "meanChangePct": null, + "bestChangePct": null, + "fastestCase": { + "name": "python_call/monty/print", + "us": 15.41 + }, + "bestImprovement": null } ], "evalRuns": [ @@ -5640,6 +5661,15 @@ "detail": "96 parity/perf cases on mykhailosmac-macos-aarch64; output match 100%.", "metric": 49.9, "source": "crates/bashkit-bench/results/bench-mykhailosmac-macos-aarch64-1788629833.json" + }, + { + "date": "2026-10-06", + "timestamp": "2026-10-06T20:42:07.000Z", + "kind": "Criterion", + "title": "python", + "detail": "python_call/monty/print at 15.41 us median", + "metric": 6917.3, + "source": "crates/bashkit/benches/results/criterion-python-vm-linux-x86_64-1791319327.md" } ] } diff --git a/site/src/pages/docs/_meta.ts b/site/src/pages/docs/_meta.ts index 02f3b85e5..3f3b39ef0 100644 --- a/site/src/pages/docs/_meta.ts +++ b/site/src/pages/docs/_meta.ts @@ -229,6 +229,17 @@ export const DOC_META: DocMeta[] = [ collection: "rustdocs", editPath: "crates/bashkit/docs/python.md", }, + { + slug: "cpython", + title: "CPython builtin", + summary: "Real CPython 3.14 in a WebAssembly sandbox: full stdlib, CLI, limits.", + seoTitle: "Bashkit CPython builtin: sandboxed Python 3.14 in WebAssembly", + seoDescription: + "Run real CPython 3.14 inside Bashkit as python3, compiled to WebAssembly with a pre-initialized snapshot, VFS access, resource limits, and per-call isolation.", + section: "Runtimes", + collection: "rustdocs", + editPath: "crates/bashkit/docs/cpython.md", + }, { slug: "builtin_typescript", title: "TypeScript builtin", diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 03a7f0ab4..a7b15539c 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -4,6 +4,9 @@ [cargo-vet] version = "0.10" +[imports.bytecodealliance] +url = "https://raw.githubusercontent.com/bytecodealliance/wasmtime/main/supply-chain/audits.toml" + [imports.mozilla] url = "https://raw.githubusercontent.com/mozilla/supply-chain/main/audits.toml" @@ -15,6 +18,10 @@ criteria = "safe-to-deploy" audit-as-crates-io = false criteria = "safe-to-deploy" +[policy.bashkit-cpython-wasm] +audit-as-crates-io = false +criteria = "safe-to-deploy" + [policy.bashkit-js] audit-as-crates-io = false criteria = "safe-to-deploy" @@ -31,8 +38,8 @@ criteria = "safe-to-deploy" audit-as-crates-io = true notes = "Require published-base AND exact Git-commit delta audits for the pinned third-party patch; https://mozilla.github.io/cargo-vet/first-party-code.html" -[[exemptions.adler2]] -version = "2.0.1" +[[exemptions.addr2line]] +version = "0.26.1" criteria = "safe-to-deploy" [[exemptions.aead]] @@ -75,10 +82,6 @@ criteria = "safe-to-deploy" version = "0.4.0" criteria = "safe-to-run" -[[exemptions.allocator-api2]] -version = "0.2.21" -criteria = "safe-to-deploy" - [[exemptions.allocator-api2]] version = "0.4.0" criteria = "safe-to-deploy" @@ -87,10 +90,6 @@ criteria = "safe-to-deploy" version = "0.1.6" criteria = "safe-to-deploy" -[[exemptions.anes]] -version = "0.1.6" -criteria = "safe-to-run" - [[exemptions.anstyle]] version = "1.0.14" criteria = "safe-to-deploy" @@ -135,10 +134,6 @@ criteria = "safe-to-deploy" version = "1.0.3" criteria = "safe-to-deploy" -[[exemptions.atomic-waker]] -version = "1.1.2" -criteria = "safe-to-deploy" - [[exemptions.attribute-derive]] version = "0.10.5" criteria = "safe-to-deploy" @@ -199,6 +194,10 @@ criteria = "safe-to-deploy" version = "0.11.0-rc.6" criteria = "safe-to-deploy" +[[exemptions.block-buffer]] +version = "0.10.4" +criteria = "safe-to-deploy" + [[exemptions.block-buffer]] version = "0.12.1" criteria = "safe-to-deploy" @@ -219,10 +218,6 @@ criteria = "safe-to-deploy" version = "1.13.1" criteria = "safe-to-deploy" -[[exemptions.bumpalo]] -version = "3.20.3" -criteria = "safe-to-deploy" - [[exemptions.bytecount]] version = "0.6.9" criteria = "safe-to-deploy" @@ -301,10 +296,6 @@ criteria = "safe-to-run" version = "0.2.2" criteria = "safe-to-run" -[[exemptions.cipher]] -version = "0.4.4" -criteria = "safe-to-deploy" - [[exemptions.cipher]] version = "0.5.2" criteria = "safe-to-deploy" @@ -337,10 +328,6 @@ criteria = "safe-to-deploy" version = "0.5.4" criteria = "safe-to-deploy" -[[exemptions.cobs]] -version = "0.3.0" -criteria = "safe-to-deploy" - [[exemptions.collection_literals]] version = "1.0.3" criteria = "safe-to-deploy" @@ -381,14 +368,14 @@ criteria = "safe-to-deploy" version = "0.10.1" criteria = "safe-to-deploy" -[[exemptions.core-foundation-sys]] -version = "0.8.7" -criteria = "safe-to-deploy" - [[exemptions.cow-utils]] version = "0.1.3" criteria = "safe-to-deploy" +[[exemptions.cpp_demangle]] +version = "0.3.5" +criteria = "safe-to-deploy" + [[exemptions.cpubits]] version = "0.1.1" criteria = "safe-to-deploy" @@ -423,7 +410,7 @@ criteria = "safe-to-deploy" [[exemptions.crossbeam-deque]] version = "0.8.7" -criteria = "safe-to-run" +criteria = "safe-to-deploy" [[exemptions.crossbeam-epoch]] version = "0.9.20" @@ -513,6 +500,10 @@ criteria = "safe-to-deploy" version = "0.1.13" criteria = "safe-to-run" +[[exemptions.digest]] +version = "0.10.7" +criteria = "safe-to-deploy" + [[exemptions.digest]] version = "0.11.3" criteria = "safe-to-deploy" @@ -553,14 +544,6 @@ criteria = "safe-to-deploy" version = "0.14.1" criteria = "safe-to-deploy" -[[exemptions.embedded-io]] -version = "0.4.0" -criteria = "safe-to-deploy" - -[[exemptions.embedded-io]] -version = "0.6.1" -criteria = "safe-to-deploy" - [[exemptions.encode_unicode]] version = "1.0.0" criteria = "safe-to-run" @@ -589,10 +572,6 @@ criteria = "safe-to-deploy" version = "0.5.1" criteria = "safe-to-deploy" -[[exemptions.fallible-iterator]] -version = "0.3.0" -criteria = "safe-to-deploy" - [[exemptions.fancy-regex]] version = "0.17.0" criteria = "safe-to-deploy" @@ -717,6 +696,10 @@ criteria = "safe-to-deploy" version = "0.6.0" criteria = "safe-to-deploy" +[[exemptions.gimli]] +version = "0.33.0" +criteria = "safe-to-deploy" + [[exemptions.gloo-timers]] version = "0.4.0" criteria = "safe-to-deploy" @@ -741,10 +724,6 @@ criteria = "safe-to-deploy" version = "0.7.17" criteria = "safe-to-deploy" -[[exemptions.heck]] -version = "0.5.0" -criteria = "safe-to-deploy" - [[exemptions.hermit-abi]] version = "0.5.2" criteria = "safe-to-deploy" @@ -805,10 +784,6 @@ criteria = "safe-to-deploy" version = "0.1.65" criteria = "safe-to-deploy" -[[exemptions.iana-time-zone-haiku]] -version = "0.1.2" -criteria = "safe-to-deploy" - [[exemptions.icu_collator]] version = "2.3.0" criteria = "safe-to-deploy" @@ -861,6 +836,10 @@ criteria = "safe-to-deploy" version = "2.3.0" criteria = "safe-to-deploy" +[[exemptions.id-arena]] +version = "2.3.0" +criteria = "safe-to-deploy" + [[exemptions.idna]] version = "1.1.0" criteria = "safe-to-deploy" @@ -921,10 +900,6 @@ criteria = "safe-to-deploy" version = "3.1.0" criteria = "safe-to-deploy" -[[exemptions.jaq-json]] -version = "2.0.2" -criteria = "safe-to-deploy" - [[exemptions.jaq-std]] version = "3.0.2" criteria = "safe-to-deploy" @@ -1042,6 +1017,10 @@ criteria = "safe-to-deploy" version = "0.7.2" criteria = "safe-to-deploy" +[[exemptions.mach2]] +version = "0.6.0" +criteria = "safe-to-deploy" + [[exemptions.manyhow]] version = "0.11.4" criteria = "safe-to-deploy" @@ -1050,10 +1029,6 @@ criteria = "safe-to-deploy" version = "0.11.4" criteria = "safe-to-deploy" -[[exemptions.matchers]] -version = "0.2.0" -criteria = "safe-to-deploy" - [[exemptions.md-5]] version = "0.11.0" criteria = "safe-to-deploy" @@ -1066,6 +1041,10 @@ criteria = "safe-to-deploy" version = "2.8.3" criteria = "safe-to-deploy" +[[exemptions.memfd]] +version = "0.6.6" +criteria = "safe-to-deploy" + [[exemptions.memoffset]] version = "0.9.1" criteria = "safe-to-deploy" @@ -1150,10 +1129,6 @@ criteria = "safe-to-deploy" version = "0.5.5" criteria = "safe-to-deploy" -[[exemptions.nu-ansi-term]] -version = "0.50.3" -criteria = "safe-to-deploy" - [[exemptions.num-bigint]] version = "0.4.8" criteria = "safe-to-deploy" @@ -1162,8 +1137,8 @@ criteria = "safe-to-deploy" version = "0.1.47" criteria = "safe-to-deploy" -[[exemptions.num-traits]] -version = "0.2.19" +[[exemptions.object]] +version = "0.39.1" criteria = "safe-to-deploy" [[exemptions.once_cell]] @@ -1306,10 +1281,6 @@ criteria = "safe-to-deploy" version = "1.0.0" criteria = "safe-to-deploy" -[[exemptions.percent-encoding]] -version = "2.3.2" -criteria = "safe-to-deploy" - [[exemptions.phc]] version = "0.6.1" criteria = "safe-to-deploy" @@ -1412,10 +1383,6 @@ criteria = "safe-to-deploy" version = "0.2.7" criteria = "safe-to-deploy" -[[exemptions.postcard]] -version = "1.1.3" -criteria = "safe-to-deploy" - [[exemptions.potential_utf]] version = "0.1.6" criteria = "safe-to-deploy" @@ -1554,11 +1521,11 @@ criteria = "safe-to-deploy" [[exemptions.rayon]] version = "1.12.0" -criteria = "safe-to-run" +criteria = "safe-to-deploy" [[exemptions.rayon-core]] version = "1.13.0" -criteria = "safe-to-run" +criteria = "safe-to-deploy" [[exemptions.redox_syscall]] version = "0.5.18" @@ -1636,12 +1603,12 @@ criteria = "safe-to-deploy" version = "0.52.0" criteria = "safe-to-deploy" -[[exemptions.rustc-hash]] -version = "2.1.3" +[[exemptions.rustc-demangle]] +version = "0.1.28" criteria = "safe-to-deploy" -[[exemptions.rustc_version]] -version = "0.4.1" +[[exemptions.rustc-hash]] +version = "2.1.3" criteria = "safe-to-deploy" [[exemptions.rustc_version_runtime]] @@ -1732,10 +1699,6 @@ criteria = "safe-to-deploy" version = "1.3.0" criteria = "safe-to-deploy" -[[exemptions.semver]] -version = "1.0.28" -criteria = "safe-to-deploy" - [[exemptions.send_wrapper]] version = "0.6.0" criteria = "safe-to-deploy" @@ -1784,8 +1747,8 @@ criteria = "safe-to-run" version = "0.11.0" criteria = "safe-to-deploy" -[[exemptions.sha1_smol]] -version = "1.0.1" +[[exemptions.sha2]] +version = "0.10.9" criteria = "safe-to-deploy" [[exemptions.sha2]] @@ -1800,14 +1763,6 @@ criteria = "safe-to-deploy" version = "0.12.0" criteria = "safe-to-deploy" -[[exemptions.sharded-slab]] -version = "0.1.7" -criteria = "safe-to-deploy" - -[[exemptions.shlex]] -version = "2.0.1" -criteria = "safe-to-deploy" - [[exemptions.shuttle]] version = "0.8.1" criteria = "safe-to-deploy" @@ -1896,10 +1851,6 @@ criteria = "safe-to-deploy" version = "1.2.1" criteria = "safe-to-deploy" -[[exemptions.static_assertions]] -version = "1.1.0" -criteria = "safe-to-deploy" - [[exemptions.strum]] version = "0.26.3" criteria = "safe-to-deploy" @@ -1952,6 +1903,10 @@ criteria = "safe-to-deploy" version = "3.27.0" criteria = "safe-to-deploy" +[[exemptions.termcolor]] +version = "1.4.1" +criteria = "safe-to-deploy" + [[exemptions.terminal_size]] version = "0.4.4" criteria = "safe-to-deploy" @@ -1988,10 +1943,6 @@ criteria = "safe-to-run" version = "1.12.0" criteria = "safe-to-deploy" -[[exemptions.tinyvec_macros]] -version = "0.1.1" -criteria = "safe-to-deploy" - [[exemptions.tokio-macros]] version = "2.7.2" criteria = "safe-to-deploy" @@ -2040,10 +1991,6 @@ criteria = "safe-to-deploy" version = "0.1.36" criteria = "safe-to-deploy" -[[exemptions.tracing-log]] -version = "0.2.0" -criteria = "safe-to-deploy" - [[exemptions.tracing-subscriber]] version = "0.3.23" criteria = "safe-to-deploy" @@ -2080,10 +2027,6 @@ criteria = "safe-to-deploy" version = "1.20.1" criteria = "safe-to-deploy" -[[exemptions.unarray]] -version = "0.1.4" -criteria = "safe-to-run" - [[exemptions.uncased]] version = "0.9.10" criteria = "safe-to-deploy" @@ -2182,10 +2125,6 @@ criteria = "safe-to-deploy" version = "0.11.1+wasi-snapshot-preview1" criteria = "safe-to-deploy" -[[exemptions.wasip2]] -version = "1.0.4+wasi-0.2.12" -criteria = "safe-to-deploy" - [[exemptions.wasm-bindgen]] version = "0.2.127" criteria = "safe-to-deploy" @@ -2326,10 +2265,6 @@ criteria = "safe-to-deploy" version = "0.52.6" criteria = "safe-to-deploy" -[[exemptions.wit-bindgen]] -version = "0.57.1" -criteria = "safe-to-deploy" - [[exemptions.wnaf]] version = "0.14.0" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index ea2a1b884..f4d16232f 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -1,6 +1,109 @@ # cargo-vet imports lock +[[publisher.arbitrary]] +version = "1.4.2" +when = "2025-08-14" +user-id = 696 +user-login = "fitzgen" +user-name = "Nick Fitzgerald" + +[[publisher.bumpalo]] +version = "3.20.3" +when = "2026-05-22" +user-id = 696 +user-login = "fitzgen" +user-name = "Nick Fitzgerald" + +[[publisher.core-foundation-sys]] +version = "0.8.4" +when = "2023-04-03" +user-id = 5946 +user-login = "jrmuizel" +user-name = "Jeff Muizelaar" + +[[publisher.cranelift-assembler-x64]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-assembler-x64-meta]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-bforest]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-bitset]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-codegen]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-codegen-meta]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-codegen-shared]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-control]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-entity]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-frontend]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-isle]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-native]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.cranelift-srcgen]] +version = "0.135.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.pulley-interpreter]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.pulley-macros]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.regalloc2]] +version = "0.15.2" +when = "2026-07-20" +user-id = 3726 +user-login = "cfallin" +user-name = "Chris Fallin" + [[publisher.serde_spanned]] version = "1.1.1" when = "2026-03-31" @@ -78,6 +181,93 @@ user-id = 4484 user-login = "hsivonen" user-name = "Henri Sivonen" +[[publisher.wasip2]] +version = "1.0.4+wasi-0.2.12" +when = "2026-06-12" +user-id = 1 +user-login = "alexcrichton" +user-name = "Alex Crichton" + +[[publisher.wasm-encoder]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[publisher.wasm-metadata]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[publisher.wasmparser]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[publisher.wasmprinter]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[publisher.wasmtime]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-environ]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-component-macro]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-component-util]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-core]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-cranelift]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-fiber]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-jit-debug]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-jit-icache-coherence]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-unwinder]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-versioned-export-macros]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + +[[publisher.wasmtime-internal-wit-bindgen]] +version = "48.0.5" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasmtime" + [[publisher.winnow]] version = "1.0.4" when = "2026-07-13" @@ -85,6 +275,576 @@ user-id = 6743 user-login = "epage" user-name = "Ed Page" +[[publisher.wit-bindgen]] +version = "0.57.1" +when = "2026-04-17" +trusted-publisher = "github:bytecodealliance/wit-bindgen" + +[[publisher.wit-component]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[publisher.wit-parser]] +version = "0.254.2" +when = "2026-10-02" +trusted-publisher = "github:bytecodealliance/wasm-tools" + +[[audits.bytecodealliance.wildcard-audits.arbitrary]] +who = "Nick Fitzgerald " +criteria = "safe-to-deploy" +user-id = 696 # Nick Fitzgerald (fitzgen) +start = "2020-01-14" +end = "2026-08-21" +notes = "I am an author of this crate." + +[[audits.bytecodealliance.wildcard-audits.bumpalo]] +who = "Nick Fitzgerald " +criteria = "safe-to-deploy" +user-id = 696 # Nick Fitzgerald (fitzgen) +start = "2019-03-16" +end = "2026-08-21" + +[[audits.bytecodealliance.wildcard-audits.cranelift-assembler-x64]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-assembler-x64-meta]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-bforest]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-bitset]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-codegen]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-codegen-meta]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-codegen-shared]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-control]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-entity]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-frontend]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-isle]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-native]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.cranelift-srcgen]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.pulley-interpreter]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.pulley-macros]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.regalloc2]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +user-id = 3726 # Chris Fallin (cfallin) +start = "2021-12-03" +end = "2026-08-21" +notes = "We (Bytecode Alliance) are the primary authors of regalloc2 and co-develop it with Cranelift/Wasmtime, with the same code-review, testing/fuzzing, and security standards." + +[[audits.bytecodealliance.wildcard-audits.wasip2]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +user-id = 1 # Alex Crichton (alexcrichton) +start = "2025-08-10" +end = "2026-08-21" +notes = """ +This is a Bytecode Alliance authored crate. +""" + +[[audits.bytecodealliance.wildcard-audits.wasm-encoder]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2025-08-14" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasm-metadata]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2026-02-10" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmparser]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2025-08-14" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmprinter]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2025-08-14" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-environ]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-component-macro]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-component-util]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-core]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-cranelift]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-fiber]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-jit-debug]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-jit-icache-coherence]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-unwinder]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-versioned-export-macros]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wasmtime-internal-wit-bindgen]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasmtime" +start = "2026-01-07" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wit-bindgen]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wit-bindgen" +start = "2025-08-13" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wit-component]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2025-08-14" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.wildcard-audits.wit-parser]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +trusted-publisher = "github:bytecodealliance/wasm-tools" +start = "2025-08-14" +end = "2027-01-08" +notes = "The Bytecode Alliance is the author of this crate" + +[[audits.bytecodealliance.audits.adler2]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "2.0.0" +notes = "Fork of the original `adler` crate, zero unsfae code, works in `no_std`, does what it says on th tin." + +[[audits.bytecodealliance.audits.allocator-api2]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +delta = "0.2.18 -> 0.2.20" +notes = """ +The changes appear to be reasonable updates from Rust's stdlib imported into +`allocator-api2`'s copy of this code. +""" + +[[audits.bytecodealliance.audits.anes]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.1.6" +notes = "Contains no unsafe code, no IO, no build.rs." + +[[audits.bytecodealliance.audits.atomic-waker]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.1.2" +notes = "Contains `unsafe` code but it's well-documented and scoped to what it's intended to be doing. Otherwise a well-focused and straightforward crate." + +[[audits.bytecodealliance.audits.cipher]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "0.4.4" +notes = "Most unsafe is hidden by `inout` dependency; only remaining unsafe is raw-splitting a slice and an unreachable hint. Older versions of this regularly reach ~150k daily downloads." + +[[audits.bytecodealliance.audits.cobs]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.2.3" +notes = "No `unsafe` code in the crate and no usage of `std`" + +[[audits.bytecodealliance.audits.cobs]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.2.3 -> 0.3.0" +notes = "Nothing out of the ordinary, virtually no unsafe code." + +[[audits.bytecodealliance.audits.core-foundation-sys]] +who = "Dan Gohman " +criteria = "safe-to-deploy" +delta = "0.8.4 -> 0.8.6" +notes = """ +The changes here are all typical bindings updates: new functions, types, and +constants. I have not audited all the bindings for ABI conformance. +""" + +[[audits.bytecodealliance.audits.cpp_demangle]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.3.5 -> 0.4.3" +notes = "No substantive changes to `unsafe` code and otherwise all looks good." + +[[audits.bytecodealliance.audits.cpp_demangle]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.3 -> 0.5.1" +notes = "API refactors, support for new symbols, nothing awry." + +[[audits.bytecodealliance.audits.embedded-io]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.4.0" +notes = "No `unsafe` code and only uses `std` in ways one would expect the crate to do so." + +[[audits.bytecodealliance.audits.embedded-io]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.0 -> 0.6.1" +notes = "Major updates, but almost all safe code. Lots of pruning/deletions, nothing out of the ordrinary." + +[[audits.bytecodealliance.audits.fallible-iterator]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.2.0 -> 0.3.0" +notes = """ +This major version update has a few minor breaking changes but everything +this crate has to do with iterators and `Result` and such. No `unsafe` or +anything like that, all looks good. +""" + +[[audits.bytecodealliance.audits.heck]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.4.0" +notes = "Contains `forbid_unsafe` and only uses `std::fmt` from the standard library. Otherwise only contains string manipulation." + +[[audits.bytecodealliance.audits.heck]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.1 -> 0.5.0" +notes = "Minor changes for a `no_std` upgrade but otherwise everything looks as expected." + +[[audits.bytecodealliance.audits.iana-time-zone-haiku]] +who = "Dan Gohman " +criteria = "safe-to-deploy" +version = "0.1.2" + +[[audits.bytecodealliance.audits.leb128fmt]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.1.0" +notes = "Well-scoped crate do doing LEB encoding with no `unsafe` code and does what it says on the tin." + +[[audits.bytecodealliance.audits.matchers]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.1.0" + +[[audits.bytecodealliance.audits.matchers]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.1.0 -> 0.2.0" +notes = "Some unsafe code, but not more than before. Nothing awry." + +[[audits.bytecodealliance.audits.nu-ansi-term]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.46.0" +notes = "one use of unsafe to call windows specific api to get console handle." + +[[audits.bytecodealliance.audits.nu-ansi-term]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.46.0 -> 0.50.1" +notes = "Lots of stylistic/rust-related changes, plus new features, but nothing out of the ordrinary." + +[[audits.bytecodealliance.audits.nu-ansi-term]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.50.1 -> 0.50.3" +notes = "CI changes, Rust changes, nothing out of the ordinary." + +[[audits.bytecodealliance.audits.num-traits]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "0.2.19" +notes = "As advertised: a numeric library. The only `unsafe` is from some float-to-int conversions, which seems expected." + +[[audits.bytecodealliance.audits.percent-encoding]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "2.2.0" +notes = """ +This crate is a single-file crate that does what it says on the tin. There are +a few `unsafe` blocks related to utf-8 validation which are locally verifiable +as correct and otherwise this crate is good to go. +""" + +[[audits.bytecodealliance.audits.postcard]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.0.8" +notes = """ +I've audited the unsafe code to do what it looks like it's doing. Otherwise the +crate is a standard serializer/deserializer crate. +""" + +[[audits.bytecodealliance.audits.postcard]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "1.0.8 -> 1.1.3" +notes = "Substantial updates, but nothing out of the ordinary one would expect from a serialization crate. Minor `unsafe` updates, but nothing major from what was already there." + +[[audits.bytecodealliance.audits.rustc_version]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.0 -> 0.4.1" +notes = "Minor changes, nothing worrisome" + +[[audits.bytecodealliance.audits.semver]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "1.0.17" +notes = "plenty of unsafe pointer and vec tricks, but in well-structured and commented code that appears to be correct" + +[[audits.bytecodealliance.audits.sha1_smol]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.0.1" + +[[audits.bytecodealliance.audits.sharded-slab]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.1.4" +notes = "I always really enjoy reading eliza's code, she left perfect comments at every use of unsafe." + +[[audits.bytecodealliance.audits.shlex]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.1.0" +notes = "Only minor `unsafe` code blocks which look valid and otherwise does what it says on the tin." + +[[audits.bytecodealliance.audits.static_assertions]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "1.1.0" +notes = "No dependencies and completely a compile-time crate as advertised. Uses `unsafe` in one module as a compile-time check only: `mem::transmute` and `ptr::write` are wrapped in an impossible-to-run closure." + +[[audits.bytecodealliance.audits.tinyvec_macros]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.1.0" +notes = """ +This is a trivial crate which only contains a singular macro definition which is +intended to multiplex across the internal representation of a tinyvec, +presumably. This trivially doesn't contain anything bad. +""" + +[[audits.bytecodealliance.audits.tracing-log]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.1.3" +notes = """ +This is a standard adapter between the `log` ecosystem and the `tracing` +ecosystem. There's one `unsafe` block in this crate and it's well-scoped. +""" + +[[audits.bytecodealliance.audits.tracing-log]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.1.3 -> 0.2.0" +notes = "Nothing out of the ordinary, a typical major version update and nothing awry." + +[[audits.bytecodealliance.audits.unarray]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.1.4" +notes = """ +Crate is sound, albeit leaky, and not actively malicious. Probably not the best +crate to use in practice but it's suitable for testing dependencies. +""" + +[[audits.mozilla.wildcard-audits.core-foundation-sys]] +who = "Bobby Holley " +criteria = "safe-to-deploy" +user-id = 5946 # Jeff Muizelaar (jrmuizel) +start = "2020-10-14" +end = "2023-05-04" +renew = false +notes = "I've reviewed every source contribution that was neither authored nor reviewed by Mozilla." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.wildcard-audits.unicode-normalization]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -121,6 +881,24 @@ end = "2024-06-16" notes = "Maintained by Henri Sivonen who works at Mozilla." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.adler2]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "2.0.0 -> 2.0.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.allocator-api2]] +who = "Nicolas Silva " +criteria = "safe-to-deploy" +version = "0.2.18" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.allocator-api2]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.2.20 -> 0.2.21" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.bit-set]] who = "Aria Beingessner " criteria = "safe-to-deploy" @@ -171,6 +949,18 @@ criteria = "safe-to-deploy" delta = "0.2.1 -> 0.2.2" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.core-foundation-sys]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.8.6 -> 0.8.7" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.fallible-iterator]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +version = "0.2.0" +aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" + [[audits.mozilla.audits.fnv]] who = "Bobby Holley " criteria = "safe-to-deploy" @@ -226,6 +1016,12 @@ criteria = "safe-to-deploy" delta = "0.17.0 -> 0.17.1" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.heck]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.4.0 -> 0.4.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.hex]] who = "Simon Friedberger " criteria = "safe-to-deploy" @@ -252,6 +1048,68 @@ version = "11.1.5" notes = "Small random number generator, explicitly not cryptographically secure, no use of unsafe code, no dependencies" aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" +[[audits.mozilla.audits.percent-encoding]] +who = "Valentin Gosu " +criteria = "safe-to-deploy" +delta = "2.2.0 -> 2.3.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.percent-encoding]] +who = "Valentin Gosu " +criteria = "safe-to-deploy" +delta = "2.3.0 -> 2.3.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.percent-encoding]] +who = "edgul " +criteria = "safe-to-deploy" +delta = "2.3.1 -> 2.3.2" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rustc_version]] +who = "Nika Layzell " +criteria = "safe-to-deploy" +version = "0.4.0" +notes = """ +Use of powerful capabilities is limited to invoking `rustc -vV` to get version +information for parsing version information. +""" +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + +[[audits.mozilla.audits.semver]] +who = "Ben Dean-Kawamura " +criteria = "safe-to-deploy" +delta = "1.0.16 -> 1.0.28" +notes = "Very few changes, mostly removes support for older Rust versions. Some unsafe code refactored, but it seemed to be functionally equivalent to me." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.semver]] +who = "Bobby Holley " +criteria = "safe-to-deploy" +delta = "1.0.17 -> 1.0.16" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.sharded-slab]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.1.4 -> 0.1.7" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.shlex]] +who = "Max Inden " +criteria = "safe-to-deploy" +delta = "1.1.0 -> 1.3.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.shlex]] +who = "Emilio Cobos Álvarez " +criteria = "safe-to-deploy" +delta = "1.3.0 -> 2.0.1" +notes = """ +Mostly removes some deprecated and unsound APIs. +""" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.simdutf8]] who = "Henri Sivonen " criteria = "safe-to-deploy" @@ -318,6 +1176,12 @@ criteria = "safe-to-deploy" delta = "0.16.1 -> 0.16.2" aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" +[[audits.mozilla.audits.tinyvec_macros]] +who = "Drew Willcoxon " +criteria = "safe-to-deploy" +delta = "0.1.0 -> 0.1.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.unicode-linebreak]] who = "Jan-Erik Rediger " criteria = "safe-to-deploy"