From 0fbe539ef7d74b1dcb98dd468b35f394812a35cf Mon Sep 17 00:00:00 2001 From: Alexander Medvedev Date: Fri, 20 Nov 2026 15:03:44 +0100 Subject: [PATCH 1/2] Update to stock CPython 3.15.2 --- .github/workflows/release.yaml | 14 ++++---- .github/workflows/test.yaml | 4 +-- Cargo.lock | 25 +++++++------ Cargo.toml | 2 +- alt-build.rs | 2 +- build.rs | 65 ++++++++++++++++++++-------------- pyo3-config.txt | 4 +-- runtime/Cargo.toml | 2 +- runtime/pyo3-config-clippy.txt | 4 +-- src/prelink.rs | 10 +++--- 10 files changed, 71 insertions(+), 61 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a5541716..e2f7e90a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,7 @@ jobs: matrix: config: - { - os: "ubuntu-24.04", + os: "ubuntu-latest", arch: "amd64", maturin_target: "x86_64", wasiSDK: "x86_64-linux", @@ -36,7 +36,7 @@ jobs: targetDir: "target/release", } - { - os: "ubuntu-24.04", + os: "ubuntu-latest", arch: "aarch64", maturin_target: "aarch64", wasiSDK: "x86_64-linux", @@ -83,7 +83,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.14.0" + python-version: "3.15" - name: Install latest Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -91,7 +91,7 @@ jobs: targets: wasm32-wasip2 wasm32-wasip3 ${{ matrix.config.target }} - uses: Swatinem/rust-cache@v2 - if: matrix.config.os == 'ubuntu-24.04' && matrix.config.arch == 'amd64' + if: matrix.config.os == 'ubuntu-latest' && matrix.config.arch == 'amd64' with: shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" cache-on-failure: "false" @@ -166,7 +166,7 @@ jobs: checksums: name: Generate checksums - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest needs: release steps: - name: Set the release version (tag) @@ -204,7 +204,7 @@ jobs: sdist: name: Build Python sdist - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -226,7 +226,7 @@ jobs: name: Update canary release needs: [checksums, sdist] if: github.ref == 'refs/heads/main' - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4baee1d6..58818740 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,7 +22,7 @@ jobs: fail-fast: false matrix: config: - - { os: "ubuntu-24.04", wasiSDK: "x86_64-linux" } + - { os: "ubuntu-latest", wasiSDK: "x86_64-linux" } - { os: "macos-latest", wasiSDK: "arm64-macos" } - { os: "windows-latest", wasiSDK: "x86_64-windows" } runs-on: ${{ matrix.config.os }} @@ -53,7 +53,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.14.0" + python-version: "3.15" - name: Test shell: bash diff --git a/Cargo.lock b/Cargo.lock index 9b2378ac..0aa71808 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,9 +2329,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" dependencies = [ "libc", "num-bigint", @@ -2345,18 +2345,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" dependencies = [ "libc", "pyo3-build-config", @@ -2364,27 +2364,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6360055a..84eaad77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ wit-component = "0.258.0" wasmparser = "0.258.0" indexmap = "2.6.0" heck = "0.5.0" -pyo3 = { version = "0.28.3", features = [ +pyo3 = { version = "0.30.1", features = [ "abi3-py39", "extension-module", ], optional = true } diff --git a/alt-build.rs b/alt-build.rs index c47551da..46295b16 100644 --- a/alt-build.rs +++ b/alt-build.rs @@ -22,7 +22,7 @@ fn main() { "libc.so.zst", "libcomponentize_py_runtime_async.so.zst", "libcomponentize_py_runtime_sync.so.zst", - "libpython3.14.so.zst", + "libpython3.15.so.zst", "libwasi-emulated-getpid.so.zst", "libwasi-emulated-mman.so.zst", "libwasi-emulated-process-clocks.so.zst", diff --git a/build.rs b/build.rs index 56f7e95d..557f1a05 100644 --- a/build.rs +++ b/build.rs @@ -36,13 +36,11 @@ const CLANG_EXECUTABLE: &str = "clang.exe"; #[cfg(not(target_os = "windows"))] const CLANG_EXECUTABLE: &str = "clang"; -// TODO: switch to upstream release per -// https://github.com/bytecodealliance/componentize-py/issues/215 const CPYTHON_SOURCE_TARBALL_URL: &str = - "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-34-v2"; -const CPYTHON_SOURCE_TARBALL_BASE_DIR: &str = "dicej-cpython-ff50203"; + "https://www.python.org/ftp/python/3.15.2/Python-3.15.2.tgz"; +const CPYTHON_SOURCE_TARBALL_BASE_DIR: &str = "Python-3.15.2"; -const CPYTHON_BINARY_TARBALL_URL: &str = "https://github.com/dicej/cpython/releases/download/v3.14.0-wasi-sdk-34-v2/cpython-v3.14.0-wasi-sdk-34-v2.tar.zst"; +const CPYTHON_BINARY_TARBALL_URL: &str = "https://github.com/dicej/cpython/releases/download/v3.15.2-wasi-sdk-34/cpython-v3.15.2-wasi-sdk-34.tar.zst"; static TARGETS: &[&str] = &["wasip2", "wasip3"]; @@ -70,7 +68,7 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { let libraries = [ "libcomponentize_py_runtime.so.zst", - "libpython3.14.so.zst", + "libpython3.15.so.zst", "libc.so.zst", "threads/libc.so.zst", "libwasi-emulated-mman.so.zst", @@ -234,10 +232,10 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> { true, )?; - compress(&cpython_wasi_dir, "libpython3.14.so", &library_dir, true)?; + compress(&cpython_wasi_dir, "libpython3.15.so", &library_dir, true)?; } - let path = repo_dir.join("cpython/builddir/wasip2/install/lib/python3.14"); + let path = repo_dir.join("cpython/builddir/wasip2/install/lib/python3.15"); if path.exists() { let mut builder = Builder::new(Encoder::new( @@ -328,7 +326,25 @@ fn add(builder: &mut Builder, root: &Path, path: &Path) -> Result<() fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { let cpython_dir = repo_dir.join("cpython"); if !cpython_dir.exists() { - if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { + let build_from_source = if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { + true + } else { + let url = &env::var("CPYTHON_BINARY_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); + println!("cargo:warning=downloading CPython build from {url}"); + let dir = cpython_dir.join("builddir"); + fs::create_dir_all(&dir)?; + if fetch_extract_zstd(url, &dir).is_ok() { + return Ok(()); + } + fs::remove_dir_all(&cpython_dir)?; + println!( + "cargo:warning=prebuilt CPython build not available at {url}; building from source" + ); + true + }; + + if build_from_source { let url = &env::var("CPYTHON_SOURCE_TARBALL_URL") .unwrap_or_else(|_| CPYTHON_SOURCE_TARBALL_URL.into()); let base_dir = &env::var_os("CPYTHON_SOURCE_TARBALL_BASE_DIR") @@ -336,21 +352,13 @@ fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result< println!("cargo:warning=downloading CPython source code from {url}"); fetch_extract_gzip(url, repo_dir)?; fs::rename(repo_dir.join(base_dir), &cpython_dir)?; - } else { - let url = &env::var("CPYTHON_BINARY_TARBALL_URL") - .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); - println!("cargo:warning=downloading CPython build from {url}"); - let dir = cpython_dir.join("builddir"); - fs::create_dir_all(&dir)?; - fetch_extract_zstd(url, &dir)?; - return Ok(()); } } let cpython_wasi_dir = cpython_dir.join("builddir").join(target); - if !cpython_wasi_dir.join("libpython3.14.so").exists() { + if !cpython_wasi_dir.join("libpython3.15.so").exists() { fs::create_dir_all(&cpython_wasi_dir)?; - if !cpython_wasi_dir.join("libpython3.14.a").exists() { + if !cpython_wasi_dir.join("libpython3.15.a").exists() { let cpython_native_dir = repo_dir.join("cpython/builddir/build"); if !cpython_native_dir.join(PYTHON_EXECUTABLE).exists() { fs::create_dir_all(&cpython_native_dir)?; @@ -385,8 +393,9 @@ fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result< run(Command::new("../../Tools/wasm/wasi-env") .env( "CONFIG_SITE", - "../../Tools/wasm/wasi/config.site-wasm32-wasi", + "../../Platforms/WASI/config.site-wasm32-wasi", ) + .env("HOSTRUNNER", "wasmtime") .env( "CFLAGS", format!("--target=wasm32-{target} -fPIC -I{dir}/deps/include"), @@ -406,6 +415,8 @@ fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result< &format!("--prefix={dir}/install"), "--disable-test-modules", "--enable-ipv6", + "--enable-wasm-dynamic-linking", + "--without-system-libmpdec", ]))?; // Write Modules/Setup.local to force-enable _sqlite3 @@ -417,14 +428,14 @@ fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result< .args(["build_all", "install"]))?; } - // Link libpython3.14.so - now includes libsqlite3.a + // Link libpython3.15.so - now includes libsqlite3.a run(Command::new(wasi_sdk.join("bin/clang")) .arg(format!("--target=wasm32-{target}")) .arg("-shared") .arg("-o") - .arg(cpython_wasi_dir.join("libpython3.14.so")) + .arg(cpython_wasi_dir.join("libpython3.15.so")) .arg("-Wl,--whole-archive") - .arg(cpython_wasi_dir.join("libpython3.14.a")) + .arg(cpython_wasi_dir.join("libpython3.15.a")) .arg("-Wl,--no-whole-archive") .arg(cpython_wasi_dir.join("Modules/_hacl/libHacl_HMAC.a")) .arg(cpython_wasi_dir.join("Modules/_hacl/libHacl_Hash_BLAKE2.a")) @@ -454,7 +465,7 @@ fn write_setup_local(cpython_wasi_dir: &Path) -> Result<()> { let deps_dir = cpython_wasi_dir.join("deps"); // The _sqlite3 module source files (relative to Modules/) - // These are the files that make up the _sqlite3 extension in CPython 3.14 + // These are the files that make up the _sqlite3 extension in CPython 3.15 // Note: blob.c is required - it defines pysqlite_close_all_blobs and pysqlite_blob_setup_types let include_dir = deps_dir.join("include"); let lib_dir = deps_dir.join("lib"); @@ -569,7 +580,7 @@ fn make_runtime( -Clink-args=-Wl,--export-if-defined=__wasm_library_tls_info \ -Clink-args=-shared \ -Clink-args=-L{} \ - -Clink-args=-lpython3.14 \ + -Clink-args=-lpython3.15 \ -Clink-self-contained=n", cpython_wasi_dir.to_str().unwrap() ), @@ -604,7 +615,7 @@ fn make_runtime( } fn fetch_extract_gzip(url: &str, out_dir: &Path) -> Result<()> { - let response = reqwest::blocking::get(url)?; + let response = reqwest::blocking::get(url)?.error_for_status()?; let decoder = flate2::read::GzDecoder::new(response); let mut archive = tar::Archive::new(decoder); archive.unpack(out_dir)?; @@ -612,7 +623,7 @@ fn fetch_extract_gzip(url: &str, out_dir: &Path) -> Result<()> { } fn fetch_extract_zstd(url: &str, out_dir: &Path) -> Result<()> { - let response = reqwest::blocking::get(url)?; + let response = reqwest::blocking::get(url)?.error_for_status()?; let decoder = zstd::Decoder::new(response)?; let mut archive = tar::Archive::new(decoder); archive.unpack(out_dir)?; diff --git a/pyo3-config.txt b/pyo3-config.txt index 57644954..80ce4bae 100644 --- a/pyo3-config.txt +++ b/pyo3-config.txt @@ -1,8 +1,8 @@ implementation=CPython -version=3.14 +version=3.15 shared=true abi3=false -lib_name=python3.14 +lib_name=python3.15 pointer_width=32 build_flags= suppress_build_script_link_lines=false diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 7f8469f4..f7975baa 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1.0.91" once_cell = "1.20.2" -pyo3 = { version="0.28.3", features = ["abi3-py312", "num-bigint"] } +pyo3 = { version = "0.30.1", features = ["abi3-py312", "num-bigint"] } num-bigint = "0.4.6" wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc", "async"] } wit-dylib-ffi = { git = "https://github.com/bytecodealliance/wasm-tools", tag = "v1.258.0", features = ["async-raw"] } diff --git a/runtime/pyo3-config-clippy.txt b/runtime/pyo3-config-clippy.txt index df49ac5f..2bf50df6 100644 --- a/runtime/pyo3-config-clippy.txt +++ b/runtime/pyo3-config-clippy.txt @@ -1,8 +1,8 @@ implementation=CPython -version=3.14 +version=3.15 shared=false abi3=false -lib_name=python3.14 +lib_name=python3.15 pointer_width=64 build_flags= suppress_build_script_link_lines=false diff --git a/src/prelink.rs b/src/prelink.rs index a01a1cf9..e6970de8 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -18,7 +18,7 @@ use crate::{ ComponentizePyConfig, ConfigContext, Library, Multithreading, RawComponentizePyConfig, Target, }; -static NATIVE_EXTENSION_SUFFIXES: &[&str] = &[".cpython-314-wasm32-wasi.so", ".abi3.so"]; +static NATIVE_EXTENSION_SUFFIXES: &[&str] = &[".cpython-315-wasm32-wasi.so", ".abi3.so"]; /// Check if a file starts with the WASM magic bytes (`\0asm`). fn is_wasm_file(path: &Path) -> bool { @@ -71,10 +71,10 @@ pub fn bundled_libraries(target: Target) -> Result> { dl_openable: false, }, Library { - name: "libpython3.14.so".into(), + name: "libpython3.15.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/wasip2/libpython3.14.so.zst" + "/wasip2/libpython3.15.so.zst" ))))?, dl_openable: false, }, @@ -153,10 +153,10 @@ pub fn bundled_libraries(target: Target) -> Result> { dl_openable: false, }, Library { - name: "libpython3.14.so".into(), + name: "libpython3.15.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/wasip3/libpython3.14.so.zst" + "/wasip3/libpython3.15.so.zst" ))))?, dl_openable: false, }, From caaae5b9baa5b2c480b1db4c52c5232597d0635b Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Fri, 20 Nov 2026 12:40:43 -0700 Subject: [PATCH 2/2] update matrix-math example --- build.rs | 28 +++++++++------------------- examples/matrix-math/README.md | 4 ++-- tests/componentize.rs | 4 +++- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/build.rs b/build.rs index 557f1a05..cc7566bd 100644 --- a/build.rs +++ b/build.rs @@ -326,25 +326,7 @@ fn add(builder: &mut Builder, root: &Path, path: &Path) -> Result<() fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { let cpython_dir = repo_dir.join("cpython"); if !cpython_dir.exists() { - let build_from_source = if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { - true - } else { - let url = &env::var("CPYTHON_BINARY_TARBALL_URL") - .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); - println!("cargo:warning=downloading CPython build from {url}"); - let dir = cpython_dir.join("builddir"); - fs::create_dir_all(&dir)?; - if fetch_extract_zstd(url, &dir).is_ok() { - return Ok(()); - } - fs::remove_dir_all(&cpython_dir)?; - println!( - "cargo:warning=prebuilt CPython build not available at {url}; building from source" - ); - true - }; - - if build_from_source { + if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { let url = &env::var("CPYTHON_SOURCE_TARBALL_URL") .unwrap_or_else(|_| CPYTHON_SOURCE_TARBALL_URL.into()); let base_dir = &env::var_os("CPYTHON_SOURCE_TARBALL_BASE_DIR") @@ -352,6 +334,14 @@ fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result< println!("cargo:warning=downloading CPython source code from {url}"); fetch_extract_gzip(url, repo_dir)?; fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + } else { + let url = &env::var("CPYTHON_BINARY_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); + println!("cargo:warning=downloading CPython build from {url}"); + let dir = cpython_dir.join("builddir"); + fs::create_dir_all(&dir)?; + fetch_extract_zstd(url, &dir)?; + return Ok(()); } } diff --git a/examples/matrix-math/README.md b/examples/matrix-math/README.md index ec27e3fc..9fb44843 100644 --- a/examples/matrix-math/README.md +++ b/examples/matrix-math/README.md @@ -24,14 +24,14 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v38.0.0. ``` cargo install --version 38.0.0 wasmtime-cli pip install componentize-py==0.25.0 -curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.2/numpy-wasi.tar.gz +curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.3/numpy-wasi.tar.gz tar xf numpy-wasi.tar.gz ``` ## Running the demo ``` -componentize-py -d ../../wit -w matrix-math componentize app -o matrix-math.wasm +componentize-py -d ../../wit -w matrix-math componentize --target wasm32-wasip3 app -o matrix-math.wasm wasmtime run matrix-math.wasm '[[1, 2], [4, 5], [6, 7]]' '[[1, 2, 3], [4, 5, 6]]' ``` diff --git a/tests/componentize.rs b/tests/componentize.rs index e89adca4..be18ae12 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -304,6 +304,8 @@ fn matrix_math_example() -> anyhow::Result<()> { "-w", "matrix-math", "componentize", + "--target", + "wasm32-wasip3", "app", "-o", "matrix-math.wasm", @@ -466,7 +468,7 @@ fn venv_path(path: &Path) -> PathBuf { fn install_numpy(path: &Path) -> anyhow::Result<()> { let bytes = reqwest::blocking::get( - "https://github.com/dicej/wasi-wheels/releases/download/v0.0.2/numpy-wasi.tar.gz", + "https://github.com/dicej/wasi-wheels/releases/download/v0.0.3/numpy-wasi.tar.gz", )? .error_for_status()? .bytes()?;