From 883edfbf89c04fb5165649abc9bda7efe8e719bd Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:25:47 -0700 Subject: [PATCH 01/18] fix(rust): stream bundled runtime installation Compare runtime assets in bounded chunks, stage changed entries without whole-image buffers, and validate the archive before atomic publication. Preserve read-only warm caches, repair unreadable or corrupt files, and reject duplicate normalized destinations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/src/embeddedcli.rs | 763 ++++++++++++++++++++++++++++++++++------ 1 file changed, 652 insertions(+), 111 deletions(-) diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 574ed42a6f..41827e9f25 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -13,19 +13,27 @@ //! A non-atomic write, a multi-process race, or antivirus quarantining the //! freshly-written executable can leave a truncated or corrupt image that, if //! handed back as "good", fails to launch (e.g. Windows `ERROR_BAD_EXE_FORMAT`). -//! Installation therefore: extracts to a unique temp file in the target dir, +//! Full CLI installation therefore: extracts to a unique temp file in the target dir, //! fsyncs and marks it executable, verifies the staged bytes against the //! trusted in-memory image, atomically renames it into place, re-verifies the //! published file, and records an integrity marker. Subsequent runs trust an //! existing install only after a cheap re-check (size marker + executable-image //! header); anything that looks truncated or quarantined is re-extracted, and //! the whole publish is retried before surfacing a clear, actionable error. +//! +//! Runtime assets are compared and extracted with bounded buffers instead of +//! retaining whole native images in memory. Matching files need only read +//! access; changed files are staged beside their targets and published only +//! after archive validation, including the gzip trailer. Installation is +//! atomic per file, not a transaction across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure // std-only logic that doesn't touch the embedded archive, so they compile // whenever the binary is bundled *or* we're building the test harness — // the standard `cargo test --no-default-features` job has `has_bundled_cli` // off but still needs to exercise them. +#[cfg(has_bundled_cli)] +use std::collections::HashSet; #[cfg(any(has_bundled_cli, test))] use std::fs; #[cfg(has_bundled_cli)] @@ -276,17 +284,17 @@ const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { fs::create_dir_all(install_dir) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; - install_hostless_assets(install_dir, archive)?; - install_runtime_pair(install_dir, archive)?; + let root = fs::canonicalize(install_dir) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let mut required = vec![RUNTIME_BINARY_NAME, RUNTIME_NODE_NAME]; #[cfg(feature = "bundled-in-process")] - install_runtime_library(install_dir, archive)?; - Ok(install_dir.join(RUNTIME_BINARY_NAME)) -} - -#[cfg(has_bundled_cli)] -fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { + required.push(RUNTIME_LIBRARY_NAME); + let mut pending = Vec::new(); + let mut seen = HashSet::new(); + let mut changed = HashSet::new(); let gz = flate2::read::GzDecoder::new(archive); let mut tar = tar::Archive::new(gz); + for entry in tar .entries() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? @@ -300,17 +308,6 @@ fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), Emb .path() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? .into_owned(); - let file_name = path.file_name().and_then(|name| name.to_str()); - if path == Path::new(CLI_BINARY_NAME) - || matches!( - file_name, - Some("copilot_runtime.dll") - | Some("libcopilot_runtime.dylib") - | Some("libcopilot_runtime.so") - ) - { - continue; - } if path.is_absolute() || path.components().any(|component| { matches!( @@ -326,90 +323,283 @@ fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), Emb format!("unsafe embedded runtime asset path: {}", path.display()), )); } - let mut bytes = Vec::with_capacity(entry.size() as usize); - entry - .read_to_end(&mut bytes) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - let target = install_dir.join(&path); - if fs::read(&target) - .map(|installed| installed == bytes) - .unwrap_or(false) - { + let path: PathBuf = path + .components() + .filter(|component| *component != std::path::Component::CurDir) + .collect(); + let file_name = path.file_name().and_then(|name| name.to_str()); + let is_library = matches!( + file_name, + Some("copilot_runtime.dll") + | Some("libcopilot_runtime.dylib") + | Some("libcopilot_runtime.so") + ); + if path == Path::new(CLI_BINARY_NAME) { + continue; + } + if is_library { + #[cfg(feature = "bundled-in-process")] + if path != Path::new(RUNTIME_LIBRARY_NAME) { + continue; + } + #[cfg(not(feature = "bundled-in-process"))] continue; } + if !seen.insert(path.clone()) { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!("duplicate embedded runtime asset path: {}", path.display()), + )); + } + if let Some(index) = required.iter().position(|name| path == Path::new(name)) { + if entry.size() == 0 { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!("embedded runtime artifact is empty: {}", path.display()), + )); + } + required.remove(index); + } + let target = root.join(&path); let parent = target.parent().ok_or_else(|| { EmbeddedCliError::with_message( EmbeddedCliErrorKind::Archive, format!("embedded runtime asset has no parent: {}", path.display()), ) })?; - fs::create_dir_all(parent) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; - let tmp = write_temp_file(parent, &bytes)?; - #[cfg(unix)] + check_runtime_asset_parent(&root, parent, false)?; + match existing_runtime_file(&entry, &target)? { + Some(mut file) => { + if !runtime_entry_matches(&mut entry, &mut file)? { + changed.insert(path); + } + } + None => { + check_runtime_asset_parent(&root, parent, true)?; + pending.push(stage_runtime_entry(&mut entry, &target)?); + } + } + } + + // tar stops at its end marker, before GzDecoder necessarily verifies the + // gzip CRC and length trailer. Validate those before publishing any files. + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !required.is_empty() { + return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); + } + // A failed comparison has consumed part of the trusted entry. Rewind the + // archive once for all such files rather than buffering their prefixes. + // Cold installs and valid warm installs need only the first pass. + if !changed.is_empty() { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar + .entries() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? { - use std::os::unix::fs::PermissionsExt; - let mode = entry.header().mode().unwrap_or(0o644) & 0o777; - fs::set_permissions(&tmp, fs::Permissions::from_mode(mode)) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let mut entry = + entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path: PathBuf = entry + .path() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + .components() + .filter(|component| *component != std::path::Component::CurDir) + .collect(); + if changed.contains(&path) { + pending.push(stage_runtime_entry(&mut entry, &root.join(path))?); + } } - if let Err(error) = publish(&tmp, &target) { - let _ = fs::remove_file(&tmp); - return Err(error); + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + } + for staged in pending { + publish(&staged.temporary, &staged.target)?; + } + Ok(install_dir.join(RUNTIME_BINARY_NAME)) +} + +#[cfg(has_bundled_cli)] +fn check_runtime_asset_parent( + root: &Path, + parent: &Path, + create: bool, +) -> Result<(), EmbeddedCliError> { + let mut current = root.to_path_buf(); + for component in parent + .strip_prefix(root) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + .components() + { + current.push(component); + if create { + match fs::create_dir(¤t) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e)), + } + } + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(e) if !create && e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), + }; + if !metadata.is_dir() { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime asset parent is not a directory: {}", + current.display() + ), + )); } } Ok(()) } #[cfg(has_bundled_cli)] -fn install_runtime_pair(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { - install_adjacent_file(install_dir, archive, RUNTIME_NODE_NAME, "runtime.node")?; - install_adjacent_file( - install_dir, - archive, - RUNTIME_BINARY_NAME, - "copilot runtime wrapper", - ) +struct StagedRuntimeFile { + temporary: PathBuf, + target: PathBuf, } -#[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] -fn install_runtime_library(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { - install_adjacent_file( - install_dir, - archive, - RUNTIME_LIBRARY_NAME, - "in-process FFI runtime library", - ) +#[cfg(has_bundled_cli)] +impl Drop for StagedRuntimeFile { + fn drop(&mut self) { + if let Err(error) = fs::remove_file(&self.temporary) + && error.kind() != std::io::ErrorKind::NotFound + { + warn!(path = %self.temporary.display(), %error, "failed to remove staged runtime asset"); + } + } } #[cfg(has_bundled_cli)] -fn install_adjacent_file( - install_dir: &Path, - archive: &[u8], - file_name: &str, - label: &str, +fn existing_runtime_file( + entry: &tar::Entry<'_, R>, + target: &Path, +) -> Result, EmbeddedCliError> { + let metadata = match fs::symlink_metadata(target) { + Ok(metadata) if metadata.is_file() => Some(metadata), + Ok(_) => { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!("runtime asset is not a regular file: {}", target.display()), + )); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), + }; + let size = entry.size(); + let matches = metadata + .as_ref() + .is_some_and(|metadata| metadata.len() == size); + #[cfg(unix)] + let mode = entry + .header() + .mode() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + & 0o777; + #[cfg(unix)] + let matches = { + use std::os::unix::fs::PermissionsExt; + matches + && metadata + .as_ref() + .is_some_and(|metadata| metadata.permissions().mode() & 0o777 == mode) + }; + if matches { + match fs::File::open(target) { + Ok(file) => return Ok(Some(file)), + Err(e) => { + tracing::debug!(path = %target.display(), error = %e, + "existing runtime asset cannot be read; repairing"); + } + } + } + Ok(None) +} + +#[cfg(has_bundled_cli)] +fn runtime_entry_matches( + entry: &mut tar::Entry<'_, R>, + existing: &mut fs::File, +) -> Result { + let mut buffer = [0u8; 64 * 1024]; + let mut installed = [0u8; 64 * 1024]; + let mut remaining = entry.size(); + while remaining > 0 { + let length = remaining.min(buffer.len() as u64) as usize; + entry + .read_exact(&mut buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + remaining -= length as u64; + if let Err(e) = existing.read_exact(&mut installed[..length]) { + tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); + return Ok(false); + } + if installed[..length] != buffer[..length] { + return Ok(false); + } + } + match existing.read(&mut installed[..1]) { + Ok(read) => Ok(read == 0), + Err(e) => { + tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); + Ok(false) + } + } +} + +#[cfg(has_bundled_cli)] +fn stage_runtime_entry( + entry: &mut tar::Entry<'_, R>, + target: &Path, +) -> Result { + let parent = target.parent().expect("runtime asset has a checked parent"); + let (temporary, mut file) = create_temp_file(parent)?; + let staged = StagedRuntimeFile { + temporary, + target: target.to_path_buf(), + }; + let result = write_runtime_entry(entry, &mut file); + // Close handles before cleanup or replacement, including on Windows. + drop(file); + result?; + Ok(staged) +} + +#[cfg(has_bundled_cli)] +fn write_runtime_entry( + entry: &mut tar::Entry<'_, R>, + file: &mut fs::File, ) -> Result<(), EmbeddedCliError> { - let target = install_dir.join(file_name); - let bytes = extract_binary(archive, file_name)?; - if bytes.is_empty() { + let size = entry.size(); + // Entry is already limited by tar, and take enforces that limit at the + // copy boundary too. A premature EOF must not publish a truncated file. + let written = std::io::copy(&mut (&mut *entry).take(size), &mut *file) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + if written != size { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!("embedded {label} is empty"), + format!("runtime entry size mismatch: read {written} bytes, expected {size}"), )); } - if fs::read(&target) - .map(|installed| installed == bytes) - .unwrap_or(false) + #[cfg(unix)] { - return Ok(()); - } - let tmp = write_temp_file(install_dir, &bytes)?; - if let Err(e) = publish(&tmp, &target) { - let _ = fs::remove_file(&tmp); - return Err(e); + use std::os::unix::fs::PermissionsExt; + let mode = entry + .header() + .mode() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + & 0o777; + file.set_permissions(fs::Permissions::from_mode(mode)) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; } - tracing::debug!(path = %target.display(), %label, "embedded runtime artifact installed"); - Ok(()) + file.sync_all() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)) } #[cfg(has_bundled_cli)] @@ -556,27 +746,7 @@ fn publish_verified( /// bytes to disk and marking it executable on unix before returning its path. #[cfg(any(has_bundled_cli, test))] fn write_temp_file(dir: &Path, contents: &[u8]) -> Result { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let unique = format!( - ".copilot-cli.tmp.{}.{}.{}", - std::process::id(), - COUNTER.fetch_add(1, Ordering::Relaxed), - nanos - ); - let tmp = dir.join(unique); - - // `create_new` guarantees we never clobber a sibling's in-flight temp - // file (the pid + counter + nanos name already makes that practically - // impossible). - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let (tmp, mut file) = create_temp_file(dir)?; if let Err(e) = file .write_all(contents) @@ -602,24 +772,35 @@ fn write_temp_file(dir: &Path, contents: &[u8]) -> Result Result<(PathBuf, fs::File), EmbeddedCliError> { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let tmp = dir.join(format!( + ".copilot-cli.tmp.{}.{}.{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed), + nanos + )); + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + Ok((tmp, file)) +} + /// Atomically move the staged temp file onto `final_path`. /// -/// `rename` replaces the target atomically on POSIX, but on Windows it fails -/// when the target already exists — so on that error we remove the stale file -/// and retry. The remove-then-rename is the only non-atomic window, and it's -/// guarded upstream: callers re-verify the published file and, on a lost race, -/// accept a peer's identical install instead of erroring. +/// Rust uses rename on POSIX and MoveFileExW with MOVEFILE_REPLACE_EXISTING on +/// Windows. Never unlink the destination on failure: readers must retain the +/// previous complete file if replacement is blocked. #[cfg(any(has_bundled_cli, test))] fn publish(tmp: &Path, final_path: &Path) -> Result<(), EmbeddedCliError> { - match fs::rename(tmp, final_path) { - Ok(()) => Ok(()), - Err(_) if final_path.exists() => { - let _ = fs::remove_file(final_path); - fs::rename(tmp, final_path) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)) - } - Err(e) => Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)), - } + fs::rename(tmp, final_path).map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)) } /// Read the file at `path` and confirm it byte-for-byte matches the trusted @@ -1121,4 +1302,364 @@ mod tests { dir.path().join("2.0.0") ); } + + #[cfg(has_bundled_cli)] + fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> Vec { + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut entries = vec![ + (RUNTIME_BINARY_NAME, b"wrapper".as_slice(), 0o755), + (RUNTIME_NODE_NAME, b"runtime".as_slice(), 0o755), + ]; + #[cfg(feature = "bundled-in-process")] + entries.push((RUNTIME_LIBRARY_NAME, b"library".as_slice(), 0o644)); + entries.extend_from_slice(extra); + for (name, bytes, mode) in entries { + let mut header = tar::Header::new_gnu(); + // Raw names also let the installer see traversal fixtures which + // Builder::append_data would reject before reaching product code. + header.as_mut_bytes()[..name.len()].copy_from_slice(name.as_bytes()); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_cksum(); + archive.append(&header, bytes).expect("append fixture"); + } + archive.into_inner().unwrap().finish().unwrap() + } + + #[cfg(has_bundled_cli)] + fn assert_no_runtime_temps(dir: &Path) { + for entry in fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + assert!( + !entry + .file_name() + .to_string_lossy() + .starts_with(".copilot-cli.tmp.") + ); + if entry.file_type().unwrap().is_dir() { + assert_no_runtime_temps(&entry.path()); + } + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_cold_install_and_warm_reuse_preserve_bytes_and_modes() { + let dir = tempfile::tempdir().unwrap(); + let data = vec![0xAB; 3 * 64 * 1024 + 17]; + let archive = runtime_fixture(&[("nested/asset", &data, 0o640)]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + assert_eq!(wrapper, dir.path().join(RUNTIME_BINARY_NAME)); + let asset = dir.path().join("nested/asset"); + assert_eq!(fs::read(&asset).unwrap(), data); + let modified = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1234567890); + fs::File::options() + .write(true) + .open(&asset) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(modified)) + .unwrap(); + + install_runtime(dir.path(), &archive).unwrap(); + + assert_eq!(fs::metadata(&asset).unwrap().modified().unwrap(), modified); + assert_eq!(fs::read(&asset).unwrap(), data); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(&asset).unwrap().permissions().mode() & 0o777, + 0o640 + ); + assert_eq!( + fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_repairs_same_size_corruption_truncation_and_extra_bytes() { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let runtime = dir.path().join(RUNTIME_NODE_NAME); + install_runtime(dir.path(), &archive).unwrap(); + + for corrupt in [b"runtimX".as_slice(), b"run", b"", b"runtime plus garbage"] { + fs::write(&runtime, corrupt).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!(fs::read(&runtime).unwrap(), b"runtime"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_repairs_missing_execute_permission() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o644)).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!( + fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_archive_errors_do_not_publish_and_clean_up_staging() { + let dir = tempfile::tempdir().unwrap(); + let valid = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); + let mut invalid_crc = valid.clone(); + let crc = invalid_crc.len() - 8; + invalid_crc[crc] ^= 0xFF; + let mut invalid_length = valid.clone(); + let length = invalid_length.len() - 4; + invalid_length[length] ^= 0xFF; + let truncated_trailer = valid[..valid.len() - 1].to_vec(); + let truncated_body = valid[..valid.len() / 2].to_vec(); + for archive in [ + invalid_crc, + invalid_length, + truncated_trailer, + truncated_body, + ] { + let runtime = dir.path().join(RUNTIME_NODE_NAME); + fs::write(&runtime, b"previous complete runtime").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(runtime).unwrap(), b"previous complete runtime"); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_short_entry_is_rejected_without_publishing() { + let dir = tempfile::tempdir().unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_path(RUNTIME_NODE_NAME).unwrap(); + header.set_size(128 * 1024); + header.set_mode(0o755); + header.set_cksum(); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(header.as_bytes()).unwrap(); + encoder.write_all(b"short").unwrap(); + let archive = encoder.finish().unwrap(); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_missing_required_entry_is_rejected_before_publish() { + let dir = tempfile::tempdir().unwrap(); + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_size(7); + header.set_mode(0o755); + header.set_cksum(); + archive + .append_data(&mut header, RUNTIME_BINARY_NAME, b"wrapper".as_slice()) + .unwrap(); + let archive = archive.into_inner().unwrap().finish().unwrap(); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_rejects_traversal_and_cleans_preceding_entries() { + let dir = tempfile::tempdir().unwrap(); + let install_dir = dir.path().join("install"); + let archive = runtime_fixture(&[("../escaped", b"bad", 0o644)]); + assert!(install_runtime(&install_dir, &archive).is_err()); + assert!(!dir.path().join("escaped").exists()); + assert!(!install_dir.join(RUNTIME_BINARY_NAME).exists()); + assert_no_runtime_temps(&install_dir); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_rejects_symlink_parents_and_targets() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("asset"), b"outside").unwrap(); + symlink(outside.path(), dir.path().join("nested")).unwrap(); + let archive = runtime_fixture(&[("nested/asset", b"new", 0o644)]); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(outside.path().join("asset")).unwrap(), b"outside"); + assert_no_runtime_temps(dir.path()); + fs::remove_file(dir.path().join("nested")).unwrap(); + symlink( + outside.path().join("asset"), + dir.path().join(RUNTIME_NODE_NAME), + ) + .unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(outside.path().join("asset")).unwrap(), b"outside"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn concurrent_runtime_installers_publish_complete_files() { + let dir = tempfile::tempdir().unwrap(); + let data = vec![0xAB; 256 * 1024 + 1]; + let archive = runtime_fixture(&[("nested/asset", &data, 0o644)]); + let barrier = std::sync::Barrier::new(6); + std::thread::scope(|scope| { + for _ in 0..6 { + scope.spawn(|| { + barrier.wait(); + install_runtime(dir.path(), &archive).unwrap(); + }); + } + }); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), data); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_duplicate_destinations_fail_on_cold_and_warm_installs() { + for name in ["asset", "./asset"] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("asset", b"first", 0o644), (name, b"last", 0o644)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join("asset").exists()); + assert_no_runtime_temps(dir.path()); + + fs::write(dir.path().join("asset"), b"last").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join("asset")).unwrap(), b"last"); + assert_no_runtime_temps(dir.path()); + } + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(RUNTIME_NODE_NAME, b"", 0o755)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] + #[test] + fn runtime_library_aliases_are_rejected_before_repair() { + let alias = format!("./{RUNTIME_LIBRARY_NAME}"); + for duplicate in [b"another".as_slice(), b""] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(&alias, duplicate, 0o644)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_LIBRARY_NAME).exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + let library = dir.path().join(RUNTIME_LIBRARY_NAME); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(&library).unwrap(), b"library"); + fs::write(&library, b"corrupt").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(&library).unwrap(), b"corrupt"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn warm_runtime_install_needs_no_writable_cache() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); + install_runtime(dir.path(), &archive).unwrap(); + fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); + let result = install_runtime(dir.path(), &archive); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o755)).unwrap(); + result.unwrap(); + if !write_denied { + eprintln!("read-only permission enforcement unavailable (e.g. privileged user)"); + fs::remove_file(dir.path().join("write-probe")).unwrap(); + } + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"asset"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_repairs_unreadable_but_replaceable_file() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("asset", b"trusted", 0o000)]); + let asset = dir.path().join("asset"); + fs::write(&asset, b"corrupt").unwrap(); + fs::set_permissions(&asset, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::File::open(&asset).is_ok() { + eprintln!("unreadable permission enforcement unavailable (e.g. privileged user)"); + } + let result = install_runtime(dir.path(), &archive); + let mode = fs::metadata(&asset).unwrap().permissions().mode() & 0o777; + fs::set_permissions(&asset, fs::Permissions::from_mode(0o600)).unwrap(); + result.unwrap(); + assert_eq!(mode, 0o000); + assert_eq!(fs::read(asset).unwrap(), b"trusted"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_replacement_preserves_open_reader_contents() { + let dir = tempfile::tempdir().unwrap(); + let asset = dir.path().join("asset"); + let original = runtime_fixture(&[("asset", b"old complete file", 0o644)]); + install_runtime(dir.path(), &original).unwrap(); + let mut reader = fs::File::open(&asset).unwrap(); + let replacement = runtime_fixture(&[("asset", b"new complete file", 0o644)]); + install_runtime(dir.path(), &replacement).unwrap(); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"old complete file"); + assert_eq!(fs::read(&asset).unwrap(), b"new complete file"); + assert_no_runtime_temps(dir.path()); + } + + #[test] + fn failed_publish_does_not_remove_previous_file() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("installed"); + fs::write(&target, b"previous complete file").unwrap(); + assert!(publish(&dir.path().join("missing-temporary"), &target).is_err()); + assert_eq!(fs::read(target).unwrap(), b"previous complete file"); + } } From b148263e931aaed99a6b6c930e7b68bdfa549cca Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:27:27 -0700 Subject: [PATCH 02/18] test(rust): add reproducible runtime installer memory benchmark Exercise the public installer in isolated cold, warm, corrupt and truncated-cache processes. Record matched release-build memory and latency measurements, output identities, separate allocator diagnostics, and native-platform limitations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.toml | 1 + rust/README.md | 11 +- rust/benchmarks/runtime_install.md | 172 +++++++++++++++++++++++++ rust/benchmarks/runtime_install.py | 195 +++++++++++++++++++++++++++++ rust/examples/runtime_install.rs | 24 ++++ 5 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 rust/benchmarks/runtime_install.md create mode 100644 rust/benchmarks/runtime_install.py create mode 100644 rust/examples/runtime_install.rs diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4495d3928c..220283c9b7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,6 +15,7 @@ include = [ "src/**/*", "build/**/*", "examples/**/*", + "benchmarks/**/*", "tests/**/*", "build.rs", "Cargo.toml", diff --git a/rust/README.md b/rust/README.md index cdc8e07c05..8f4f23481e 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1158,7 +1158,16 @@ if let Some(path) = install_bundled_runtime() { ``` This extracts `copilot-runtime` together with adjacent `runtime.node`, then -returns the wrapper path. +returns the wrapper path. Runtime installation streams archive entries and +compares existing files in bounded chunks, without buffering whole native +images. A valid warm cache requires only read access. Missing or corrupt files +are staged in uniquely created sibling files, with archive permissions +preserved, then atomically replaced after archive validation. Replacement is +atomic per file, not across the entire bundle. + +An installer-only memory and latency reproduction is available in +[`benchmarks/runtime_install.md`](benchmarks/runtime_install.md). It exercises +the real bundled runtime without starting a client or contacting a model. ### Download cache (build-time, embed mode) diff --git a/rust/benchmarks/runtime_install.md b/rust/benchmarks/runtime_install.md new file mode 100644 index 0000000000..aa9e82ded1 --- /dev/null +++ b/rust/benchmarks/runtime_install.md @@ -0,0 +1,172 @@ +# Bundled runtime installation memory + +`examples/runtime_install.rs` exercises the public `install_bundled_runtime()` +API in a standalone process. It also checks that a second call returns the +same path. It does not launch the runtime, read credentials, authenticate, or +make service requests. Build-time downloads use the SDK's normal verified +public release artifacts. + +## Reproduce on macOS + +Requires Python 3.11+, the pinned Rust toolchain, and macOS's `/usr/bin/time`. +From `rust/` on the fixed revision: + +```sh +work=$(mktemp -d) +git worktree add --detach "$work/baseline" a675b55531a9dfc647ee015e32d74279568550f3 +cp examples/runtime_install.rs "$work/baseline/rust/examples/runtime_install.rs" +( + cd "$work/baseline/rust" + CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install +) +CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install + +python3 benchmarks/runtime_install.py \ + "$work/baseline/rust/target/release/examples/runtime_install" --runs 5 \ + > "$work/baseline.json" +python3 benchmarks/runtime_install.py \ + target/release/examples/runtime_install --runs 5 > "$work/fixed.json" +``` + +Both builds must use the same runtime version, feature set, toolchain and +profile. These commands use default features (`bundled-cli`), release +optimization level 3 and debug level 1. Check the SHA-256 of each build's +`target/release/build/github-copilot-sdk-*/out/copilot_runtime.archive`; the +hashes must agree. Do not compare binaries with different runtime releases. + +Every measured run gets a fresh process and temporary `HOME`, including the +platform cache. The environment contains only `HOME`, `TMPDIR`, and a system +`PATH`. For warm/repair runs, a separate, unmeasured installer process seeds +that home before the measured process starts. Cohorts are: + +- **Cold:** no installed files. +- **Warm:** all installed files already match. +- **Corrupt:** flip the final byte of installed `runtime.node`, retaining its size. +- **Truncated:** truncate installed `runtime.node` to 1,024 bytes. + +Each process waits before installation and remains alive for one second +after installation so the harness can observe retained memory. The harness +then lets it exit. Output sizes, permissions and streaming SHA-256 hashes +are collected outside the measured process after exit; every run must produce +the same file inventory. The JSON contains only relative output paths. +Temporary homes are removed after each run. Compare the two JSON `outputs` +objects for exact equality, not just the runtime file: + +```sh +python3 - "$work/baseline.json" "$work/fixed.json" <<'PY' +import json, sys +before, after = [json.load(open(path)) for path in sys.argv[1:]] +assert before["outputs"] == after["outputs"] +print(len(after["outputs"]), "identical installed files") +PY +``` + +## Measurement definitions + +**Peak RSS** and **peak physical footprint** are the process-lifetime kernel +high-water marks reported by `/usr/bin/time -l`. **Retained RSS** and +**retained physical footprint** are `proc_pid_rusage(RUSAGE_INFO_V0)`'s +`ri_resident_size` and `ri_phys_footprint`, sampled after the one-second idle +window. They are total process values, not baseline-subtracted allocations. +RSS includes resident file-backed pages such as the embedded compressed +archive; physical footprint is the kernel's charged-memory accounting and +is not interchangeable with RSS or live heap size. + +Elapsed time comes from Rust's `Instant` around the first installation call, +excluding startup, the second cached call, idle waits, and output hashing. +Runs are independent processes on a shared host, not CPU-isolated trials. +"Cold" means an empty installation cache, not flushed filesystem pages. +Small samples and environmental noise limit latency conclusions. + +## Observed before and after + +Measured on 2026-09-15 with an Apple M4 Pro, 48 GiB RAM, macOS 26.6.2 +(25G83), `aarch64-apple-darwin`, rustc 1.94.0 +(`4a4ef493e`, LLVM 21.1.8). Baseline SDK revision: +`a675b55531a9dfc647ee015e32d74279568550f3` (`0.0.0-dev`). +Installer-only fixed revision: +`883edfbf89c04fb5165649abc9bda7efe8e719bd`. +The fixed build changes only the runtime installer; runtime version, +dependencies, release profile and probe source are identical. + +Five runs per cohort per build. Values are **median (minimum-maximum)**. +Memory uses MiB (1,048,576 bytes). + +| Cohort | Before retained physical MiB | After retained physical MiB | Before seconds | After seconds | +| --- | --- | --- | --- | --- | +| Cold | 155.438 (154.563-156.047) | 2.000 (1.969-2.078) | 0.880 (0.863-1.037) | 0.633 (0.618-0.703) | +| Warm | 174.360 (172.735-174.907) | 1.938 (1.875-1.953) | 1.031 (0.997-1.092) | 0.747 (0.359-0.788) | +| Corrupt | 174.376 (171.438-174.485) | 2.063 (2.000-2.079) | 1.043 (1.018-1.075) | 1.007 (0.991-1.092) | +| Truncated | 174.376 (172.938-174.422) | 1.907 (1.891-1.907) | 1.038 (0.978-1.090) | 0.830 (0.797-0.863) | + +| Cohort | Before peak physical MiB | After peak physical MiB | Before peak RSS MiB | After peak RSS MiB | +| --- | --- | --- | --- | --- | +| Cold | 155.469 (154.594-156.079) | 2.032 (2.000-2.110) | 200.109 (199.219-200.703) | 46.766 (46.734-46.844) | +| Warm | 174.391 (172.766-174.938) | 1.969 (1.907-1.985) | 219.016 (217.391-219.547) | 46.688 (46.641-46.719) | +| Corrupt | 174.407 (171.469-174.516) | 2.094 (2.032-2.110) | 219.016 (216.078-219.125) | 46.812 (46.734-46.812) | +| Truncated | 174.407 (172.969-174.454) | 1.938 (1.922-1.938) | 219.031 (217.578-219.062) | 46.672 (46.641-46.672) | + +| Cohort | Before retained RSS MiB | After retained RSS MiB | +| --- | --- | --- | +| Cold | 200.078 (199.188-200.672) | 46.719 (46.688-46.797) | +| Warm | 218.984 (217.359-219.516) | 46.641 (46.594-46.672) | +| Corrupt | 218.984 (216.047-219.094) | 46.766 (46.688-46.766) | +| Truncated | 219.000 (217.547-219.031) | 46.625 (46.594-46.625) | + +The baseline/fixed median initial physical footprints were approximately +1.58/1.58 MiB before installation. All 68 output files were byte-identical, +with the same sizes and modes, across both builds and all cohorts. + +| Input/output | Identity | +| --- | --- | +| Public runtime release | `github/copilot-cli` `v1.0.84-8`, `github-copilot-1.0.84-8-darwin-arm64.tgz` | +| Filtered embedded runtime archive SHA-256 | `6c43b789080fc06b25d406af8fae709daa99f0724c4d290cc8a31160c5a3ad64` | +| Installed `runtime.node` | 71,166,736 bytes; mode `0755`; SHA-256 `839cd681c72cb92f27697d5e3e3ee96d7bb8df4234ffb5f7b442956828ec173a` | +| Installed `copilot-runtime` | 386,992 bytes; mode `0755`; SHA-256 `b1bb3f4b9f6ee4c4d72eb206e68647fe0716a0d87e874472b411f4abc5608a8f` | +| Total installed output | 68 files; 95,721,242 bytes | +| Complete inventory SHA-256 | `dde1d211fd60d155cd5ec647ee0f5123371498a2e65cb9a6c52675b1b711ce17` | +| Baseline probe binary SHA-256 | `2ad5536afcc8052ad15d2af726cccd86b503f06068b690be7c6268e5a7557bb1` | +| Fixed probe binary SHA-256 | `4f37ab4cd9d5e6a5ece13900c85f6d6858732cfce1c80f3f2e47953edecabc2d` | + +The inventory digest hashes UTF-8 +`json.dumps(outputs, sort_keys=True, separators=(",", ":"))`. +Probe binary hashes identify the measured executables, not reproducible-build +expectations: build paths and debug information may differ on another host. +Output file sizes are identity checks, not memory measurements. + +## Separate allocation diagnostics + +Run profiling separately from the comparison above: + +```sh +python3 benchmarks/runtime_install.py target/release/examples/runtime_install \ + --runs 1 --cohort warm --diagnostics "$work/fixed-diagnostics" \ + > "$work/fixed-instrumented.json" +``` + +This enables `MallocStackLogging` and `MallocStackLoggingNoCompact` and saves +`vmmap -summary`, live allocations, and allocation history. These tools can +require local profiling permission. Raw diagnostics may contain local paths; +keep them local rather than attaching them to an issue or PR. + +In a separate baseline warm run, allocation history recorded two +71,172,096-byte VM allocations through `embeddedcli::install_runtime`, one +also through `std::fs::read`. The size is the page-rounded native runtime +payload. After installation, `vmmap` reported 165.7 MiB in +`MALLOC_LARGE (empty)` regions. These were freed allocations retained by the +allocator, not evidence of a live-object leak. The fixed warm diagnostic +did not contain either runtime-sized allocation or a `MALLOC_LARGE (empty)` +region. Instrumented memory/timing values are not included in the tables. + +The fix uses bounded entry copying and 64 KiB comparison buffers. Cold and +valid warm installs traverse the archive once. Same-size corrupt files can +require one additional traversal to recover bytes consumed during comparison. +Warm verification does not stage or write matching files, so valid read-only +caches remain usable. Changed files require temporary disk space until +archive validation completes; publication is atomic per file, not for the +whole bundle. Abrupt process termination can leave temporary files, as before. + +These measurements cover only bundled runtime installation on macOS arm64. +They do not measure full CLI installation, authentication, model sessions, +in-process runtime loading, or an application's overall performance. +Native Windows, Linux, and other architecture measurements were not available. diff --git a/rust/benchmarks/runtime_install.py b/rust/benchmarks/runtime_install.py new file mode 100644 index 0000000000..4170f573f8 --- /dev/null +++ b/rust/benchmarks/runtime_install.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""macOS installer-only memory/latency probe using fresh processes and homes.""" + +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path +import select +import signal +import subprocess +import sys +import tempfile +import time + + +class RusageInfoV0(ctypes.Structure): + _fields_ = [("uuid", ctypes.c_uint8 * 16)] + [ + (name, ctypes.c_uint64) + for name in ( + "user_time", "system_time", "pkg_idle_wkups", "interrupt_wkups", + "pageins", "wired_size", "resident_size", "phys_footprint", + "proc_start_abstime", "proc_exit_abstime", + ) + ] + + +def memory(pid): + info = RusageInfoV0() + if LIBPROC.proc_pid_rusage(pid, 0, ctypes.byref(info)) != 0: + raise OSError(ctypes.get_errno(), "proc_pid_rusage failed") + return info.resident_size, info.phys_footprint + + +def read_line(process): + if not select.select([process.stdout], [], [], 120)[0]: + raise TimeoutError("installer did not respond within 120 seconds") + line = process.stdout.readline().strip() + if not line: + raise RuntimeError("installer exited without a response") + return line.split() + + +def run(binary, home, diagnostic_dir=None): + env = {"HOME": str(home), "PATH": "/usr/bin:/bin", "TMPDIR": str(home)} + if diagnostic_dir: + env.update(MallocStackLogging="1", MallocStackLoggingNoCompact="1") + with tempfile.TemporaryFile(mode="w+") as stderr: + process = subprocess.Popen( + ["/usr/bin/time", "-l", str(binary)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, + text=True, env=env, bufsize=1, + ) + pid = None + try: + ready, pid = read_line(process) + assert ready == "ready" + pid = int(pid) + initial_rss, initial_physical = memory(pid) + process.stdin.write("\n") + process.stdin.flush() + while not select.select([process.stdout], [], [], 0.005)[0]: + # The OS lifetime high-water marks below capture short spikes + # that sampling could miss. This also checks process liveness. + memory(pid) + installed, elapsed = read_line(process) + assert installed == "installed" + time.sleep(1) + retained_rss, retained_physical = memory(pid) + if diagnostic_dir: + diagnostic_dir.mkdir(parents=True, exist_ok=True) + for name, command in ( + ("vmmap.txt", ["vmmap", "-summary", str(pid)]), + ("allocations.txt", [ + "malloc_history", str(pid), "-allBySize", "-fullStacks" + ]), + ("history.txt", [ + "malloc_history", str(pid), "-allEvents", "-noContent" + ]), + ): + with (diagnostic_dir / name).open("w") as output: + subprocess.run( + command, stdout=output, stderr=subprocess.STDOUT, + check=True, timeout=120, + ) + process.stdin.write("\n") + process.stdin.flush() + if process.wait(timeout=120): + raise RuntimeError("installer process failed") + stderr.seek(0) + metrics = {} + for line in stderr: + for label, key in ( + ("maximum resident set size", "peak_rss_bytes"), + ("peak memory footprint", "peak_physical_bytes"), + ): + if label in line: + metrics[key] = int(line.split()[0]) + if len(metrics) != 2: + raise RuntimeError("macOS time did not report both memory metrics") + return dict( + elapsed_seconds=float(elapsed), + initial_rss_bytes=initial_rss, + initial_physical_bytes=initial_physical, + retained_rss_bytes=retained_rss, + retained_physical_bytes=retained_physical, + **metrics, + ) + finally: + if process.poll() is None: + if pid is not None: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.kill() + process.wait() + process.stdin.close() + process.stdout.close() + + +def digest(path): + with path.open("rb") as file: + return hashlib.file_digest(file, "sha256").hexdigest() + + +def inventory(home): + root = home / "Library/Caches/github-copilot-sdk/cli" + return { + path.relative_to(root).as_posix(): { + "bytes": path.stat().st_size, + "sha256": digest(path), + "mode": oct(path.stat().st_mode & 0o777), + } + for path in sorted(root.rglob("*")) if path.is_file() + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--diagnostics", type=Path) + parser.add_argument("--cohort", choices=("cold", "warm", "corrupt", "truncated")) + args = parser.parse_args() + binary = args.binary.resolve(strict=True) + if args.runs < 1: + parser.error("--runs must be positive") + result = { + "binary_sha256": digest(binary), "instrumented": bool(args.diagnostics), + "runs": [], "outputs": None, + } + cohorts = [args.cohort] if args.cohort else ("cold", "warm", "corrupt", "truncated") + for cohort in cohorts: + for iteration in range(args.runs): + with tempfile.TemporaryDirectory(prefix="sdk-runtime-") as directory: + home = Path(directory) + expected = None + if cohort != "cold": + run(binary, home) + expected = inventory(home) + runtime = next(home.rglob("runtime.node")) + if cohort == "corrupt": + with runtime.open("r+b") as file: + file.seek(-1, os.SEEK_END) + byte = file.read(1) + file.seek(-1, os.SEEK_END) + file.write(bytes([byte[0] ^ 0xFF])) + elif cohort == "truncated": + with runtime.open("r+b") as file: + file.truncate(1024) + diagnostic_dir = ( + args.diagnostics / f"{cohort}-{iteration}" + if args.diagnostics else None + ) + measured = run(binary, home, diagnostic_dir) + outputs = inventory(home) + if expected is not None: + assert outputs == expected, "repair changed installed output" + if result["outputs"] is not None: + assert outputs == result["outputs"], "output identity changed" + result["outputs"] = outputs + result["runs"].append(dict(cohort=cohort, iteration=iteration, **measured)) + print(f"{cohort} {iteration}: {measured}", file=sys.stderr) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + if sys.platform != "darwin": + sys.exit("This measurement harness requires macOS.") + LIBPROC = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + LIBPROC.proc_pid_rusage.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + LIBPROC.proc_pid_rusage.restype = ctypes.c_int + main() diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs new file mode 100644 index 0000000000..4a18409465 --- /dev/null +++ b/rust/examples/runtime_install.rs @@ -0,0 +1,24 @@ +//! Installer-only probe. No CLI subprocess, authentication, or model requests. +//! Run with an isolated HOME; see benchmarks/runtime_install.py. + +use std::io::{self, Write}; +use std::time::Instant; + +use github_copilot_sdk::install_bundled_runtime; + +fn main() -> Result<(), Box> { + println!("ready {}", std::process::id()); + io::stdout().flush()?; + let mut line = String::new(); + io::stdin().read_line(&mut line)?; + + let start = Instant::now(); + let path = install_bundled_runtime().ok_or("bundled runtime installation failed")?; + let elapsed = start.elapsed(); + assert_eq!(install_bundled_runtime().as_ref(), Some(&path)); + println!("installed {}", elapsed.as_secs_f64()); + io::stdout().flush()?; + line.clear(); + io::stdin().read_line(&mut line)?; + Ok(()) +} From 54bfc6bbb2cd7fac28ae8e13de0a7a8deb9b40d1 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:57:30 -0700 Subject: [PATCH 03/18] docs(rust): remove checked-in installer benchmarks Keep the small installer example and runtime behavior documentation, while retaining measurement evidence outside the repository. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.toml | 1 - rust/README.md | 6 +- rust/benchmarks/runtime_install.md | 172 ------------------------- rust/benchmarks/runtime_install.py | 195 ----------------------------- rust/examples/runtime_install.rs | 2 +- 5 files changed, 4 insertions(+), 372 deletions(-) delete mode 100644 rust/benchmarks/runtime_install.md delete mode 100644 rust/benchmarks/runtime_install.py diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 220283c9b7..4495d3928c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,6 @@ include = [ "src/**/*", "build/**/*", "examples/**/*", - "benchmarks/**/*", "tests/**/*", "build.rs", "Cargo.toml", diff --git a/rust/README.md b/rust/README.md index 8f4f23481e..1e450b6030 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1165,9 +1165,9 @@ are staged in uniquely created sibling files, with archive permissions preserved, then atomically replaced after archive validation. Replacement is atomic per file, not across the entire bundle. -An installer-only memory and latency reproduction is available in -[`benchmarks/runtime_install.md`](benchmarks/runtime_install.md). It exercises -the real bundled runtime without starting a client or contacting a model. +The [`runtime_install` example](examples/runtime_install.rs) exercises the real +bundled runtime without starting a client or contacting a model. Run it with +an isolated `HOME` to keep its installation cache separate. ### Download cache (build-time, embed mode) diff --git a/rust/benchmarks/runtime_install.md b/rust/benchmarks/runtime_install.md deleted file mode 100644 index aa9e82ded1..0000000000 --- a/rust/benchmarks/runtime_install.md +++ /dev/null @@ -1,172 +0,0 @@ -# Bundled runtime installation memory - -`examples/runtime_install.rs` exercises the public `install_bundled_runtime()` -API in a standalone process. It also checks that a second call returns the -same path. It does not launch the runtime, read credentials, authenticate, or -make service requests. Build-time downloads use the SDK's normal verified -public release artifacts. - -## Reproduce on macOS - -Requires Python 3.11+, the pinned Rust toolchain, and macOS's `/usr/bin/time`. -From `rust/` on the fixed revision: - -```sh -work=$(mktemp -d) -git worktree add --detach "$work/baseline" a675b55531a9dfc647ee015e32d74279568550f3 -cp examples/runtime_install.rs "$work/baseline/rust/examples/runtime_install.rs" -( - cd "$work/baseline/rust" - CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install -) -CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install - -python3 benchmarks/runtime_install.py \ - "$work/baseline/rust/target/release/examples/runtime_install" --runs 5 \ - > "$work/baseline.json" -python3 benchmarks/runtime_install.py \ - target/release/examples/runtime_install --runs 5 > "$work/fixed.json" -``` - -Both builds must use the same runtime version, feature set, toolchain and -profile. These commands use default features (`bundled-cli`), release -optimization level 3 and debug level 1. Check the SHA-256 of each build's -`target/release/build/github-copilot-sdk-*/out/copilot_runtime.archive`; the -hashes must agree. Do not compare binaries with different runtime releases. - -Every measured run gets a fresh process and temporary `HOME`, including the -platform cache. The environment contains only `HOME`, `TMPDIR`, and a system -`PATH`. For warm/repair runs, a separate, unmeasured installer process seeds -that home before the measured process starts. Cohorts are: - -- **Cold:** no installed files. -- **Warm:** all installed files already match. -- **Corrupt:** flip the final byte of installed `runtime.node`, retaining its size. -- **Truncated:** truncate installed `runtime.node` to 1,024 bytes. - -Each process waits before installation and remains alive for one second -after installation so the harness can observe retained memory. The harness -then lets it exit. Output sizes, permissions and streaming SHA-256 hashes -are collected outside the measured process after exit; every run must produce -the same file inventory. The JSON contains only relative output paths. -Temporary homes are removed after each run. Compare the two JSON `outputs` -objects for exact equality, not just the runtime file: - -```sh -python3 - "$work/baseline.json" "$work/fixed.json" <<'PY' -import json, sys -before, after = [json.load(open(path)) for path in sys.argv[1:]] -assert before["outputs"] == after["outputs"] -print(len(after["outputs"]), "identical installed files") -PY -``` - -## Measurement definitions - -**Peak RSS** and **peak physical footprint** are the process-lifetime kernel -high-water marks reported by `/usr/bin/time -l`. **Retained RSS** and -**retained physical footprint** are `proc_pid_rusage(RUSAGE_INFO_V0)`'s -`ri_resident_size` and `ri_phys_footprint`, sampled after the one-second idle -window. They are total process values, not baseline-subtracted allocations. -RSS includes resident file-backed pages such as the embedded compressed -archive; physical footprint is the kernel's charged-memory accounting and -is not interchangeable with RSS or live heap size. - -Elapsed time comes from Rust's `Instant` around the first installation call, -excluding startup, the second cached call, idle waits, and output hashing. -Runs are independent processes on a shared host, not CPU-isolated trials. -"Cold" means an empty installation cache, not flushed filesystem pages. -Small samples and environmental noise limit latency conclusions. - -## Observed before and after - -Measured on 2026-09-15 with an Apple M4 Pro, 48 GiB RAM, macOS 26.6.2 -(25G83), `aarch64-apple-darwin`, rustc 1.94.0 -(`4a4ef493e`, LLVM 21.1.8). Baseline SDK revision: -`a675b55531a9dfc647ee015e32d74279568550f3` (`0.0.0-dev`). -Installer-only fixed revision: -`883edfbf89c04fb5165649abc9bda7efe8e719bd`. -The fixed build changes only the runtime installer; runtime version, -dependencies, release profile and probe source are identical. - -Five runs per cohort per build. Values are **median (minimum-maximum)**. -Memory uses MiB (1,048,576 bytes). - -| Cohort | Before retained physical MiB | After retained physical MiB | Before seconds | After seconds | -| --- | --- | --- | --- | --- | -| Cold | 155.438 (154.563-156.047) | 2.000 (1.969-2.078) | 0.880 (0.863-1.037) | 0.633 (0.618-0.703) | -| Warm | 174.360 (172.735-174.907) | 1.938 (1.875-1.953) | 1.031 (0.997-1.092) | 0.747 (0.359-0.788) | -| Corrupt | 174.376 (171.438-174.485) | 2.063 (2.000-2.079) | 1.043 (1.018-1.075) | 1.007 (0.991-1.092) | -| Truncated | 174.376 (172.938-174.422) | 1.907 (1.891-1.907) | 1.038 (0.978-1.090) | 0.830 (0.797-0.863) | - -| Cohort | Before peak physical MiB | After peak physical MiB | Before peak RSS MiB | After peak RSS MiB | -| --- | --- | --- | --- | --- | -| Cold | 155.469 (154.594-156.079) | 2.032 (2.000-2.110) | 200.109 (199.219-200.703) | 46.766 (46.734-46.844) | -| Warm | 174.391 (172.766-174.938) | 1.969 (1.907-1.985) | 219.016 (217.391-219.547) | 46.688 (46.641-46.719) | -| Corrupt | 174.407 (171.469-174.516) | 2.094 (2.032-2.110) | 219.016 (216.078-219.125) | 46.812 (46.734-46.812) | -| Truncated | 174.407 (172.969-174.454) | 1.938 (1.922-1.938) | 219.031 (217.578-219.062) | 46.672 (46.641-46.672) | - -| Cohort | Before retained RSS MiB | After retained RSS MiB | -| --- | --- | --- | -| Cold | 200.078 (199.188-200.672) | 46.719 (46.688-46.797) | -| Warm | 218.984 (217.359-219.516) | 46.641 (46.594-46.672) | -| Corrupt | 218.984 (216.047-219.094) | 46.766 (46.688-46.766) | -| Truncated | 219.000 (217.547-219.031) | 46.625 (46.594-46.625) | - -The baseline/fixed median initial physical footprints were approximately -1.58/1.58 MiB before installation. All 68 output files were byte-identical, -with the same sizes and modes, across both builds and all cohorts. - -| Input/output | Identity | -| --- | --- | -| Public runtime release | `github/copilot-cli` `v1.0.84-8`, `github-copilot-1.0.84-8-darwin-arm64.tgz` | -| Filtered embedded runtime archive SHA-256 | `6c43b789080fc06b25d406af8fae709daa99f0724c4d290cc8a31160c5a3ad64` | -| Installed `runtime.node` | 71,166,736 bytes; mode `0755`; SHA-256 `839cd681c72cb92f27697d5e3e3ee96d7bb8df4234ffb5f7b442956828ec173a` | -| Installed `copilot-runtime` | 386,992 bytes; mode `0755`; SHA-256 `b1bb3f4b9f6ee4c4d72eb206e68647fe0716a0d87e874472b411f4abc5608a8f` | -| Total installed output | 68 files; 95,721,242 bytes | -| Complete inventory SHA-256 | `dde1d211fd60d155cd5ec647ee0f5123371498a2e65cb9a6c52675b1b711ce17` | -| Baseline probe binary SHA-256 | `2ad5536afcc8052ad15d2af726cccd86b503f06068b690be7c6268e5a7557bb1` | -| Fixed probe binary SHA-256 | `4f37ab4cd9d5e6a5ece13900c85f6d6858732cfce1c80f3f2e47953edecabc2d` | - -The inventory digest hashes UTF-8 -`json.dumps(outputs, sort_keys=True, separators=(",", ":"))`. -Probe binary hashes identify the measured executables, not reproducible-build -expectations: build paths and debug information may differ on another host. -Output file sizes are identity checks, not memory measurements. - -## Separate allocation diagnostics - -Run profiling separately from the comparison above: - -```sh -python3 benchmarks/runtime_install.py target/release/examples/runtime_install \ - --runs 1 --cohort warm --diagnostics "$work/fixed-diagnostics" \ - > "$work/fixed-instrumented.json" -``` - -This enables `MallocStackLogging` and `MallocStackLoggingNoCompact` and saves -`vmmap -summary`, live allocations, and allocation history. These tools can -require local profiling permission. Raw diagnostics may contain local paths; -keep them local rather than attaching them to an issue or PR. - -In a separate baseline warm run, allocation history recorded two -71,172,096-byte VM allocations through `embeddedcli::install_runtime`, one -also through `std::fs::read`. The size is the page-rounded native runtime -payload. After installation, `vmmap` reported 165.7 MiB in -`MALLOC_LARGE (empty)` regions. These were freed allocations retained by the -allocator, not evidence of a live-object leak. The fixed warm diagnostic -did not contain either runtime-sized allocation or a `MALLOC_LARGE (empty)` -region. Instrumented memory/timing values are not included in the tables. - -The fix uses bounded entry copying and 64 KiB comparison buffers. Cold and -valid warm installs traverse the archive once. Same-size corrupt files can -require one additional traversal to recover bytes consumed during comparison. -Warm verification does not stage or write matching files, so valid read-only -caches remain usable. Changed files require temporary disk space until -archive validation completes; publication is atomic per file, not for the -whole bundle. Abrupt process termination can leave temporary files, as before. - -These measurements cover only bundled runtime installation on macOS arm64. -They do not measure full CLI installation, authentication, model sessions, -in-process runtime loading, or an application's overall performance. -Native Windows, Linux, and other architecture measurements were not available. diff --git a/rust/benchmarks/runtime_install.py b/rust/benchmarks/runtime_install.py deleted file mode 100644 index 4170f573f8..0000000000 --- a/rust/benchmarks/runtime_install.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -"""macOS installer-only memory/latency probe using fresh processes and homes.""" - -import argparse -import ctypes -import hashlib -import json -import os -from pathlib import Path -import select -import signal -import subprocess -import sys -import tempfile -import time - - -class RusageInfoV0(ctypes.Structure): - _fields_ = [("uuid", ctypes.c_uint8 * 16)] + [ - (name, ctypes.c_uint64) - for name in ( - "user_time", "system_time", "pkg_idle_wkups", "interrupt_wkups", - "pageins", "wired_size", "resident_size", "phys_footprint", - "proc_start_abstime", "proc_exit_abstime", - ) - ] - - -def memory(pid): - info = RusageInfoV0() - if LIBPROC.proc_pid_rusage(pid, 0, ctypes.byref(info)) != 0: - raise OSError(ctypes.get_errno(), "proc_pid_rusage failed") - return info.resident_size, info.phys_footprint - - -def read_line(process): - if not select.select([process.stdout], [], [], 120)[0]: - raise TimeoutError("installer did not respond within 120 seconds") - line = process.stdout.readline().strip() - if not line: - raise RuntimeError("installer exited without a response") - return line.split() - - -def run(binary, home, diagnostic_dir=None): - env = {"HOME": str(home), "PATH": "/usr/bin:/bin", "TMPDIR": str(home)} - if diagnostic_dir: - env.update(MallocStackLogging="1", MallocStackLoggingNoCompact="1") - with tempfile.TemporaryFile(mode="w+") as stderr: - process = subprocess.Popen( - ["/usr/bin/time", "-l", str(binary)], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, - text=True, env=env, bufsize=1, - ) - pid = None - try: - ready, pid = read_line(process) - assert ready == "ready" - pid = int(pid) - initial_rss, initial_physical = memory(pid) - process.stdin.write("\n") - process.stdin.flush() - while not select.select([process.stdout], [], [], 0.005)[0]: - # The OS lifetime high-water marks below capture short spikes - # that sampling could miss. This also checks process liveness. - memory(pid) - installed, elapsed = read_line(process) - assert installed == "installed" - time.sleep(1) - retained_rss, retained_physical = memory(pid) - if diagnostic_dir: - diagnostic_dir.mkdir(parents=True, exist_ok=True) - for name, command in ( - ("vmmap.txt", ["vmmap", "-summary", str(pid)]), - ("allocations.txt", [ - "malloc_history", str(pid), "-allBySize", "-fullStacks" - ]), - ("history.txt", [ - "malloc_history", str(pid), "-allEvents", "-noContent" - ]), - ): - with (diagnostic_dir / name).open("w") as output: - subprocess.run( - command, stdout=output, stderr=subprocess.STDOUT, - check=True, timeout=120, - ) - process.stdin.write("\n") - process.stdin.flush() - if process.wait(timeout=120): - raise RuntimeError("installer process failed") - stderr.seek(0) - metrics = {} - for line in stderr: - for label, key in ( - ("maximum resident set size", "peak_rss_bytes"), - ("peak memory footprint", "peak_physical_bytes"), - ): - if label in line: - metrics[key] = int(line.split()[0]) - if len(metrics) != 2: - raise RuntimeError("macOS time did not report both memory metrics") - return dict( - elapsed_seconds=float(elapsed), - initial_rss_bytes=initial_rss, - initial_physical_bytes=initial_physical, - retained_rss_bytes=retained_rss, - retained_physical_bytes=retained_physical, - **metrics, - ) - finally: - if process.poll() is None: - if pid is not None: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.kill() - process.wait() - process.stdin.close() - process.stdout.close() - - -def digest(path): - with path.open("rb") as file: - return hashlib.file_digest(file, "sha256").hexdigest() - - -def inventory(home): - root = home / "Library/Caches/github-copilot-sdk/cli" - return { - path.relative_to(root).as_posix(): { - "bytes": path.stat().st_size, - "sha256": digest(path), - "mode": oct(path.stat().st_mode & 0o777), - } - for path in sorted(root.rglob("*")) if path.is_file() - } - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("binary", type=Path) - parser.add_argument("--runs", type=int, default=5) - parser.add_argument("--diagnostics", type=Path) - parser.add_argument("--cohort", choices=("cold", "warm", "corrupt", "truncated")) - args = parser.parse_args() - binary = args.binary.resolve(strict=True) - if args.runs < 1: - parser.error("--runs must be positive") - result = { - "binary_sha256": digest(binary), "instrumented": bool(args.diagnostics), - "runs": [], "outputs": None, - } - cohorts = [args.cohort] if args.cohort else ("cold", "warm", "corrupt", "truncated") - for cohort in cohorts: - for iteration in range(args.runs): - with tempfile.TemporaryDirectory(prefix="sdk-runtime-") as directory: - home = Path(directory) - expected = None - if cohort != "cold": - run(binary, home) - expected = inventory(home) - runtime = next(home.rglob("runtime.node")) - if cohort == "corrupt": - with runtime.open("r+b") as file: - file.seek(-1, os.SEEK_END) - byte = file.read(1) - file.seek(-1, os.SEEK_END) - file.write(bytes([byte[0] ^ 0xFF])) - elif cohort == "truncated": - with runtime.open("r+b") as file: - file.truncate(1024) - diagnostic_dir = ( - args.diagnostics / f"{cohort}-{iteration}" - if args.diagnostics else None - ) - measured = run(binary, home, diagnostic_dir) - outputs = inventory(home) - if expected is not None: - assert outputs == expected, "repair changed installed output" - if result["outputs"] is not None: - assert outputs == result["outputs"], "output identity changed" - result["outputs"] = outputs - result["runs"].append(dict(cohort=cohort, iteration=iteration, **measured)) - print(f"{cohort} {iteration}: {measured}", file=sys.stderr) - print(json.dumps(result, indent=2)) - - -if __name__ == "__main__": - if sys.platform != "darwin": - sys.exit("This measurement harness requires macOS.") - LIBPROC = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) - LIBPROC.proc_pid_rusage.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] - LIBPROC.proc_pid_rusage.restype = ctypes.c_int - main() diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs index 4a18409465..f0d8170259 100644 --- a/rust/examples/runtime_install.rs +++ b/rust/examples/runtime_install.rs @@ -1,5 +1,5 @@ //! Installer-only probe. No CLI subprocess, authentication, or model requests. -//! Run with an isolated HOME; see benchmarks/runtime_install.py. +//! Run with an isolated HOME to keep the installation cache separate. use std::io::{self, Write}; use std::time::Instant; From 9a1af09ee072bc2c99bec7df73180f96f9454c38 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 13:02:05 -0700 Subject: [PATCH 04/18] fix(rust): reject portable runtime destination aliases Normalize separators and deduplicate case-insensitively before artifact selection. Reject non-portable archive names rather than approximating filesystem-specific Unicode, DOS short-name, or trailing-dot aliases. Cover cold, warm and repair installs through the real installer. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/README.md | 3 + rust/src/embeddedcli.rs | 171 +++++++++++++++++++++++++++++++++------- 2 files changed, 146 insertions(+), 28 deletions(-) diff --git a/rust/README.md b/rust/README.md index 1e450b6030..a362ba5e85 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1164,6 +1164,9 @@ images. A valid warm cache requires only read access. Missing or corrupt files are staged in uniquely created sibling files, with archive permissions preserved, then atomically replaced after archive validation. Replacement is atomic per file, not across the entire bundle. +Names inside bundled archives must be portable ASCII paths and cannot differ +only by case or path separators. This restriction does not apply to the +caller-selected installation directory. The [`runtime_install` example](examples/runtime_install.rs) exercises the real bundled runtime without starting a client or contacting a model. Run it with diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 41827e9f25..f0abedc3c8 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -306,27 +306,14 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result Result Result Result Result { + let invalid = || { + EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!( + "non-portable embedded runtime asset path: {}", + path.display() + ), + ) + }; + // Bundled release assets use ASCII names. Apply the same conservative + // naming rules on every filesystem, without probing a read-only cache. + // Reject Unicode, DOS device/short names and trailing-dot/space aliases + // rather than approximating platform-specific Unicode normalization. + let name = path + .to_str() + .filter(|name| name.is_ascii()) + .ok_or_else(invalid)?; + if name.starts_with(['/', '\\']) { + return Err(invalid()); + } + let mut normalized = PathBuf::new(); + for component in name.split(['/', '\\']) { + if component.is_empty() || component == "." { + continue; + } + if component == ".." + || component.ends_with(['.', ' ']) + || component + .bytes() + .any(|byte| byte.is_ascii_control() || b"<>:\"|?*~".contains(&byte)) + { + return Err(invalid()); + } + let stem = component + .split('.') + .next() + .expect("nonempty component") + .trim_end_matches(' ') + .to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && matches!(stem.as_bytes()[3], b'1'..=b'9')) + { + return Err(invalid()); + } + normalized.push(component); + } + if normalized.as_os_str().is_empty() { + return Err(invalid()); + } + Ok(normalized) +} + #[cfg(has_bundled_cli)] fn check_runtime_asset_parent( root: &Path, @@ -1570,6 +1605,86 @@ mod tests { assert_no_runtime_temps(dir.path()); } + #[cfg(has_bundled_cli)] + #[test] + fn runtime_case_aliases_cannot_overwrite_required_artifacts() { + let names = [ + RUNTIME_NODE_NAME, + RUNTIME_BINARY_NAME, + #[cfg(feature = "bundled-in-process")] + RUNTIME_LIBRARY_NAME, + ]; + for name in names { + let alias = name.to_ascii_uppercase(); + let archive = runtime_fixture(&[(&alias, b"", 0o755)]); + let dir = tempfile::tempdir().unwrap(); + let result = install_runtime(dir.path(), &archive); + assert!(result.is_err(), "accepted alias {alias}: {result:?}"); + assert!(!dir.path().join(name).exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + let original = fs::read(dir.path().join(name)).unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join(name)).unwrap(), original); + fs::write(dir.path().join(name), b"corrupt").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join(name)).unwrap(), b"corrupt"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_mixed_separator_and_case_aliases_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[ + ("nested/asset", b"first", 0o644), + (r".\NESTED\ASSET", b"last", 0o644), + ]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime( + dir.path(), + &runtime_fixture(&[("nested/asset", b"last", 0o644)]), + ) + .unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"last"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_nonportable_alias_paths_are_rejected_before_publish() { + for name in [ + "runtime.node.", + "runtime.node ", + "runtime.node:stream", + "RUNTIM~1.NOD", + "NUL", + "con.txt", + "aux .txt", + "COM1", + "LPT9.txt", + "n\u{00e9}sted/asset", + r"..\escaped", + r"C:\escaped", + r"\\server\share\asset", + ] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(name, b"bad", 0o644)]); + assert!( + install_runtime(dir.path(), &archive).is_err(), + "accepted {name}" + ); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + } + #[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] #[test] fn runtime_library_aliases_are_rejected_before_repair() { From 0d3be10320cb0de177a6641051d5c9c37ce24974 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 13:19:36 -0700 Subject: [PATCH 05/18] docs(rust): keep installer PR focused on code and tests Remove the README additions and standalone measurement example. Preserve reproduction details and measurements in the pull request rather than shipping a benchmark surface. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/README.md | 14 +------------- rust/examples/runtime_install.rs | 24 ------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 rust/examples/runtime_install.rs diff --git a/rust/README.md b/rust/README.md index a362ba5e85..cdc8e07c05 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1158,19 +1158,7 @@ if let Some(path) = install_bundled_runtime() { ``` This extracts `copilot-runtime` together with adjacent `runtime.node`, then -returns the wrapper path. Runtime installation streams archive entries and -compares existing files in bounded chunks, without buffering whole native -images. A valid warm cache requires only read access. Missing or corrupt files -are staged in uniquely created sibling files, with archive permissions -preserved, then atomically replaced after archive validation. Replacement is -atomic per file, not across the entire bundle. -Names inside bundled archives must be portable ASCII paths and cannot differ -only by case or path separators. This restriction does not apply to the -caller-selected installation directory. - -The [`runtime_install` example](examples/runtime_install.rs) exercises the real -bundled runtime without starting a client or contacting a model. Run it with -an isolated `HOME` to keep its installation cache separate. +returns the wrapper path. ### Download cache (build-time, embed mode) diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs deleted file mode 100644 index f0d8170259..0000000000 --- a/rust/examples/runtime_install.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Installer-only probe. No CLI subprocess, authentication, or model requests. -//! Run with an isolated HOME to keep the installation cache separate. - -use std::io::{self, Write}; -use std::time::Instant; - -use github_copilot_sdk::install_bundled_runtime; - -fn main() -> Result<(), Box> { - println!("ready {}", std::process::id()); - io::stdout().flush()?; - let mut line = String::new(); - io::stdin().read_line(&mut line)?; - - let start = Instant::now(); - let path = install_bundled_runtime().ok_or("bundled runtime installation failed")?; - let elapsed = start.elapsed(); - assert_eq!(install_bundled_runtime().as_ref(), Some(&path)); - println!("installed {}", elapsed.as_secs_f64()); - io::stdout().flush()?; - line.clear(); - io::stdin().read_line(&mut line)?; - Ok(()) -} From 61cb61c95f868fd28834458c017a848bc0f7dcb9 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 21:25:43 -0700 Subject: [PATCH 06/18] fix(rust): reuse immutable runtime caches Ignore write-bit differences when validating installed runtime files, while retaining read and execute permission checks. Cover read-only installed files and repair of each missing wrapper execute bit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/src/embeddedcli.rs | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index f0abedc3c8..61a11c14a0 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -531,19 +531,20 @@ fn existing_runtime_file( let matches = metadata .as_ref() .is_some_and(|metadata| metadata.len() == size); + // Immutable caches may strip write bits without invalidating their contents. #[cfg(unix)] let mode = entry .header() .mode() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o777; + & 0o555; #[cfg(unix)] let matches = { use std::os::unix::fs::PermissionsExt; matches && metadata .as_ref() - .is_some_and(|metadata| metadata.permissions().mode() & 0o777 == mode) + .is_some_and(|metadata| metadata.permissions().mode() & 0o555 == mode) }; if matches { match fs::File::open(target) { @@ -1439,12 +1440,14 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &archive).unwrap(); - fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o644)).unwrap(); - install_runtime(dir.path(), &archive).unwrap(); - assert_eq!( - fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, - 0o755 - ); + for mode in [0o644, 0o655, 0o745, 0o754] { + fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } assert_no_runtime_temps(dir.path()); } @@ -1715,6 +1718,20 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); install_runtime(dir.path(), &archive).unwrap(); + let files = [ + RUNTIME_BINARY_NAME, + RUNTIME_NODE_NAME, + "nested/asset", + #[cfg(feature = "bundled-in-process")] + RUNTIME_LIBRARY_NAME, + ]; + let mut read_only_files = Vec::new(); + for name in files { + let path = dir.path().join(name); + let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o555; + fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); + read_only_files.push((path, mode)); + } fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); @@ -1726,6 +1743,12 @@ mod tests { eprintln!("read-only permission enforcement unavailable (e.g. privileged user)"); fs::remove_file(dir.path().join("write-probe")).unwrap(); } + for (path, mode) in read_only_files { + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + mode + ); + } assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"asset"); assert_no_runtime_temps(dir.path()); } From 4ab7f87306b99e25383b56c7f9ea4133da22cea8 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 22:02:40 -0700 Subject: [PATCH 07/18] perf(rust): verify runtime caches with trusted build manifests Generate per-file SHA256, size and mode alongside the unchanged runtime archive. Verify valid warm caches without reading compressed bytes, and perform cold or repair extraction in one bounded forward traversal with manifest and gzip integrity checks before publication. Use sha2 0.11 runtime-detected acceleration with a software fallback, preserving the existing miniz_oxide decompression backend. Cover generated output identity, zero archive reads, immutable cache reuse and mixed-cache repair failures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.lock | 62 ++++- rust/Cargo.toml | 5 +- rust/build/in_process.rs | 44 +++- rust/src/embeddedcli.rs | 541 +++++++++++++++++++++++++++++---------- 4 files changed, 496 insertions(+), 156 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b91eebd06c..5bbafdafef 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -70,6 +70,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -129,6 +138,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -154,6 +172,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -177,8 +204,18 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -546,6 +583,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1453,19 +1499,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4495d3928c..a6cdc1d320 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -59,6 +59,7 @@ getrandom = "0.2" uuid = { version = "1", default-features = false, features = ["v4"] } flate2 = { version = "1", optional = true } tar = { version = "0.4", optional = true } +sha2 = { version = "0.11", default-features = false, optional = true } # LLM inference callback transport: idiomatic HTTP/WebSocket forwarding for the # `CopilotRequestHandler`, plus base64/byte/stream plumbing for the chunk protocol. base64 = "0.22" @@ -125,7 +126,7 @@ required-features = ["test-support"] dirs = "5" flate2 = "1" serde_json = "1" -sha2 = "0.10" +sha2 = { version = "0.11", default-features = false } tar = "0.4" ureq = { version = "2", default-features = false, features = ["native-tls"] } native-tls = "0.2" diff --git a/rust/build/in_process.rs b/rust/build/in_process.rs index f779722e26..34bf473e9d 100644 --- a/rust/build/in_process.rs +++ b/rust/build/in_process.rs @@ -201,7 +201,7 @@ fn emit_embedded( platform: Platform, include_runtime: bool, ) { - let runtime_archive = + let (runtime_archive, runtime_files) = build_embedded_runtime_archive(runtime_package, platform, include_runtime); std::fs::write(out.join("copilot_cli.archive"), cli_archive) .expect("failed to write copilot_cli.archive"); @@ -213,6 +213,8 @@ fn emit_embedded( pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); pub(super) static RUNTIME_ARCHIVE: &[u8] = include_bytes!("copilot_runtime.archive"); pub(super) const CLI_BINARY_SIZE: u64 = {cli_binary_size}; +pub(super) static RUNTIME_FILES: &[super::RuntimeFile] = &[ +{runtime_files}]; "# ); @@ -223,12 +225,13 @@ fn build_embedded_runtime_archive( package: &[u8], platform: Platform, include_runtime: bool, -) -> Vec { +) -> (Vec, String) { let encoder = flate2::GzBuilder::new() .mtime(0) .write(Vec::new(), flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let runtime = append_hostless_runtime_tree(&mut archive, package, platform); + let mut files = String::new(); + let runtime = append_hostless_runtime_tree(&mut archive, package, platform, &mut files); if include_runtime { append_archive_file( &mut archive, @@ -236,19 +239,22 @@ fn build_embedded_runtime_archive( &runtime, 0o644, ); + append_runtime_manifest(&mut files, platform.runtime_library_name(), &runtime, 0o644); } let encoder = archive .into_inner() .expect("failed to finish minimal embedded CLI archive"); - encoder + let archive = encoder .finish() - .expect("failed to compress minimal embedded CLI archive") + .expect("failed to compress minimal embedded CLI archive"); + (archive, files) } fn append_hostless_runtime_tree( archive: &mut tar::Builder, package: &[u8], platform: Platform, + files: &mut String, ) -> Vec { let decoder = flate2::read::GzDecoder::new(package); let mut source = tar::Archive::new(decoder); @@ -284,6 +290,14 @@ fn append_hostless_runtime_tree( &bytes, mode, ); + append_runtime_manifest( + files, + destination + .to_str() + .expect("npm package paths are valid UTF-8"), + &bytes, + mode, + ); } runtime.unwrap_or_else(|| { panic!( @@ -293,6 +307,19 @@ fn append_hostless_runtime_tree( }) } +fn append_runtime_manifest(files: &mut String, path: &str, bytes: &[u8], mode: u32) { + use std::fmt::Write as _; + + let sha256: [u8; 32] = sha2::Sha256::digest(bytes).into(); + writeln!( + files, + " super::RuntimeFile {{ path: std::borrow::Cow::Borrowed({path:?}), size: {}, mode: {}, sha256: {sha256:?} }},", + bytes.len(), + mode & 0o777, + ) + .expect("write runtime manifest"); +} + fn hostless_runtime_path(source: &str, platform: Platform) -> Option { let relative = source.strip_prefix("package/")?; let parts: Vec<&str> = relative.split('/').collect(); @@ -966,5 +993,10 @@ fn archive_zip_entry_size(zip_bytes: &[u8], binary_name: &str) -> Option { fn verify_hash(data: &[u8], expected: &str) -> bool { let mut hasher = sha2::Sha256::new(); hasher.update(data); - format!("{:x}", hasher.finalize()) == expected + let actual: String = hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + actual == expected } diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 61a11c14a0..34265cbc53 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -21,11 +21,11 @@ //! header); anything that looks truncated or quarantined is re-extracted, and //! the whole publish is retried before surfacing a clear, actionable error. //! -//! Runtime assets are compared and extracted with bounded buffers instead of -//! retaining whole native images in memory. Matching files need only read -//! access; changed files are staged beside their targets and published only -//! after archive validation, including the gzip trailer. Installation is -//! atomic per file, not a transaction across the entire runtime bundle. +//! Runtime assets are hashed with bounded buffers against a manifest generated +//! into the consumer binary at build time. A valid warm cache needs only read +//! access and never decompresses the archive. Changed files are extracted in +//! one streaming pass and published after manifest and gzip validation. +//! Installation is atomic per file, not across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure // std-only logic that doesn't touch the embedded archive, so they compile @@ -33,7 +33,7 @@ // the standard `cargo test --no-default-features` job has `has_bundled_cli` // off but still needs to exercise them. #[cfg(has_bundled_cli)] -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; #[cfg(any(has_bundled_cli, test))] use std::fs; #[cfg(has_bundled_cli)] @@ -45,6 +45,8 @@ use std::sync::OnceLock; #[cfg(any(has_bundled_cli, test))] use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(has_bundled_cli)] +use sha2::{Digest, Sha256}; #[cfg(has_bundled_cli)] use tracing::{info, warn}; @@ -171,7 +173,7 @@ pub(crate) fn runtime_path() -> Option { #[cfg(has_bundled_cli)] { let dir = default_install_dir(CLI_VERSION); - match install_runtime(&dir, build_time::RUNTIME_ARCHIVE) { + match install_runtime(&dir, build_time::RUNTIME_ARCHIVE, build_time::RUNTIME_FILES) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -199,7 +201,11 @@ pub(crate) fn install_runtime_at(extract_dir: &Path) -> Option { return None; } }; - match install_runtime(&install_dir, build_time::RUNTIME_ARCHIVE) { + match install_runtime( + &install_dir, + build_time::RUNTIME_ARCHIVE, + build_time::RUNTIME_FILES, + ) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -281,7 +287,20 @@ const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.dylib"; const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; #[cfg(has_bundled_cli)] -fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { +#[derive(Clone, Debug)] +struct RuntimeFile { + path: std::borrow::Cow<'static, str>, + size: u64, + mode: u32, + sha256: [u8; 32], +} + +#[cfg(has_bundled_cli)] +fn install_runtime( + install_dir: &Path, + archive: impl Read, + files: &[RuntimeFile], +) -> Result { fs::create_dir_all(install_dir) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; let root = fs::canonicalize(install_dir) @@ -289,51 +308,22 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result Result { - if !runtime_entry_matches(&mut entry, &mut file)? { - changed.insert(path); - } - } - None => { - check_runtime_asset_parent(&root, parent, true)?; - pending.push(stage_runtime_entry(&mut entry, &target)?); - } - } + let valid = runtime_file_is_valid(asset, &target)?; + needs_install |= !valid; + assets.insert(path, (asset, valid)); } - - // tar stops at its end marker, before GzDecoder necessarily verifies the - // gzip CRC and length trailer. Validate those before publishing any files. - std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; if !required.is_empty() { return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); } - // A failed comparison has consumed part of the trusted entry. Rewind the - // archive once for all such files rather than buffering their prefixes. - // Cold installs and valid warm installs need only the first pass. - if !changed.is_empty() { - let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); - for entry in tar - .entries() + if !needs_install { + return Ok(install_dir.join(RUNTIME_BINARY_NAME)); + } + + let mut pending = Vec::new(); + seen.clear(); + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar + .entries() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + { + let mut entry = + entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry + .path() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + let path = runtime_asset_path(&path)?; + if !seen.insert(path.as_os_str().to_ascii_lowercase()) { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!("duplicate embedded runtime asset path: {}", path.display()), + )); + } + if !selected_runtime_asset(&path) { + continue; + } + let (asset, valid) = assets.remove(&path).ok_or_else(|| { + EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive entry is absent from manifest: {}", + path.display() + ), + ) + })?; + let mode = entry + .header() + .mode() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - { - let mut entry = - entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - if !entry.header().entry_type().is_file() { - continue; - } - let path = entry - .path() + & 0o777; + if entry.size() != asset.size || mode != asset.mode { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive metadata differs from manifest: {}", + path.display() + ), + )); + } + if valid { + let digest = hash_runtime_file(&mut entry, asset.size) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - let path = runtime_asset_path(&path)?; - if changed.contains(&path) { - pending.push(stage_runtime_entry(&mut entry, &root.join(path))?); + if digest != asset.sha256 { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive content differs from manifest: {}", + asset.path + ), + )); } + } else { + let target = root.join(&path); + check_runtime_asset_parent(&root, target.parent().expect("checked asset path"), true)?; + pending.push(stage_runtime_entry(&mut entry, &target, asset)?); } - std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + } + // TAR's end marker can precede gzip's CRC and size trailer. + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !assets.is_empty() { + return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); } for staged in pending { publish(&staged.temporary, &staged.target)?; @@ -400,6 +428,23 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result bool { + if path == Path::new(CLI_BINARY_NAME) { + return false; + } + if matches!( + path.file_name().and_then(|name| name.to_str()), + Some("copilot_runtime.dll" | "libcopilot_runtime.dylib" | "libcopilot_runtime.so") + ) { + #[cfg(feature = "bundled-in-process")] + return path == Path::new(RUNTIME_LIBRARY_NAME); + #[cfg(not(feature = "bundled-in-process"))] + return false; + } + true +} + #[cfg(has_bundled_cli)] fn runtime_asset_path(path: &Path) -> Result { let invalid = || { @@ -512,10 +557,7 @@ impl Drop for StagedRuntimeFile { } #[cfg(has_bundled_cli)] -fn existing_runtime_file( - entry: &tar::Entry<'_, R>, - target: &Path, -) -> Result, EmbeddedCliError> { +fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result { let metadata = match fs::symlink_metadata(target) { Ok(metadata) if metadata.is_file() => Some(metadata), Ok(_) => { @@ -527,17 +569,12 @@ fn existing_runtime_file( Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), }; - let size = entry.size(); let matches = metadata .as_ref() - .is_some_and(|metadata| metadata.len() == size); + .is_some_and(|metadata| metadata.len() == asset.size); // Immutable caches may strip write bits without invalidating their contents. #[cfg(unix)] - let mode = entry - .header() - .mode() - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o555; + let mode = asset.mode & 0o555; #[cfg(unix)] let matches = { use std::os::unix::fs::PermissionsExt; @@ -548,51 +585,47 @@ fn existing_runtime_file( }; if matches { match fs::File::open(target) { - Ok(file) => return Ok(Some(file)), + Ok(mut file) => match hash_runtime_file(&mut file, asset.size) { + Ok(digest) => return Ok(digest == asset.sha256), + Err(e) => { + tracing::debug!(path = %target.display(), error = %e, + "existing runtime asset cannot be verified; repairing"); + } + }, Err(e) => { tracing::debug!(path = %target.display(), error = %e, "existing runtime asset cannot be read; repairing"); } } } - Ok(None) + Ok(false) } #[cfg(has_bundled_cli)] -fn runtime_entry_matches( - entry: &mut tar::Entry<'_, R>, - existing: &mut fs::File, -) -> Result { +fn hash_runtime_file(reader: &mut impl Read, size: u64) -> std::io::Result<[u8; 32]> { let mut buffer = [0u8; 64 * 1024]; - let mut installed = [0u8; 64 * 1024]; - let mut remaining = entry.size(); + let mut hash = Sha256::new(); + let mut remaining = size; while remaining > 0 { let length = remaining.min(buffer.len() as u64) as usize; - entry - .read_exact(&mut buffer[..length]) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + reader.read_exact(&mut buffer[..length])?; remaining -= length as u64; - if let Err(e) = existing.read_exact(&mut installed[..length]) { - tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); - return Ok(false); - } - if installed[..length] != buffer[..length] { - return Ok(false); - } + hash.update(&buffer[..length]); } - match existing.read(&mut installed[..1]) { - Ok(read) => Ok(read == 0), - Err(e) => { - tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); - Ok(false) - } + if reader.read(&mut buffer[..1])? != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "runtime file exceeds manifest size", + )); } + Ok(hash.finalize().into()) } #[cfg(has_bundled_cli)] fn stage_runtime_entry( entry: &mut tar::Entry<'_, R>, target: &Path, + asset: &RuntimeFile, ) -> Result { let parent = target.parent().expect("runtime asset has a checked parent"); let (temporary, mut file) = create_temp_file(parent)?; @@ -600,7 +633,7 @@ fn stage_runtime_entry( temporary, target: target.to_path_buf(), }; - let result = write_runtime_entry(entry, &mut file); + let result = write_runtime_entry(entry, &mut file, asset); // Close handles before cleanup or replacement, including on Windows. drop(file); result?; @@ -611,27 +644,35 @@ fn stage_runtime_entry( fn write_runtime_entry( entry: &mut tar::Entry<'_, R>, file: &mut fs::File, + asset: &RuntimeFile, ) -> Result<(), EmbeddedCliError> { - let size = entry.size(); - // Entry is already limited by tar, and take enforces that limit at the - // copy boundary too. A premature EOF must not publish a truncated file. - let written = std::io::copy(&mut (&mut *entry).take(size), &mut *file) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; - if written != size { + let mut buffer = [0u8; 64 * 1024]; + let mut remaining = asset.size; + let mut hash = Sha256::new(); + while remaining > 0 { + let length = remaining.min(buffer.len() as u64) as usize; + entry + .read_exact(&mut buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + remaining -= length as u64; + file.write_all(&buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + hash.update(&buffer[..length]); + } + let digest: [u8; 32] = hash.finalize().into(); + if digest != asset.sha256 { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!("runtime entry size mismatch: read {written} bytes, expected {size}"), + format!( + "runtime archive content differs from manifest: {}", + asset.path + ), )); } #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let mode = entry - .header() - .mode() - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o777; - file.set_permissions(fs::Permissions::from_mode(mode)) + file.set_permissions(fs::Permissions::from_mode(asset.mode)) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; } file.sync_all() @@ -1306,7 +1347,12 @@ mod tests { fs::write(dir.path().join(RUNTIME_NODE_NAME), b"stale runtime").expect("seed runtime"); fs::write(dir.path().join(RUNTIME_BINARY_NAME), b"stale wrapper").expect("seed wrapper"); - install_runtime(dir.path(), build_time::RUNTIME_ARCHIVE).expect("install runtime"); + super::install_runtime( + dir.path(), + build_time::RUNTIME_ARCHIVE, + build_time::RUNTIME_FILES, + ) + .expect("install runtime"); assert_eq!( fs::read(dir.path().join(RUNTIME_NODE_NAME)).expect("read runtime"), @@ -1340,7 +1386,19 @@ mod tests { } #[cfg(has_bundled_cli)] - fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> Vec { + #[derive(Clone)] + struct RuntimeFixture { + archive: Vec, + files: Vec, + } + + #[cfg(has_bundled_cli)] + fn install_runtime(dir: &Path, fixture: &RuntimeFixture) -> Result { + super::install_runtime(dir, fixture.archive.as_slice(), &fixture.files) + } + + #[cfg(has_bundled_cli)] + fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> RuntimeFixture { let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); let mut archive = tar::Builder::new(encoder); let mut entries = vec![ @@ -1350,6 +1408,7 @@ mod tests { #[cfg(feature = "bundled-in-process")] entries.push((RUNTIME_LIBRARY_NAME, b"library".as_slice(), 0o644)); entries.extend_from_slice(extra); + let mut files = Vec::new(); for (name, bytes, mode) in entries { let mut header = tar::Header::new_gnu(); // Raw names also let the installer see traversal fixtures which @@ -1359,8 +1418,198 @@ mod tests { header.set_mode(mode); header.set_cksum(); archive.append(&header, bytes).expect("append fixture"); + files.push(RuntimeFile { + path: name.to_owned().into(), + size: bytes.len() as u64, + mode, + sha256: Sha256::digest(bytes).into(), + }); + } + RuntimeFixture { + archive: archive.into_inner().unwrap().finish().unwrap(), + files, + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn generated_manifest_matches_every_embedded_runtime_file() { + let mut archive = + tar::Archive::new(flate2::read::GzDecoder::new(build_time::RUNTIME_ARCHIVE)); + let mut manifest: HashMap<_, _> = build_time::RUNTIME_FILES + .iter() + .map(|file| { + ( + runtime_asset_path(Path::new(file.path.as_ref())).unwrap(), + file, + ) + }) + .collect(); + assert_eq!(manifest.len(), build_time::RUNTIME_FILES.len()); + assert!(manifest.contains_key(Path::new(RUNTIME_NODE_NAME))); + assert!(manifest.contains_key(Path::new(RUNTIME_BINARY_NAME))); + #[cfg(feature = "bundled-in-process")] + assert!(manifest.contains_key(Path::new(RUNTIME_LIBRARY_NAME))); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + assert!(entry.header().entry_type().is_file()); + let path = runtime_asset_path(&entry.path().unwrap()).unwrap(); + let file = manifest.remove(&path).expect("manifest entry"); + assert_eq!(entry.size(), file.size); + assert_eq!(entry.header().mode().unwrap() & 0o777, file.mode); + assert_eq!( + hash_runtime_file(&mut entry, file.size).unwrap(), + file.sha256 + ); + } + assert!(manifest.is_empty()); + std::io::copy(&mut archive.into_inner(), &mut std::io::sink()).unwrap(); + } + + #[cfg(has_bundled_cli)] + struct UnreadableArchive; + + #[cfg(has_bundled_cli)] + impl Read for UnreadableArchive { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + panic!("valid warm installation must not read or decompress the archive") + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn warm_runtime_verifies_all_files_without_reading_archive() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[("nested/asset", &[0xAB; 65_537], 0o644)]); + install_runtime(dir.path(), &fixture).unwrap(); + let path = super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); + assert_eq!(path, dir.path().join(RUNTIME_BINARY_NAME)); + assert_no_runtime_temps(dir.path()); + + let mut changed_manifest = fixture.files.clone(); + changed_manifest[0].sha256[0] ^= 1; + // Same size and mode are insufficient: the hash must come from the + // trusted manifest, not any mutable cache-side metadata. + assert!( + super::install_runtime(dir.path(), b"invalid archive".as_slice(), &changed_manifest) + .is_err() + ); + } + + #[cfg(has_bundled_cli)] + #[test] + fn cold_and_repair_consume_one_forward_only_archive() { + struct Counted<'a> { + bytes: &'a [u8], + consumed: usize, + } + impl Read for Counted<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.bytes.read(buffer)?; + self.consumed += count; + Ok(count) + } + } + let dir = tempfile::tempdir().unwrap(); + let bytes = vec![0xAB; 65_537]; + let fixture = runtime_fixture(&[("nested/asset", &bytes, 0o644)]); + for corruption in [None, Some(b"changed".as_slice()), Some(b"run"), Some(b"")] { + if let Some(corruption) = corruption { + fs::write(dir.path().join(RUNTIME_NODE_NAME), corruption).unwrap(); + } + let mut reader = Counted { + bytes: &fixture.archive, + consumed: 0, + }; + super::install_runtime(dir.path(), &mut reader, &fixture.files).unwrap(); + assert_eq!(reader.consumed, fixture.archive.len()); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), bytes); + assert_no_runtime_temps(dir.path()); } - archive.into_inner().unwrap().finish().unwrap() + } + + #[cfg(has_bundled_cli)] + #[test] + fn manifest_mismatches_never_publish_staged_files() { + let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); + let wrong_content = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); + let wrong_size = runtime_fixture(&[("nested/asset", b"short", 0o644)]); + let wrong_mode = runtime_fixture(&[("nested/asset", b"trusted", 0o755)]); + let extra = runtime_fixture(&[ + ("nested/asset", b"trusted", 0o644), + ("extra", b"bad", 0o644), + ]); + let duplicate = runtime_fixture(&[ + ("nested/asset", b"trusted", 0o644), + ("./NESTED/ASSET", b"bad", 0o644), + ]); + let missing = runtime_fixture(&[]); + for other in [ + wrong_content, + wrong_size, + wrong_mode, + extra, + duplicate, + missing, + ] { + let dir = tempfile::tempdir().unwrap(); + assert!( + super::install_runtime(dir.path(), other.archive.as_slice(), &fixture.files) + .is_err() + ); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn repair_verifies_archive_entries_even_when_cached_files_are_valid() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); + install_runtime(dir.path(), &fixture).unwrap(); + fs::remove_file(dir.path().join(RUNTIME_NODE_NAME)).unwrap(); + let mismatched = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); + + assert!( + super::install_runtime(dir.path(), mismatched.archive.as_slice(), &fixture.files) + .is_err() + ); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_eq!( + fs::read(dir.path().join("nested/asset")).unwrap(), + b"trusted" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn warm_runtime_rejects_same_size_corruption_despite_unchanged_metadata() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[]); + install_runtime(dir.path(), &fixture).unwrap(); + let runtime = dir.path().join(RUNTIME_NODE_NAME); + let original = fs::metadata(&runtime).unwrap(); + fs::write(&runtime, b"corrupt").unwrap(); + fs::File::options() + .write(true) + .open(&runtime) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(original.modified().unwrap())) + .unwrap(); + assert!( + super::install_runtime(dir.path(), b"invalid archive".as_slice(), &fixture.files) + .is_err() + ); + assert_eq!(fs::read(&runtime).unwrap(), b"corrupt"); + install_runtime(dir.path(), &fixture).unwrap(); + assert_eq!(fs::read(&runtime).unwrap(), b"runtime"); } #[cfg(has_bundled_cli)] @@ -1457,13 +1706,15 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let valid = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); let mut invalid_crc = valid.clone(); - let crc = invalid_crc.len() - 8; - invalid_crc[crc] ^= 0xFF; + let crc = invalid_crc.archive.len() - 8; + invalid_crc.archive[crc] ^= 0xFF; let mut invalid_length = valid.clone(); - let length = invalid_length.len() - 4; - invalid_length[length] ^= 0xFF; - let truncated_trailer = valid[..valid.len() - 1].to_vec(); - let truncated_body = valid[..valid.len() / 2].to_vec(); + let length = invalid_length.archive.len() - 4; + invalid_length.archive[length] ^= 0xFF; + let mut truncated_trailer = valid.clone(); + truncated_trailer.archive.truncate(valid.archive.len() - 1); + let mut truncated_body = valid.clone(); + truncated_body.archive.truncate(valid.archive.len() / 2); for archive in [ invalid_crc, invalid_length, @@ -1492,7 +1743,14 @@ mod tests { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(header.as_bytes()).unwrap(); encoder.write_all(b"short").unwrap(); - let archive = encoder.finish().unwrap(); + let mut archive = runtime_fixture(&[]); + archive.archive = encoder.finish().unwrap(); + archive + .files + .iter_mut() + .find(|file| file.path == RUNTIME_NODE_NAME) + .unwrap() + .size = 128 * 1024; assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); @@ -1512,7 +1770,10 @@ mod tests { archive .append_data(&mut header, RUNTIME_BINARY_NAME, b"wrapper".as_slice()) .unwrap(); - let archive = archive.into_inner().unwrap().finish().unwrap(); + let archive = RuntimeFixture { + archive: archive.into_inner().unwrap().finish().unwrap(), + files: runtime_fixture(&[]).files, + }; assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); From 4c217387d71500e9cf9836ad3352a6c120a64d6e Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 22:11:52 -0700 Subject: [PATCH 08/18] fix(rust): reuse owner-only immutable runtime caches Verify readability through bounded hashing and executable access using effective process credentials rather than requiring every permission class to match the archive. Reuse locked rustix for a safe Unix access check and cover owner-only cache modes without re-extraction. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.lock | 1 + rust/Cargo.toml | 5 +++- rust/src/embeddedcli.rs | 65 +++++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5bbafdafef..df117a08c5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -477,6 +477,7 @@ dependencies = [ "regex", "reqwest", "rusqlite", + "rustix", "schemars", "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a6cdc1d320..19b2170d83 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2", "dep:rustix"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -69,6 +69,9 @@ futures-util = "0.3" reqwest = { version = "0.12", default-features = false, features = ["stream", "http2", "default-tls"] } tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "native-tls"] } +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", features = ["fs"], optional = true } + [target.'cfg(windows)'.dependencies] zip = { version = "2", default-features = false, features = ["deflate"], optional = true } windows-sys = { version = "0.61", default-features = false, features = [ diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 34265cbc53..cfd6717cfe 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -572,18 +572,19 @@ fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result match hash_runtime_file(&mut file, asset.size) { Ok(digest) => return Ok(digest == asset.sha256), @@ -1689,7 +1690,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &archive).unwrap(); - for mode in [0o644, 0o655, 0o745, 0o754] { + for mode in [0o644, 0o400] { fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); install_runtime(dir.path(), &archive).unwrap(); assert_eq!( @@ -1700,6 +1701,25 @@ mod tests { assert_no_runtime_temps(dir.path()); } + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_reuses_executable_modes_without_group_or_other_access() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &fixture).unwrap(); + for mode in [0o745, 0o754, 0o700, 0o500] { + fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + mode + ); + } + assert_no_runtime_temps(dir.path()); + } + #[cfg(has_bundled_cli)] #[test] fn runtime_archive_errors_do_not_publish_and_clean_up_staging() { @@ -1974,6 +1994,17 @@ mod tests { #[cfg(all(has_bundled_cli, unix))] #[test] fn warm_runtime_install_needs_no_writable_cache() { + assert_read_only_runtime_cache(0o555); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn warm_runtime_reuses_owner_only_immutable_cache() { + assert_read_only_runtime_cache(0o500); + } + + #[cfg(all(has_bundled_cli, unix))] + fn assert_read_only_runtime_cache(permission_mask: u32) { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); @@ -1989,12 +2020,16 @@ mod tests { let mut read_only_files = Vec::new(); for name in files { let path = dir.path().join(name); - let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o555; + let mode = fs::metadata(&path).unwrap().permissions().mode() & permission_mask; fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); read_only_files.push((path, mode)); } - fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); - fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + fs::set_permissions( + dir.path().join("nested"), + fs::Permissions::from_mode(permission_mask), + ) + .unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(permission_mask)).unwrap(); let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); let result = install_runtime(dir.path(), &archive); fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); From bbd67ca593733f163a1884a7352d2c552da62efe Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:25:47 -0700 Subject: [PATCH 09/18] fix(rust): stream bundled runtime installation Compare runtime assets in bounded chunks, stage changed entries without whole-image buffers, and validate the archive before atomic publication. Preserve read-only warm caches, repair unreadable or corrupt files, and reject duplicate normalized destinations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/src/embeddedcli.rs | 763 ++++++++++++++++++++++++++++++++++------ 1 file changed, 652 insertions(+), 111 deletions(-) diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 574ed42a6f..41827e9f25 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -13,19 +13,27 @@ //! A non-atomic write, a multi-process race, or antivirus quarantining the //! freshly-written executable can leave a truncated or corrupt image that, if //! handed back as "good", fails to launch (e.g. Windows `ERROR_BAD_EXE_FORMAT`). -//! Installation therefore: extracts to a unique temp file in the target dir, +//! Full CLI installation therefore: extracts to a unique temp file in the target dir, //! fsyncs and marks it executable, verifies the staged bytes against the //! trusted in-memory image, atomically renames it into place, re-verifies the //! published file, and records an integrity marker. Subsequent runs trust an //! existing install only after a cheap re-check (size marker + executable-image //! header); anything that looks truncated or quarantined is re-extracted, and //! the whole publish is retried before surfacing a clear, actionable error. +//! +//! Runtime assets are compared and extracted with bounded buffers instead of +//! retaining whole native images in memory. Matching files need only read +//! access; changed files are staged beside their targets and published only +//! after archive validation, including the gzip trailer. Installation is +//! atomic per file, not a transaction across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure // std-only logic that doesn't touch the embedded archive, so they compile // whenever the binary is bundled *or* we're building the test harness — // the standard `cargo test --no-default-features` job has `has_bundled_cli` // off but still needs to exercise them. +#[cfg(has_bundled_cli)] +use std::collections::HashSet; #[cfg(any(has_bundled_cli, test))] use std::fs; #[cfg(has_bundled_cli)] @@ -276,17 +284,17 @@ const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { fs::create_dir_all(install_dir) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; - install_hostless_assets(install_dir, archive)?; - install_runtime_pair(install_dir, archive)?; + let root = fs::canonicalize(install_dir) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let mut required = vec![RUNTIME_BINARY_NAME, RUNTIME_NODE_NAME]; #[cfg(feature = "bundled-in-process")] - install_runtime_library(install_dir, archive)?; - Ok(install_dir.join(RUNTIME_BINARY_NAME)) -} - -#[cfg(has_bundled_cli)] -fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { + required.push(RUNTIME_LIBRARY_NAME); + let mut pending = Vec::new(); + let mut seen = HashSet::new(); + let mut changed = HashSet::new(); let gz = flate2::read::GzDecoder::new(archive); let mut tar = tar::Archive::new(gz); + for entry in tar .entries() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? @@ -300,17 +308,6 @@ fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), Emb .path() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? .into_owned(); - let file_name = path.file_name().and_then(|name| name.to_str()); - if path == Path::new(CLI_BINARY_NAME) - || matches!( - file_name, - Some("copilot_runtime.dll") - | Some("libcopilot_runtime.dylib") - | Some("libcopilot_runtime.so") - ) - { - continue; - } if path.is_absolute() || path.components().any(|component| { matches!( @@ -326,90 +323,283 @@ fn install_hostless_assets(install_dir: &Path, archive: &[u8]) -> Result<(), Emb format!("unsafe embedded runtime asset path: {}", path.display()), )); } - let mut bytes = Vec::with_capacity(entry.size() as usize); - entry - .read_to_end(&mut bytes) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - let target = install_dir.join(&path); - if fs::read(&target) - .map(|installed| installed == bytes) - .unwrap_or(false) - { + let path: PathBuf = path + .components() + .filter(|component| *component != std::path::Component::CurDir) + .collect(); + let file_name = path.file_name().and_then(|name| name.to_str()); + let is_library = matches!( + file_name, + Some("copilot_runtime.dll") + | Some("libcopilot_runtime.dylib") + | Some("libcopilot_runtime.so") + ); + if path == Path::new(CLI_BINARY_NAME) { + continue; + } + if is_library { + #[cfg(feature = "bundled-in-process")] + if path != Path::new(RUNTIME_LIBRARY_NAME) { + continue; + } + #[cfg(not(feature = "bundled-in-process"))] continue; } + if !seen.insert(path.clone()) { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!("duplicate embedded runtime asset path: {}", path.display()), + )); + } + if let Some(index) = required.iter().position(|name| path == Path::new(name)) { + if entry.size() == 0 { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!("embedded runtime artifact is empty: {}", path.display()), + )); + } + required.remove(index); + } + let target = root.join(&path); let parent = target.parent().ok_or_else(|| { EmbeddedCliError::with_message( EmbeddedCliErrorKind::Archive, format!("embedded runtime asset has no parent: {}", path.display()), ) })?; - fs::create_dir_all(parent) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; - let tmp = write_temp_file(parent, &bytes)?; - #[cfg(unix)] + check_runtime_asset_parent(&root, parent, false)?; + match existing_runtime_file(&entry, &target)? { + Some(mut file) => { + if !runtime_entry_matches(&mut entry, &mut file)? { + changed.insert(path); + } + } + None => { + check_runtime_asset_parent(&root, parent, true)?; + pending.push(stage_runtime_entry(&mut entry, &target)?); + } + } + } + + // tar stops at its end marker, before GzDecoder necessarily verifies the + // gzip CRC and length trailer. Validate those before publishing any files. + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !required.is_empty() { + return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); + } + // A failed comparison has consumed part of the trusted entry. Rewind the + // archive once for all such files rather than buffering their prefixes. + // Cold installs and valid warm installs need only the first pass. + if !changed.is_empty() { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar + .entries() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? { - use std::os::unix::fs::PermissionsExt; - let mode = entry.header().mode().unwrap_or(0o644) & 0o777; - fs::set_permissions(&tmp, fs::Permissions::from_mode(mode)) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let mut entry = + entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path: PathBuf = entry + .path() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + .components() + .filter(|component| *component != std::path::Component::CurDir) + .collect(); + if changed.contains(&path) { + pending.push(stage_runtime_entry(&mut entry, &root.join(path))?); + } } - if let Err(error) = publish(&tmp, &target) { - let _ = fs::remove_file(&tmp); - return Err(error); + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + } + for staged in pending { + publish(&staged.temporary, &staged.target)?; + } + Ok(install_dir.join(RUNTIME_BINARY_NAME)) +} + +#[cfg(has_bundled_cli)] +fn check_runtime_asset_parent( + root: &Path, + parent: &Path, + create: bool, +) -> Result<(), EmbeddedCliError> { + let mut current = root.to_path_buf(); + for component in parent + .strip_prefix(root) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + .components() + { + current.push(component); + if create { + match fs::create_dir(¤t) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e)), + } + } + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(e) if !create && e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), + }; + if !metadata.is_dir() { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime asset parent is not a directory: {}", + current.display() + ), + )); } } Ok(()) } #[cfg(has_bundled_cli)] -fn install_runtime_pair(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { - install_adjacent_file(install_dir, archive, RUNTIME_NODE_NAME, "runtime.node")?; - install_adjacent_file( - install_dir, - archive, - RUNTIME_BINARY_NAME, - "copilot runtime wrapper", - ) +struct StagedRuntimeFile { + temporary: PathBuf, + target: PathBuf, } -#[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] -fn install_runtime_library(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { - install_adjacent_file( - install_dir, - archive, - RUNTIME_LIBRARY_NAME, - "in-process FFI runtime library", - ) +#[cfg(has_bundled_cli)] +impl Drop for StagedRuntimeFile { + fn drop(&mut self) { + if let Err(error) = fs::remove_file(&self.temporary) + && error.kind() != std::io::ErrorKind::NotFound + { + warn!(path = %self.temporary.display(), %error, "failed to remove staged runtime asset"); + } + } } #[cfg(has_bundled_cli)] -fn install_adjacent_file( - install_dir: &Path, - archive: &[u8], - file_name: &str, - label: &str, +fn existing_runtime_file( + entry: &tar::Entry<'_, R>, + target: &Path, +) -> Result, EmbeddedCliError> { + let metadata = match fs::symlink_metadata(target) { + Ok(metadata) if metadata.is_file() => Some(metadata), + Ok(_) => { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!("runtime asset is not a regular file: {}", target.display()), + )); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), + }; + let size = entry.size(); + let matches = metadata + .as_ref() + .is_some_and(|metadata| metadata.len() == size); + #[cfg(unix)] + let mode = entry + .header() + .mode() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + & 0o777; + #[cfg(unix)] + let matches = { + use std::os::unix::fs::PermissionsExt; + matches + && metadata + .as_ref() + .is_some_and(|metadata| metadata.permissions().mode() & 0o777 == mode) + }; + if matches { + match fs::File::open(target) { + Ok(file) => return Ok(Some(file)), + Err(e) => { + tracing::debug!(path = %target.display(), error = %e, + "existing runtime asset cannot be read; repairing"); + } + } + } + Ok(None) +} + +#[cfg(has_bundled_cli)] +fn runtime_entry_matches( + entry: &mut tar::Entry<'_, R>, + existing: &mut fs::File, +) -> Result { + let mut buffer = [0u8; 64 * 1024]; + let mut installed = [0u8; 64 * 1024]; + let mut remaining = entry.size(); + while remaining > 0 { + let length = remaining.min(buffer.len() as u64) as usize; + entry + .read_exact(&mut buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + remaining -= length as u64; + if let Err(e) = existing.read_exact(&mut installed[..length]) { + tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); + return Ok(false); + } + if installed[..length] != buffer[..length] { + return Ok(false); + } + } + match existing.read(&mut installed[..1]) { + Ok(read) => Ok(read == 0), + Err(e) => { + tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); + Ok(false) + } + } +} + +#[cfg(has_bundled_cli)] +fn stage_runtime_entry( + entry: &mut tar::Entry<'_, R>, + target: &Path, +) -> Result { + let parent = target.parent().expect("runtime asset has a checked parent"); + let (temporary, mut file) = create_temp_file(parent)?; + let staged = StagedRuntimeFile { + temporary, + target: target.to_path_buf(), + }; + let result = write_runtime_entry(entry, &mut file); + // Close handles before cleanup or replacement, including on Windows. + drop(file); + result?; + Ok(staged) +} + +#[cfg(has_bundled_cli)] +fn write_runtime_entry( + entry: &mut tar::Entry<'_, R>, + file: &mut fs::File, ) -> Result<(), EmbeddedCliError> { - let target = install_dir.join(file_name); - let bytes = extract_binary(archive, file_name)?; - if bytes.is_empty() { + let size = entry.size(); + // Entry is already limited by tar, and take enforces that limit at the + // copy boundary too. A premature EOF must not publish a truncated file. + let written = std::io::copy(&mut (&mut *entry).take(size), &mut *file) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + if written != size { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!("embedded {label} is empty"), + format!("runtime entry size mismatch: read {written} bytes, expected {size}"), )); } - if fs::read(&target) - .map(|installed| installed == bytes) - .unwrap_or(false) + #[cfg(unix)] { - return Ok(()); - } - let tmp = write_temp_file(install_dir, &bytes)?; - if let Err(e) = publish(&tmp, &target) { - let _ = fs::remove_file(&tmp); - return Err(e); + use std::os::unix::fs::PermissionsExt; + let mode = entry + .header() + .mode() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + & 0o777; + file.set_permissions(fs::Permissions::from_mode(mode)) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; } - tracing::debug!(path = %target.display(), %label, "embedded runtime artifact installed"); - Ok(()) + file.sync_all() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)) } #[cfg(has_bundled_cli)] @@ -556,27 +746,7 @@ fn publish_verified( /// bytes to disk and marking it executable on unix before returning its path. #[cfg(any(has_bundled_cli, test))] fn write_temp_file(dir: &Path, contents: &[u8]) -> Result { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let unique = format!( - ".copilot-cli.tmp.{}.{}.{}", - std::process::id(), - COUNTER.fetch_add(1, Ordering::Relaxed), - nanos - ); - let tmp = dir.join(unique); - - // `create_new` guarantees we never clobber a sibling's in-flight temp - // file (the pid + counter + nanos name already makes that practically - // impossible). - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + let (tmp, mut file) = create_temp_file(dir)?; if let Err(e) = file .write_all(contents) @@ -602,24 +772,35 @@ fn write_temp_file(dir: &Path, contents: &[u8]) -> Result Result<(PathBuf, fs::File), EmbeddedCliError> { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let tmp = dir.join(format!( + ".copilot-cli.tmp.{}.{}.{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed), + nanos + )); + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + Ok((tmp, file)) +} + /// Atomically move the staged temp file onto `final_path`. /// -/// `rename` replaces the target atomically on POSIX, but on Windows it fails -/// when the target already exists — so on that error we remove the stale file -/// and retry. The remove-then-rename is the only non-atomic window, and it's -/// guarded upstream: callers re-verify the published file and, on a lost race, -/// accept a peer's identical install instead of erroring. +/// Rust uses rename on POSIX and MoveFileExW with MOVEFILE_REPLACE_EXISTING on +/// Windows. Never unlink the destination on failure: readers must retain the +/// previous complete file if replacement is blocked. #[cfg(any(has_bundled_cli, test))] fn publish(tmp: &Path, final_path: &Path) -> Result<(), EmbeddedCliError> { - match fs::rename(tmp, final_path) { - Ok(()) => Ok(()), - Err(_) if final_path.exists() => { - let _ = fs::remove_file(final_path); - fs::rename(tmp, final_path) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)) - } - Err(e) => Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)), - } + fs::rename(tmp, final_path).map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Publish, e)) } /// Read the file at `path` and confirm it byte-for-byte matches the trusted @@ -1121,4 +1302,364 @@ mod tests { dir.path().join("2.0.0") ); } + + #[cfg(has_bundled_cli)] + fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> Vec { + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut entries = vec![ + (RUNTIME_BINARY_NAME, b"wrapper".as_slice(), 0o755), + (RUNTIME_NODE_NAME, b"runtime".as_slice(), 0o755), + ]; + #[cfg(feature = "bundled-in-process")] + entries.push((RUNTIME_LIBRARY_NAME, b"library".as_slice(), 0o644)); + entries.extend_from_slice(extra); + for (name, bytes, mode) in entries { + let mut header = tar::Header::new_gnu(); + // Raw names also let the installer see traversal fixtures which + // Builder::append_data would reject before reaching product code. + header.as_mut_bytes()[..name.len()].copy_from_slice(name.as_bytes()); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_cksum(); + archive.append(&header, bytes).expect("append fixture"); + } + archive.into_inner().unwrap().finish().unwrap() + } + + #[cfg(has_bundled_cli)] + fn assert_no_runtime_temps(dir: &Path) { + for entry in fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + assert!( + !entry + .file_name() + .to_string_lossy() + .starts_with(".copilot-cli.tmp.") + ); + if entry.file_type().unwrap().is_dir() { + assert_no_runtime_temps(&entry.path()); + } + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_cold_install_and_warm_reuse_preserve_bytes_and_modes() { + let dir = tempfile::tempdir().unwrap(); + let data = vec![0xAB; 3 * 64 * 1024 + 17]; + let archive = runtime_fixture(&[("nested/asset", &data, 0o640)]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + assert_eq!(wrapper, dir.path().join(RUNTIME_BINARY_NAME)); + let asset = dir.path().join("nested/asset"); + assert_eq!(fs::read(&asset).unwrap(), data); + let modified = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1234567890); + fs::File::options() + .write(true) + .open(&asset) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(modified)) + .unwrap(); + + install_runtime(dir.path(), &archive).unwrap(); + + assert_eq!(fs::metadata(&asset).unwrap().modified().unwrap(), modified); + assert_eq!(fs::read(&asset).unwrap(), data); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(&asset).unwrap().permissions().mode() & 0o777, + 0o640 + ); + assert_eq!( + fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_repairs_same_size_corruption_truncation_and_extra_bytes() { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let runtime = dir.path().join(RUNTIME_NODE_NAME); + install_runtime(dir.path(), &archive).unwrap(); + + for corrupt in [b"runtimX".as_slice(), b"run", b"", b"runtime plus garbage"] { + fs::write(&runtime, corrupt).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!(fs::read(&runtime).unwrap(), b"runtime"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_repairs_missing_execute_permission() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o644)).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!( + fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_archive_errors_do_not_publish_and_clean_up_staging() { + let dir = tempfile::tempdir().unwrap(); + let valid = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); + let mut invalid_crc = valid.clone(); + let crc = invalid_crc.len() - 8; + invalid_crc[crc] ^= 0xFF; + let mut invalid_length = valid.clone(); + let length = invalid_length.len() - 4; + invalid_length[length] ^= 0xFF; + let truncated_trailer = valid[..valid.len() - 1].to_vec(); + let truncated_body = valid[..valid.len() / 2].to_vec(); + for archive in [ + invalid_crc, + invalid_length, + truncated_trailer, + truncated_body, + ] { + let runtime = dir.path().join(RUNTIME_NODE_NAME); + fs::write(&runtime, b"previous complete runtime").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(runtime).unwrap(), b"previous complete runtime"); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_short_entry_is_rejected_without_publishing() { + let dir = tempfile::tempdir().unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_path(RUNTIME_NODE_NAME).unwrap(); + header.set_size(128 * 1024); + header.set_mode(0o755); + header.set_cksum(); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(header.as_bytes()).unwrap(); + encoder.write_all(b"short").unwrap(); + let archive = encoder.finish().unwrap(); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_missing_required_entry_is_rejected_before_publish() { + let dir = tempfile::tempdir().unwrap(); + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_size(7); + header.set_mode(0o755); + header.set_cksum(); + archive + .append_data(&mut header, RUNTIME_BINARY_NAME, b"wrapper".as_slice()) + .unwrap(); + let archive = archive.into_inner().unwrap().finish().unwrap(); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_rejects_traversal_and_cleans_preceding_entries() { + let dir = tempfile::tempdir().unwrap(); + let install_dir = dir.path().join("install"); + let archive = runtime_fixture(&[("../escaped", b"bad", 0o644)]); + assert!(install_runtime(&install_dir, &archive).is_err()); + assert!(!dir.path().join("escaped").exists()); + assert!(!install_dir.join(RUNTIME_BINARY_NAME).exists()); + assert_no_runtime_temps(&install_dir); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_rejects_symlink_parents_and_targets() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("asset"), b"outside").unwrap(); + symlink(outside.path(), dir.path().join("nested")).unwrap(); + let archive = runtime_fixture(&[("nested/asset", b"new", 0o644)]); + + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(outside.path().join("asset")).unwrap(), b"outside"); + assert_no_runtime_temps(dir.path()); + fs::remove_file(dir.path().join("nested")).unwrap(); + symlink( + outside.path().join("asset"), + dir.path().join(RUNTIME_NODE_NAME), + ) + .unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(outside.path().join("asset")).unwrap(), b"outside"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn concurrent_runtime_installers_publish_complete_files() { + let dir = tempfile::tempdir().unwrap(); + let data = vec![0xAB; 256 * 1024 + 1]; + let archive = runtime_fixture(&[("nested/asset", &data, 0o644)]); + let barrier = std::sync::Barrier::new(6); + std::thread::scope(|scope| { + for _ in 0..6 { + scope.spawn(|| { + barrier.wait(); + install_runtime(dir.path(), &archive).unwrap(); + }); + } + }); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), data); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_duplicate_destinations_fail_on_cold_and_warm_installs() { + for name in ["asset", "./asset"] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("asset", b"first", 0o644), (name, b"last", 0o644)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join("asset").exists()); + assert_no_runtime_temps(dir.path()); + + fs::write(dir.path().join("asset"), b"last").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join("asset")).unwrap(), b"last"); + assert_no_runtime_temps(dir.path()); + } + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(RUNTIME_NODE_NAME, b"", 0o755)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] + #[test] + fn runtime_library_aliases_are_rejected_before_repair() { + let alias = format!("./{RUNTIME_LIBRARY_NAME}"); + for duplicate in [b"another".as_slice(), b""] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(&alias, duplicate, 0o644)]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join(RUNTIME_LIBRARY_NAME).exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + let library = dir.path().join(RUNTIME_LIBRARY_NAME); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(&library).unwrap(), b"library"); + fs::write(&library, b"corrupt").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(&library).unwrap(), b"corrupt"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn warm_runtime_install_needs_no_writable_cache() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); + install_runtime(dir.path(), &archive).unwrap(); + fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); + let result = install_runtime(dir.path(), &archive); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o755)).unwrap(); + result.unwrap(); + if !write_denied { + eprintln!("read-only permission enforcement unavailable (e.g. privileged user)"); + fs::remove_file(dir.path().join("write-probe")).unwrap(); + } + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"asset"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_repairs_unreadable_but_replaceable_file() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[("asset", b"trusted", 0o000)]); + let asset = dir.path().join("asset"); + fs::write(&asset, b"corrupt").unwrap(); + fs::set_permissions(&asset, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::File::open(&asset).is_ok() { + eprintln!("unreadable permission enforcement unavailable (e.g. privileged user)"); + } + let result = install_runtime(dir.path(), &archive); + let mode = fs::metadata(&asset).unwrap().permissions().mode() & 0o777; + fs::set_permissions(&asset, fs::Permissions::from_mode(0o600)).unwrap(); + result.unwrap(); + assert_eq!(mode, 0o000); + assert_eq!(fs::read(asset).unwrap(), b"trusted"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_replacement_preserves_open_reader_contents() { + let dir = tempfile::tempdir().unwrap(); + let asset = dir.path().join("asset"); + let original = runtime_fixture(&[("asset", b"old complete file", 0o644)]); + install_runtime(dir.path(), &original).unwrap(); + let mut reader = fs::File::open(&asset).unwrap(); + let replacement = runtime_fixture(&[("asset", b"new complete file", 0o644)]); + install_runtime(dir.path(), &replacement).unwrap(); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"old complete file"); + assert_eq!(fs::read(&asset).unwrap(), b"new complete file"); + assert_no_runtime_temps(dir.path()); + } + + #[test] + fn failed_publish_does_not_remove_previous_file() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("installed"); + fs::write(&target, b"previous complete file").unwrap(); + assert!(publish(&dir.path().join("missing-temporary"), &target).is_err()); + assert_eq!(fs::read(target).unwrap(), b"previous complete file"); + } } From d6af3545686c7569f78922f63bd224666849e6f1 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:27:27 -0700 Subject: [PATCH 10/18] test(rust): add reproducible runtime installer memory benchmark Exercise the public installer in isolated cold, warm, corrupt and truncated-cache processes. Record matched release-build memory and latency measurements, output identities, separate allocator diagnostics, and native-platform limitations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.toml | 1 + rust/README.md | 11 +- rust/benchmarks/runtime_install.md | 172 +++++++++++++++++++++++++ rust/benchmarks/runtime_install.py | 195 +++++++++++++++++++++++++++++ rust/examples/runtime_install.rs | 24 ++++ 5 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 rust/benchmarks/runtime_install.md create mode 100644 rust/benchmarks/runtime_install.py create mode 100644 rust/examples/runtime_install.rs diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4495d3928c..220283c9b7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,6 +15,7 @@ include = [ "src/**/*", "build/**/*", "examples/**/*", + "benchmarks/**/*", "tests/**/*", "build.rs", "Cargo.toml", diff --git a/rust/README.md b/rust/README.md index 11d9637b22..3ffd952137 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1165,7 +1165,16 @@ if let Some(path) = install_bundled_runtime() { ``` This extracts `copilot-runtime` together with adjacent `runtime.node`, then -returns the wrapper path. +returns the wrapper path. Runtime installation streams archive entries and +compares existing files in bounded chunks, without buffering whole native +images. A valid warm cache requires only read access. Missing or corrupt files +are staged in uniquely created sibling files, with archive permissions +preserved, then atomically replaced after archive validation. Replacement is +atomic per file, not across the entire bundle. + +An installer-only memory and latency reproduction is available in +[`benchmarks/runtime_install.md`](benchmarks/runtime_install.md). It exercises +the real bundled runtime without starting a client or contacting a model. ### Download cache (build-time, embed mode) diff --git a/rust/benchmarks/runtime_install.md b/rust/benchmarks/runtime_install.md new file mode 100644 index 0000000000..aa9e82ded1 --- /dev/null +++ b/rust/benchmarks/runtime_install.md @@ -0,0 +1,172 @@ +# Bundled runtime installation memory + +`examples/runtime_install.rs` exercises the public `install_bundled_runtime()` +API in a standalone process. It also checks that a second call returns the +same path. It does not launch the runtime, read credentials, authenticate, or +make service requests. Build-time downloads use the SDK's normal verified +public release artifacts. + +## Reproduce on macOS + +Requires Python 3.11+, the pinned Rust toolchain, and macOS's `/usr/bin/time`. +From `rust/` on the fixed revision: + +```sh +work=$(mktemp -d) +git worktree add --detach "$work/baseline" a675b55531a9dfc647ee015e32d74279568550f3 +cp examples/runtime_install.rs "$work/baseline/rust/examples/runtime_install.rs" +( + cd "$work/baseline/rust" + CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install +) +CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install + +python3 benchmarks/runtime_install.py \ + "$work/baseline/rust/target/release/examples/runtime_install" --runs 5 \ + > "$work/baseline.json" +python3 benchmarks/runtime_install.py \ + target/release/examples/runtime_install --runs 5 > "$work/fixed.json" +``` + +Both builds must use the same runtime version, feature set, toolchain and +profile. These commands use default features (`bundled-cli`), release +optimization level 3 and debug level 1. Check the SHA-256 of each build's +`target/release/build/github-copilot-sdk-*/out/copilot_runtime.archive`; the +hashes must agree. Do not compare binaries with different runtime releases. + +Every measured run gets a fresh process and temporary `HOME`, including the +platform cache. The environment contains only `HOME`, `TMPDIR`, and a system +`PATH`. For warm/repair runs, a separate, unmeasured installer process seeds +that home before the measured process starts. Cohorts are: + +- **Cold:** no installed files. +- **Warm:** all installed files already match. +- **Corrupt:** flip the final byte of installed `runtime.node`, retaining its size. +- **Truncated:** truncate installed `runtime.node` to 1,024 bytes. + +Each process waits before installation and remains alive for one second +after installation so the harness can observe retained memory. The harness +then lets it exit. Output sizes, permissions and streaming SHA-256 hashes +are collected outside the measured process after exit; every run must produce +the same file inventory. The JSON contains only relative output paths. +Temporary homes are removed after each run. Compare the two JSON `outputs` +objects for exact equality, not just the runtime file: + +```sh +python3 - "$work/baseline.json" "$work/fixed.json" <<'PY' +import json, sys +before, after = [json.load(open(path)) for path in sys.argv[1:]] +assert before["outputs"] == after["outputs"] +print(len(after["outputs"]), "identical installed files") +PY +``` + +## Measurement definitions + +**Peak RSS** and **peak physical footprint** are the process-lifetime kernel +high-water marks reported by `/usr/bin/time -l`. **Retained RSS** and +**retained physical footprint** are `proc_pid_rusage(RUSAGE_INFO_V0)`'s +`ri_resident_size` and `ri_phys_footprint`, sampled after the one-second idle +window. They are total process values, not baseline-subtracted allocations. +RSS includes resident file-backed pages such as the embedded compressed +archive; physical footprint is the kernel's charged-memory accounting and +is not interchangeable with RSS or live heap size. + +Elapsed time comes from Rust's `Instant` around the first installation call, +excluding startup, the second cached call, idle waits, and output hashing. +Runs are independent processes on a shared host, not CPU-isolated trials. +"Cold" means an empty installation cache, not flushed filesystem pages. +Small samples and environmental noise limit latency conclusions. + +## Observed before and after + +Measured on 2026-09-15 with an Apple M4 Pro, 48 GiB RAM, macOS 26.6.2 +(25G83), `aarch64-apple-darwin`, rustc 1.94.0 +(`4a4ef493e`, LLVM 21.1.8). Baseline SDK revision: +`a675b55531a9dfc647ee015e32d74279568550f3` (`0.0.0-dev`). +Installer-only fixed revision: +`883edfbf89c04fb5165649abc9bda7efe8e719bd`. +The fixed build changes only the runtime installer; runtime version, +dependencies, release profile and probe source are identical. + +Five runs per cohort per build. Values are **median (minimum-maximum)**. +Memory uses MiB (1,048,576 bytes). + +| Cohort | Before retained physical MiB | After retained physical MiB | Before seconds | After seconds | +| --- | --- | --- | --- | --- | +| Cold | 155.438 (154.563-156.047) | 2.000 (1.969-2.078) | 0.880 (0.863-1.037) | 0.633 (0.618-0.703) | +| Warm | 174.360 (172.735-174.907) | 1.938 (1.875-1.953) | 1.031 (0.997-1.092) | 0.747 (0.359-0.788) | +| Corrupt | 174.376 (171.438-174.485) | 2.063 (2.000-2.079) | 1.043 (1.018-1.075) | 1.007 (0.991-1.092) | +| Truncated | 174.376 (172.938-174.422) | 1.907 (1.891-1.907) | 1.038 (0.978-1.090) | 0.830 (0.797-0.863) | + +| Cohort | Before peak physical MiB | After peak physical MiB | Before peak RSS MiB | After peak RSS MiB | +| --- | --- | --- | --- | --- | +| Cold | 155.469 (154.594-156.079) | 2.032 (2.000-2.110) | 200.109 (199.219-200.703) | 46.766 (46.734-46.844) | +| Warm | 174.391 (172.766-174.938) | 1.969 (1.907-1.985) | 219.016 (217.391-219.547) | 46.688 (46.641-46.719) | +| Corrupt | 174.407 (171.469-174.516) | 2.094 (2.032-2.110) | 219.016 (216.078-219.125) | 46.812 (46.734-46.812) | +| Truncated | 174.407 (172.969-174.454) | 1.938 (1.922-1.938) | 219.031 (217.578-219.062) | 46.672 (46.641-46.672) | + +| Cohort | Before retained RSS MiB | After retained RSS MiB | +| --- | --- | --- | +| Cold | 200.078 (199.188-200.672) | 46.719 (46.688-46.797) | +| Warm | 218.984 (217.359-219.516) | 46.641 (46.594-46.672) | +| Corrupt | 218.984 (216.047-219.094) | 46.766 (46.688-46.766) | +| Truncated | 219.000 (217.547-219.031) | 46.625 (46.594-46.625) | + +The baseline/fixed median initial physical footprints were approximately +1.58/1.58 MiB before installation. All 68 output files were byte-identical, +with the same sizes and modes, across both builds and all cohorts. + +| Input/output | Identity | +| --- | --- | +| Public runtime release | `github/copilot-cli` `v1.0.84-8`, `github-copilot-1.0.84-8-darwin-arm64.tgz` | +| Filtered embedded runtime archive SHA-256 | `6c43b789080fc06b25d406af8fae709daa99f0724c4d290cc8a31160c5a3ad64` | +| Installed `runtime.node` | 71,166,736 bytes; mode `0755`; SHA-256 `839cd681c72cb92f27697d5e3e3ee96d7bb8df4234ffb5f7b442956828ec173a` | +| Installed `copilot-runtime` | 386,992 bytes; mode `0755`; SHA-256 `b1bb3f4b9f6ee4c4d72eb206e68647fe0716a0d87e874472b411f4abc5608a8f` | +| Total installed output | 68 files; 95,721,242 bytes | +| Complete inventory SHA-256 | `dde1d211fd60d155cd5ec647ee0f5123371498a2e65cb9a6c52675b1b711ce17` | +| Baseline probe binary SHA-256 | `2ad5536afcc8052ad15d2af726cccd86b503f06068b690be7c6268e5a7557bb1` | +| Fixed probe binary SHA-256 | `4f37ab4cd9d5e6a5ece13900c85f6d6858732cfce1c80f3f2e47953edecabc2d` | + +The inventory digest hashes UTF-8 +`json.dumps(outputs, sort_keys=True, separators=(",", ":"))`. +Probe binary hashes identify the measured executables, not reproducible-build +expectations: build paths and debug information may differ on another host. +Output file sizes are identity checks, not memory measurements. + +## Separate allocation diagnostics + +Run profiling separately from the comparison above: + +```sh +python3 benchmarks/runtime_install.py target/release/examples/runtime_install \ + --runs 1 --cohort warm --diagnostics "$work/fixed-diagnostics" \ + > "$work/fixed-instrumented.json" +``` + +This enables `MallocStackLogging` and `MallocStackLoggingNoCompact` and saves +`vmmap -summary`, live allocations, and allocation history. These tools can +require local profiling permission. Raw diagnostics may contain local paths; +keep them local rather than attaching them to an issue or PR. + +In a separate baseline warm run, allocation history recorded two +71,172,096-byte VM allocations through `embeddedcli::install_runtime`, one +also through `std::fs::read`. The size is the page-rounded native runtime +payload. After installation, `vmmap` reported 165.7 MiB in +`MALLOC_LARGE (empty)` regions. These were freed allocations retained by the +allocator, not evidence of a live-object leak. The fixed warm diagnostic +did not contain either runtime-sized allocation or a `MALLOC_LARGE (empty)` +region. Instrumented memory/timing values are not included in the tables. + +The fix uses bounded entry copying and 64 KiB comparison buffers. Cold and +valid warm installs traverse the archive once. Same-size corrupt files can +require one additional traversal to recover bytes consumed during comparison. +Warm verification does not stage or write matching files, so valid read-only +caches remain usable. Changed files require temporary disk space until +archive validation completes; publication is atomic per file, not for the +whole bundle. Abrupt process termination can leave temporary files, as before. + +These measurements cover only bundled runtime installation on macOS arm64. +They do not measure full CLI installation, authentication, model sessions, +in-process runtime loading, or an application's overall performance. +Native Windows, Linux, and other architecture measurements were not available. diff --git a/rust/benchmarks/runtime_install.py b/rust/benchmarks/runtime_install.py new file mode 100644 index 0000000000..4170f573f8 --- /dev/null +++ b/rust/benchmarks/runtime_install.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""macOS installer-only memory/latency probe using fresh processes and homes.""" + +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path +import select +import signal +import subprocess +import sys +import tempfile +import time + + +class RusageInfoV0(ctypes.Structure): + _fields_ = [("uuid", ctypes.c_uint8 * 16)] + [ + (name, ctypes.c_uint64) + for name in ( + "user_time", "system_time", "pkg_idle_wkups", "interrupt_wkups", + "pageins", "wired_size", "resident_size", "phys_footprint", + "proc_start_abstime", "proc_exit_abstime", + ) + ] + + +def memory(pid): + info = RusageInfoV0() + if LIBPROC.proc_pid_rusage(pid, 0, ctypes.byref(info)) != 0: + raise OSError(ctypes.get_errno(), "proc_pid_rusage failed") + return info.resident_size, info.phys_footprint + + +def read_line(process): + if not select.select([process.stdout], [], [], 120)[0]: + raise TimeoutError("installer did not respond within 120 seconds") + line = process.stdout.readline().strip() + if not line: + raise RuntimeError("installer exited without a response") + return line.split() + + +def run(binary, home, diagnostic_dir=None): + env = {"HOME": str(home), "PATH": "/usr/bin:/bin", "TMPDIR": str(home)} + if diagnostic_dir: + env.update(MallocStackLogging="1", MallocStackLoggingNoCompact="1") + with tempfile.TemporaryFile(mode="w+") as stderr: + process = subprocess.Popen( + ["/usr/bin/time", "-l", str(binary)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, + text=True, env=env, bufsize=1, + ) + pid = None + try: + ready, pid = read_line(process) + assert ready == "ready" + pid = int(pid) + initial_rss, initial_physical = memory(pid) + process.stdin.write("\n") + process.stdin.flush() + while not select.select([process.stdout], [], [], 0.005)[0]: + # The OS lifetime high-water marks below capture short spikes + # that sampling could miss. This also checks process liveness. + memory(pid) + installed, elapsed = read_line(process) + assert installed == "installed" + time.sleep(1) + retained_rss, retained_physical = memory(pid) + if diagnostic_dir: + diagnostic_dir.mkdir(parents=True, exist_ok=True) + for name, command in ( + ("vmmap.txt", ["vmmap", "-summary", str(pid)]), + ("allocations.txt", [ + "malloc_history", str(pid), "-allBySize", "-fullStacks" + ]), + ("history.txt", [ + "malloc_history", str(pid), "-allEvents", "-noContent" + ]), + ): + with (diagnostic_dir / name).open("w") as output: + subprocess.run( + command, stdout=output, stderr=subprocess.STDOUT, + check=True, timeout=120, + ) + process.stdin.write("\n") + process.stdin.flush() + if process.wait(timeout=120): + raise RuntimeError("installer process failed") + stderr.seek(0) + metrics = {} + for line in stderr: + for label, key in ( + ("maximum resident set size", "peak_rss_bytes"), + ("peak memory footprint", "peak_physical_bytes"), + ): + if label in line: + metrics[key] = int(line.split()[0]) + if len(metrics) != 2: + raise RuntimeError("macOS time did not report both memory metrics") + return dict( + elapsed_seconds=float(elapsed), + initial_rss_bytes=initial_rss, + initial_physical_bytes=initial_physical, + retained_rss_bytes=retained_rss, + retained_physical_bytes=retained_physical, + **metrics, + ) + finally: + if process.poll() is None: + if pid is not None: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.kill() + process.wait() + process.stdin.close() + process.stdout.close() + + +def digest(path): + with path.open("rb") as file: + return hashlib.file_digest(file, "sha256").hexdigest() + + +def inventory(home): + root = home / "Library/Caches/github-copilot-sdk/cli" + return { + path.relative_to(root).as_posix(): { + "bytes": path.stat().st_size, + "sha256": digest(path), + "mode": oct(path.stat().st_mode & 0o777), + } + for path in sorted(root.rglob("*")) if path.is_file() + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--diagnostics", type=Path) + parser.add_argument("--cohort", choices=("cold", "warm", "corrupt", "truncated")) + args = parser.parse_args() + binary = args.binary.resolve(strict=True) + if args.runs < 1: + parser.error("--runs must be positive") + result = { + "binary_sha256": digest(binary), "instrumented": bool(args.diagnostics), + "runs": [], "outputs": None, + } + cohorts = [args.cohort] if args.cohort else ("cold", "warm", "corrupt", "truncated") + for cohort in cohorts: + for iteration in range(args.runs): + with tempfile.TemporaryDirectory(prefix="sdk-runtime-") as directory: + home = Path(directory) + expected = None + if cohort != "cold": + run(binary, home) + expected = inventory(home) + runtime = next(home.rglob("runtime.node")) + if cohort == "corrupt": + with runtime.open("r+b") as file: + file.seek(-1, os.SEEK_END) + byte = file.read(1) + file.seek(-1, os.SEEK_END) + file.write(bytes([byte[0] ^ 0xFF])) + elif cohort == "truncated": + with runtime.open("r+b") as file: + file.truncate(1024) + diagnostic_dir = ( + args.diagnostics / f"{cohort}-{iteration}" + if args.diagnostics else None + ) + measured = run(binary, home, diagnostic_dir) + outputs = inventory(home) + if expected is not None: + assert outputs == expected, "repair changed installed output" + if result["outputs"] is not None: + assert outputs == result["outputs"], "output identity changed" + result["outputs"] = outputs + result["runs"].append(dict(cohort=cohort, iteration=iteration, **measured)) + print(f"{cohort} {iteration}: {measured}", file=sys.stderr) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + if sys.platform != "darwin": + sys.exit("This measurement harness requires macOS.") + LIBPROC = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + LIBPROC.proc_pid_rusage.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + LIBPROC.proc_pid_rusage.restype = ctypes.c_int + main() diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs new file mode 100644 index 0000000000..4a18409465 --- /dev/null +++ b/rust/examples/runtime_install.rs @@ -0,0 +1,24 @@ +//! Installer-only probe. No CLI subprocess, authentication, or model requests. +//! Run with an isolated HOME; see benchmarks/runtime_install.py. + +use std::io::{self, Write}; +use std::time::Instant; + +use github_copilot_sdk::install_bundled_runtime; + +fn main() -> Result<(), Box> { + println!("ready {}", std::process::id()); + io::stdout().flush()?; + let mut line = String::new(); + io::stdin().read_line(&mut line)?; + + let start = Instant::now(); + let path = install_bundled_runtime().ok_or("bundled runtime installation failed")?; + let elapsed = start.elapsed(); + assert_eq!(install_bundled_runtime().as_ref(), Some(&path)); + println!("installed {}", elapsed.as_secs_f64()); + io::stdout().flush()?; + line.clear(); + io::stdin().read_line(&mut line)?; + Ok(()) +} From 235158a4c15576cb7bebf769a2ca90948bcf992e Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 12:57:30 -0700 Subject: [PATCH 11/18] docs(rust): remove checked-in installer benchmarks Keep the small installer example and runtime behavior documentation, while retaining measurement evidence outside the repository. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.toml | 1 - rust/README.md | 6 +- rust/benchmarks/runtime_install.md | 172 ------------------------- rust/benchmarks/runtime_install.py | 195 ----------------------------- rust/examples/runtime_install.rs | 2 +- 5 files changed, 4 insertions(+), 372 deletions(-) delete mode 100644 rust/benchmarks/runtime_install.md delete mode 100644 rust/benchmarks/runtime_install.py diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 220283c9b7..4495d3928c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,6 @@ include = [ "src/**/*", "build/**/*", "examples/**/*", - "benchmarks/**/*", "tests/**/*", "build.rs", "Cargo.toml", diff --git a/rust/README.md b/rust/README.md index 3ffd952137..91f7fa4462 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1172,9 +1172,9 @@ are staged in uniquely created sibling files, with archive permissions preserved, then atomically replaced after archive validation. Replacement is atomic per file, not across the entire bundle. -An installer-only memory and latency reproduction is available in -[`benchmarks/runtime_install.md`](benchmarks/runtime_install.md). It exercises -the real bundled runtime without starting a client or contacting a model. +The [`runtime_install` example](examples/runtime_install.rs) exercises the real +bundled runtime without starting a client or contacting a model. Run it with +an isolated `HOME` to keep its installation cache separate. ### Download cache (build-time, embed mode) diff --git a/rust/benchmarks/runtime_install.md b/rust/benchmarks/runtime_install.md deleted file mode 100644 index aa9e82ded1..0000000000 --- a/rust/benchmarks/runtime_install.md +++ /dev/null @@ -1,172 +0,0 @@ -# Bundled runtime installation memory - -`examples/runtime_install.rs` exercises the public `install_bundled_runtime()` -API in a standalone process. It also checks that a second call returns the -same path. It does not launch the runtime, read credentials, authenticate, or -make service requests. Build-time downloads use the SDK's normal verified -public release artifacts. - -## Reproduce on macOS - -Requires Python 3.11+, the pinned Rust toolchain, and macOS's `/usr/bin/time`. -From `rust/` on the fixed revision: - -```sh -work=$(mktemp -d) -git worktree add --detach "$work/baseline" a675b55531a9dfc647ee015e32d74279568550f3 -cp examples/runtime_install.rs "$work/baseline/rust/examples/runtime_install.rs" -( - cd "$work/baseline/rust" - CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install -) -CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --locked --release --example runtime_install - -python3 benchmarks/runtime_install.py \ - "$work/baseline/rust/target/release/examples/runtime_install" --runs 5 \ - > "$work/baseline.json" -python3 benchmarks/runtime_install.py \ - target/release/examples/runtime_install --runs 5 > "$work/fixed.json" -``` - -Both builds must use the same runtime version, feature set, toolchain and -profile. These commands use default features (`bundled-cli`), release -optimization level 3 and debug level 1. Check the SHA-256 of each build's -`target/release/build/github-copilot-sdk-*/out/copilot_runtime.archive`; the -hashes must agree. Do not compare binaries with different runtime releases. - -Every measured run gets a fresh process and temporary `HOME`, including the -platform cache. The environment contains only `HOME`, `TMPDIR`, and a system -`PATH`. For warm/repair runs, a separate, unmeasured installer process seeds -that home before the measured process starts. Cohorts are: - -- **Cold:** no installed files. -- **Warm:** all installed files already match. -- **Corrupt:** flip the final byte of installed `runtime.node`, retaining its size. -- **Truncated:** truncate installed `runtime.node` to 1,024 bytes. - -Each process waits before installation and remains alive for one second -after installation so the harness can observe retained memory. The harness -then lets it exit. Output sizes, permissions and streaming SHA-256 hashes -are collected outside the measured process after exit; every run must produce -the same file inventory. The JSON contains only relative output paths. -Temporary homes are removed after each run. Compare the two JSON `outputs` -objects for exact equality, not just the runtime file: - -```sh -python3 - "$work/baseline.json" "$work/fixed.json" <<'PY' -import json, sys -before, after = [json.load(open(path)) for path in sys.argv[1:]] -assert before["outputs"] == after["outputs"] -print(len(after["outputs"]), "identical installed files") -PY -``` - -## Measurement definitions - -**Peak RSS** and **peak physical footprint** are the process-lifetime kernel -high-water marks reported by `/usr/bin/time -l`. **Retained RSS** and -**retained physical footprint** are `proc_pid_rusage(RUSAGE_INFO_V0)`'s -`ri_resident_size` and `ri_phys_footprint`, sampled after the one-second idle -window. They are total process values, not baseline-subtracted allocations. -RSS includes resident file-backed pages such as the embedded compressed -archive; physical footprint is the kernel's charged-memory accounting and -is not interchangeable with RSS or live heap size. - -Elapsed time comes from Rust's `Instant` around the first installation call, -excluding startup, the second cached call, idle waits, and output hashing. -Runs are independent processes on a shared host, not CPU-isolated trials. -"Cold" means an empty installation cache, not flushed filesystem pages. -Small samples and environmental noise limit latency conclusions. - -## Observed before and after - -Measured on 2026-09-15 with an Apple M4 Pro, 48 GiB RAM, macOS 26.6.2 -(25G83), `aarch64-apple-darwin`, rustc 1.94.0 -(`4a4ef493e`, LLVM 21.1.8). Baseline SDK revision: -`a675b55531a9dfc647ee015e32d74279568550f3` (`0.0.0-dev`). -Installer-only fixed revision: -`883edfbf89c04fb5165649abc9bda7efe8e719bd`. -The fixed build changes only the runtime installer; runtime version, -dependencies, release profile and probe source are identical. - -Five runs per cohort per build. Values are **median (minimum-maximum)**. -Memory uses MiB (1,048,576 bytes). - -| Cohort | Before retained physical MiB | After retained physical MiB | Before seconds | After seconds | -| --- | --- | --- | --- | --- | -| Cold | 155.438 (154.563-156.047) | 2.000 (1.969-2.078) | 0.880 (0.863-1.037) | 0.633 (0.618-0.703) | -| Warm | 174.360 (172.735-174.907) | 1.938 (1.875-1.953) | 1.031 (0.997-1.092) | 0.747 (0.359-0.788) | -| Corrupt | 174.376 (171.438-174.485) | 2.063 (2.000-2.079) | 1.043 (1.018-1.075) | 1.007 (0.991-1.092) | -| Truncated | 174.376 (172.938-174.422) | 1.907 (1.891-1.907) | 1.038 (0.978-1.090) | 0.830 (0.797-0.863) | - -| Cohort | Before peak physical MiB | After peak physical MiB | Before peak RSS MiB | After peak RSS MiB | -| --- | --- | --- | --- | --- | -| Cold | 155.469 (154.594-156.079) | 2.032 (2.000-2.110) | 200.109 (199.219-200.703) | 46.766 (46.734-46.844) | -| Warm | 174.391 (172.766-174.938) | 1.969 (1.907-1.985) | 219.016 (217.391-219.547) | 46.688 (46.641-46.719) | -| Corrupt | 174.407 (171.469-174.516) | 2.094 (2.032-2.110) | 219.016 (216.078-219.125) | 46.812 (46.734-46.812) | -| Truncated | 174.407 (172.969-174.454) | 1.938 (1.922-1.938) | 219.031 (217.578-219.062) | 46.672 (46.641-46.672) | - -| Cohort | Before retained RSS MiB | After retained RSS MiB | -| --- | --- | --- | -| Cold | 200.078 (199.188-200.672) | 46.719 (46.688-46.797) | -| Warm | 218.984 (217.359-219.516) | 46.641 (46.594-46.672) | -| Corrupt | 218.984 (216.047-219.094) | 46.766 (46.688-46.766) | -| Truncated | 219.000 (217.547-219.031) | 46.625 (46.594-46.625) | - -The baseline/fixed median initial physical footprints were approximately -1.58/1.58 MiB before installation. All 68 output files were byte-identical, -with the same sizes and modes, across both builds and all cohorts. - -| Input/output | Identity | -| --- | --- | -| Public runtime release | `github/copilot-cli` `v1.0.84-8`, `github-copilot-1.0.84-8-darwin-arm64.tgz` | -| Filtered embedded runtime archive SHA-256 | `6c43b789080fc06b25d406af8fae709daa99f0724c4d290cc8a31160c5a3ad64` | -| Installed `runtime.node` | 71,166,736 bytes; mode `0755`; SHA-256 `839cd681c72cb92f27697d5e3e3ee96d7bb8df4234ffb5f7b442956828ec173a` | -| Installed `copilot-runtime` | 386,992 bytes; mode `0755`; SHA-256 `b1bb3f4b9f6ee4c4d72eb206e68647fe0716a0d87e874472b411f4abc5608a8f` | -| Total installed output | 68 files; 95,721,242 bytes | -| Complete inventory SHA-256 | `dde1d211fd60d155cd5ec647ee0f5123371498a2e65cb9a6c52675b1b711ce17` | -| Baseline probe binary SHA-256 | `2ad5536afcc8052ad15d2af726cccd86b503f06068b690be7c6268e5a7557bb1` | -| Fixed probe binary SHA-256 | `4f37ab4cd9d5e6a5ece13900c85f6d6858732cfce1c80f3f2e47953edecabc2d` | - -The inventory digest hashes UTF-8 -`json.dumps(outputs, sort_keys=True, separators=(",", ":"))`. -Probe binary hashes identify the measured executables, not reproducible-build -expectations: build paths and debug information may differ on another host. -Output file sizes are identity checks, not memory measurements. - -## Separate allocation diagnostics - -Run profiling separately from the comparison above: - -```sh -python3 benchmarks/runtime_install.py target/release/examples/runtime_install \ - --runs 1 --cohort warm --diagnostics "$work/fixed-diagnostics" \ - > "$work/fixed-instrumented.json" -``` - -This enables `MallocStackLogging` and `MallocStackLoggingNoCompact` and saves -`vmmap -summary`, live allocations, and allocation history. These tools can -require local profiling permission. Raw diagnostics may contain local paths; -keep them local rather than attaching them to an issue or PR. - -In a separate baseline warm run, allocation history recorded two -71,172,096-byte VM allocations through `embeddedcli::install_runtime`, one -also through `std::fs::read`. The size is the page-rounded native runtime -payload. After installation, `vmmap` reported 165.7 MiB in -`MALLOC_LARGE (empty)` regions. These were freed allocations retained by the -allocator, not evidence of a live-object leak. The fixed warm diagnostic -did not contain either runtime-sized allocation or a `MALLOC_LARGE (empty)` -region. Instrumented memory/timing values are not included in the tables. - -The fix uses bounded entry copying and 64 KiB comparison buffers. Cold and -valid warm installs traverse the archive once. Same-size corrupt files can -require one additional traversal to recover bytes consumed during comparison. -Warm verification does not stage or write matching files, so valid read-only -caches remain usable. Changed files require temporary disk space until -archive validation completes; publication is atomic per file, not for the -whole bundle. Abrupt process termination can leave temporary files, as before. - -These measurements cover only bundled runtime installation on macOS arm64. -They do not measure full CLI installation, authentication, model sessions, -in-process runtime loading, or an application's overall performance. -Native Windows, Linux, and other architecture measurements were not available. diff --git a/rust/benchmarks/runtime_install.py b/rust/benchmarks/runtime_install.py deleted file mode 100644 index 4170f573f8..0000000000 --- a/rust/benchmarks/runtime_install.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -"""macOS installer-only memory/latency probe using fresh processes and homes.""" - -import argparse -import ctypes -import hashlib -import json -import os -from pathlib import Path -import select -import signal -import subprocess -import sys -import tempfile -import time - - -class RusageInfoV0(ctypes.Structure): - _fields_ = [("uuid", ctypes.c_uint8 * 16)] + [ - (name, ctypes.c_uint64) - for name in ( - "user_time", "system_time", "pkg_idle_wkups", "interrupt_wkups", - "pageins", "wired_size", "resident_size", "phys_footprint", - "proc_start_abstime", "proc_exit_abstime", - ) - ] - - -def memory(pid): - info = RusageInfoV0() - if LIBPROC.proc_pid_rusage(pid, 0, ctypes.byref(info)) != 0: - raise OSError(ctypes.get_errno(), "proc_pid_rusage failed") - return info.resident_size, info.phys_footprint - - -def read_line(process): - if not select.select([process.stdout], [], [], 120)[0]: - raise TimeoutError("installer did not respond within 120 seconds") - line = process.stdout.readline().strip() - if not line: - raise RuntimeError("installer exited without a response") - return line.split() - - -def run(binary, home, diagnostic_dir=None): - env = {"HOME": str(home), "PATH": "/usr/bin:/bin", "TMPDIR": str(home)} - if diagnostic_dir: - env.update(MallocStackLogging="1", MallocStackLoggingNoCompact="1") - with tempfile.TemporaryFile(mode="w+") as stderr: - process = subprocess.Popen( - ["/usr/bin/time", "-l", str(binary)], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, - text=True, env=env, bufsize=1, - ) - pid = None - try: - ready, pid = read_line(process) - assert ready == "ready" - pid = int(pid) - initial_rss, initial_physical = memory(pid) - process.stdin.write("\n") - process.stdin.flush() - while not select.select([process.stdout], [], [], 0.005)[0]: - # The OS lifetime high-water marks below capture short spikes - # that sampling could miss. This also checks process liveness. - memory(pid) - installed, elapsed = read_line(process) - assert installed == "installed" - time.sleep(1) - retained_rss, retained_physical = memory(pid) - if diagnostic_dir: - diagnostic_dir.mkdir(parents=True, exist_ok=True) - for name, command in ( - ("vmmap.txt", ["vmmap", "-summary", str(pid)]), - ("allocations.txt", [ - "malloc_history", str(pid), "-allBySize", "-fullStacks" - ]), - ("history.txt", [ - "malloc_history", str(pid), "-allEvents", "-noContent" - ]), - ): - with (diagnostic_dir / name).open("w") as output: - subprocess.run( - command, stdout=output, stderr=subprocess.STDOUT, - check=True, timeout=120, - ) - process.stdin.write("\n") - process.stdin.flush() - if process.wait(timeout=120): - raise RuntimeError("installer process failed") - stderr.seek(0) - metrics = {} - for line in stderr: - for label, key in ( - ("maximum resident set size", "peak_rss_bytes"), - ("peak memory footprint", "peak_physical_bytes"), - ): - if label in line: - metrics[key] = int(line.split()[0]) - if len(metrics) != 2: - raise RuntimeError("macOS time did not report both memory metrics") - return dict( - elapsed_seconds=float(elapsed), - initial_rss_bytes=initial_rss, - initial_physical_bytes=initial_physical, - retained_rss_bytes=retained_rss, - retained_physical_bytes=retained_physical, - **metrics, - ) - finally: - if process.poll() is None: - if pid is not None: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.kill() - process.wait() - process.stdin.close() - process.stdout.close() - - -def digest(path): - with path.open("rb") as file: - return hashlib.file_digest(file, "sha256").hexdigest() - - -def inventory(home): - root = home / "Library/Caches/github-copilot-sdk/cli" - return { - path.relative_to(root).as_posix(): { - "bytes": path.stat().st_size, - "sha256": digest(path), - "mode": oct(path.stat().st_mode & 0o777), - } - for path in sorted(root.rglob("*")) if path.is_file() - } - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("binary", type=Path) - parser.add_argument("--runs", type=int, default=5) - parser.add_argument("--diagnostics", type=Path) - parser.add_argument("--cohort", choices=("cold", "warm", "corrupt", "truncated")) - args = parser.parse_args() - binary = args.binary.resolve(strict=True) - if args.runs < 1: - parser.error("--runs must be positive") - result = { - "binary_sha256": digest(binary), "instrumented": bool(args.diagnostics), - "runs": [], "outputs": None, - } - cohorts = [args.cohort] if args.cohort else ("cold", "warm", "corrupt", "truncated") - for cohort in cohorts: - for iteration in range(args.runs): - with tempfile.TemporaryDirectory(prefix="sdk-runtime-") as directory: - home = Path(directory) - expected = None - if cohort != "cold": - run(binary, home) - expected = inventory(home) - runtime = next(home.rglob("runtime.node")) - if cohort == "corrupt": - with runtime.open("r+b") as file: - file.seek(-1, os.SEEK_END) - byte = file.read(1) - file.seek(-1, os.SEEK_END) - file.write(bytes([byte[0] ^ 0xFF])) - elif cohort == "truncated": - with runtime.open("r+b") as file: - file.truncate(1024) - diagnostic_dir = ( - args.diagnostics / f"{cohort}-{iteration}" - if args.diagnostics else None - ) - measured = run(binary, home, diagnostic_dir) - outputs = inventory(home) - if expected is not None: - assert outputs == expected, "repair changed installed output" - if result["outputs"] is not None: - assert outputs == result["outputs"], "output identity changed" - result["outputs"] = outputs - result["runs"].append(dict(cohort=cohort, iteration=iteration, **measured)) - print(f"{cohort} {iteration}: {measured}", file=sys.stderr) - print(json.dumps(result, indent=2)) - - -if __name__ == "__main__": - if sys.platform != "darwin": - sys.exit("This measurement harness requires macOS.") - LIBPROC = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) - LIBPROC.proc_pid_rusage.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] - LIBPROC.proc_pid_rusage.restype = ctypes.c_int - main() diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs index 4a18409465..f0d8170259 100644 --- a/rust/examples/runtime_install.rs +++ b/rust/examples/runtime_install.rs @@ -1,5 +1,5 @@ //! Installer-only probe. No CLI subprocess, authentication, or model requests. -//! Run with an isolated HOME; see benchmarks/runtime_install.py. +//! Run with an isolated HOME to keep the installation cache separate. use std::io::{self, Write}; use std::time::Instant; From a7ffbd36919d5d3186cb5b7236cace7f90129ef9 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 13:02:05 -0700 Subject: [PATCH 12/18] fix(rust): reject portable runtime destination aliases Normalize separators and deduplicate case-insensitively before artifact selection. Reject non-portable archive names rather than approximating filesystem-specific Unicode, DOS short-name, or trailing-dot aliases. Cover cold, warm and repair installs through the real installer. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/README.md | 3 + rust/src/embeddedcli.rs | 171 +++++++++++++++++++++++++++++++++------- 2 files changed, 146 insertions(+), 28 deletions(-) diff --git a/rust/README.md b/rust/README.md index 91f7fa4462..93ecd43ee6 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1171,6 +1171,9 @@ images. A valid warm cache requires only read access. Missing or corrupt files are staged in uniquely created sibling files, with archive permissions preserved, then atomically replaced after archive validation. Replacement is atomic per file, not across the entire bundle. +Names inside bundled archives must be portable ASCII paths and cannot differ +only by case or path separators. This restriction does not apply to the +caller-selected installation directory. The [`runtime_install` example](examples/runtime_install.rs) exercises the real bundled runtime without starting a client or contacting a model. Run it with diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 41827e9f25..f0abedc3c8 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -306,27 +306,14 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result Result Result Result Result { + let invalid = || { + EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!( + "non-portable embedded runtime asset path: {}", + path.display() + ), + ) + }; + // Bundled release assets use ASCII names. Apply the same conservative + // naming rules on every filesystem, without probing a read-only cache. + // Reject Unicode, DOS device/short names and trailing-dot/space aliases + // rather than approximating platform-specific Unicode normalization. + let name = path + .to_str() + .filter(|name| name.is_ascii()) + .ok_or_else(invalid)?; + if name.starts_with(['/', '\\']) { + return Err(invalid()); + } + let mut normalized = PathBuf::new(); + for component in name.split(['/', '\\']) { + if component.is_empty() || component == "." { + continue; + } + if component == ".." + || component.ends_with(['.', ' ']) + || component + .bytes() + .any(|byte| byte.is_ascii_control() || b"<>:\"|?*~".contains(&byte)) + { + return Err(invalid()); + } + let stem = component + .split('.') + .next() + .expect("nonempty component") + .trim_end_matches(' ') + .to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && matches!(stem.as_bytes()[3], b'1'..=b'9')) + { + return Err(invalid()); + } + normalized.push(component); + } + if normalized.as_os_str().is_empty() { + return Err(invalid()); + } + Ok(normalized) +} + #[cfg(has_bundled_cli)] fn check_runtime_asset_parent( root: &Path, @@ -1570,6 +1605,86 @@ mod tests { assert_no_runtime_temps(dir.path()); } + #[cfg(has_bundled_cli)] + #[test] + fn runtime_case_aliases_cannot_overwrite_required_artifacts() { + let names = [ + RUNTIME_NODE_NAME, + RUNTIME_BINARY_NAME, + #[cfg(feature = "bundled-in-process")] + RUNTIME_LIBRARY_NAME, + ]; + for name in names { + let alias = name.to_ascii_uppercase(); + let archive = runtime_fixture(&[(&alias, b"", 0o755)]); + let dir = tempfile::tempdir().unwrap(); + let result = install_runtime(dir.path(), &archive); + assert!(result.is_err(), "accepted alias {alias}: {result:?}"); + assert!(!dir.path().join(name).exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime(dir.path(), &runtime_fixture(&[])).unwrap(); + let original = fs::read(dir.path().join(name)).unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join(name)).unwrap(), original); + fs::write(dir.path().join(name), b"corrupt").unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join(name)).unwrap(), b"corrupt"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_mixed_separator_and_case_aliases_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[ + ("nested/asset", b"first", 0o644), + (r".\NESTED\ASSET", b"last", 0o644), + ]); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + + install_runtime( + dir.path(), + &runtime_fixture(&[("nested/asset", b"last", 0o644)]), + ) + .unwrap(); + assert!(install_runtime(dir.path(), &archive).is_err()); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"last"); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn runtime_nonportable_alias_paths_are_rejected_before_publish() { + for name in [ + "runtime.node.", + "runtime.node ", + "runtime.node:stream", + "RUNTIM~1.NOD", + "NUL", + "con.txt", + "aux .txt", + "COM1", + "LPT9.txt", + "n\u{00e9}sted/asset", + r"..\escaped", + r"C:\escaped", + r"\\server\share\asset", + ] { + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[(name, b"bad", 0o644)]); + assert!( + install_runtime(dir.path(), &archive).is_err(), + "accepted {name}" + ); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_no_runtime_temps(dir.path()); + } + } + #[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] #[test] fn runtime_library_aliases_are_rejected_before_repair() { From 972571bcb89a48428cfd89cf16654d46fe86a6f3 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 13:19:36 -0700 Subject: [PATCH 13/18] docs(rust): keep installer PR focused on code and tests Remove the README additions and standalone measurement example. Preserve reproduction details and measurements in the pull request rather than shipping a benchmark surface. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/README.md | 14 +------------- rust/examples/runtime_install.rs | 24 ------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 rust/examples/runtime_install.rs diff --git a/rust/README.md b/rust/README.md index 93ecd43ee6..11d9637b22 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1165,19 +1165,7 @@ if let Some(path) = install_bundled_runtime() { ``` This extracts `copilot-runtime` together with adjacent `runtime.node`, then -returns the wrapper path. Runtime installation streams archive entries and -compares existing files in bounded chunks, without buffering whole native -images. A valid warm cache requires only read access. Missing or corrupt files -are staged in uniquely created sibling files, with archive permissions -preserved, then atomically replaced after archive validation. Replacement is -atomic per file, not across the entire bundle. -Names inside bundled archives must be portable ASCII paths and cannot differ -only by case or path separators. This restriction does not apply to the -caller-selected installation directory. - -The [`runtime_install` example](examples/runtime_install.rs) exercises the real -bundled runtime without starting a client or contacting a model. Run it with -an isolated `HOME` to keep its installation cache separate. +returns the wrapper path. ### Download cache (build-time, embed mode) diff --git a/rust/examples/runtime_install.rs b/rust/examples/runtime_install.rs deleted file mode 100644 index f0d8170259..0000000000 --- a/rust/examples/runtime_install.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Installer-only probe. No CLI subprocess, authentication, or model requests. -//! Run with an isolated HOME to keep the installation cache separate. - -use std::io::{self, Write}; -use std::time::Instant; - -use github_copilot_sdk::install_bundled_runtime; - -fn main() -> Result<(), Box> { - println!("ready {}", std::process::id()); - io::stdout().flush()?; - let mut line = String::new(); - io::stdin().read_line(&mut line)?; - - let start = Instant::now(); - let path = install_bundled_runtime().ok_or("bundled runtime installation failed")?; - let elapsed = start.elapsed(); - assert_eq!(install_bundled_runtime().as_ref(), Some(&path)); - println!("installed {}", elapsed.as_secs_f64()); - io::stdout().flush()?; - line.clear(); - io::stdin().read_line(&mut line)?; - Ok(()) -} From f12afd065bf05a6465a75f8ecd0dc3aee20dad5d Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 21:25:43 -0700 Subject: [PATCH 14/18] fix(rust): reuse immutable runtime caches Ignore write-bit differences when validating installed runtime files, while retaining read and execute permission checks. Cover read-only installed files and repair of each missing wrapper execute bit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/src/embeddedcli.rs | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index f0abedc3c8..61a11c14a0 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -531,19 +531,20 @@ fn existing_runtime_file( let matches = metadata .as_ref() .is_some_and(|metadata| metadata.len() == size); + // Immutable caches may strip write bits without invalidating their contents. #[cfg(unix)] let mode = entry .header() .mode() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o777; + & 0o555; #[cfg(unix)] let matches = { use std::os::unix::fs::PermissionsExt; matches && metadata .as_ref() - .is_some_and(|metadata| metadata.permissions().mode() & 0o777 == mode) + .is_some_and(|metadata| metadata.permissions().mode() & 0o555 == mode) }; if matches { match fs::File::open(target) { @@ -1439,12 +1440,14 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &archive).unwrap(); - fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o644)).unwrap(); - install_runtime(dir.path(), &archive).unwrap(); - assert_eq!( - fs::metadata(wrapper).unwrap().permissions().mode() & 0o777, - 0o755 - ); + for mode in [0o644, 0o655, 0o745, 0o754] { + fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } assert_no_runtime_temps(dir.path()); } @@ -1715,6 +1718,20 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); install_runtime(dir.path(), &archive).unwrap(); + let files = [ + RUNTIME_BINARY_NAME, + RUNTIME_NODE_NAME, + "nested/asset", + #[cfg(feature = "bundled-in-process")] + RUNTIME_LIBRARY_NAME, + ]; + let mut read_only_files = Vec::new(); + for name in files { + let path = dir.path().join(name); + let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o555; + fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); + read_only_files.push((path, mode)); + } fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); @@ -1726,6 +1743,12 @@ mod tests { eprintln!("read-only permission enforcement unavailable (e.g. privileged user)"); fs::remove_file(dir.path().join("write-probe")).unwrap(); } + for (path, mode) in read_only_files { + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + mode + ); + } assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), b"asset"); assert_no_runtime_temps(dir.path()); } From a021eca8aea9289c023b2218d8e279376a61d0b0 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 22:02:40 -0700 Subject: [PATCH 15/18] perf(rust): verify runtime caches with trusted build manifests Generate per-file SHA256, size and mode alongside the unchanged runtime archive. Verify valid warm caches without reading compressed bytes, and perform cold or repair extraction in one bounded forward traversal with manifest and gzip integrity checks before publication. Use sha2 0.11 runtime-detected acceleration with a software fallback, preserving the existing miniz_oxide decompression backend. Cover generated output identity, zero archive reads, immutable cache reuse and mixed-cache repair failures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.lock | 62 ++++- rust/Cargo.toml | 5 +- rust/build/in_process.rs | 44 +++- rust/src/embeddedcli.rs | 541 +++++++++++++++++++++++++++++---------- 4 files changed, 496 insertions(+), 156 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b91eebd06c..5bbafdafef 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -70,6 +70,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -129,6 +138,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -154,6 +172,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -177,8 +204,18 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -546,6 +583,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1453,19 +1499,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4495d3928c..a6cdc1d320 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -59,6 +59,7 @@ getrandom = "0.2" uuid = { version = "1", default-features = false, features = ["v4"] } flate2 = { version = "1", optional = true } tar = { version = "0.4", optional = true } +sha2 = { version = "0.11", default-features = false, optional = true } # LLM inference callback transport: idiomatic HTTP/WebSocket forwarding for the # `CopilotRequestHandler`, plus base64/byte/stream plumbing for the chunk protocol. base64 = "0.22" @@ -125,7 +126,7 @@ required-features = ["test-support"] dirs = "5" flate2 = "1" serde_json = "1" -sha2 = "0.10" +sha2 = { version = "0.11", default-features = false } tar = "0.4" ureq = { version = "2", default-features = false, features = ["native-tls"] } native-tls = "0.2" diff --git a/rust/build/in_process.rs b/rust/build/in_process.rs index 0a8bdb0fcf..f72f99cbdb 100644 --- a/rust/build/in_process.rs +++ b/rust/build/in_process.rs @@ -201,7 +201,7 @@ fn emit_embedded( platform: Platform, include_runtime: bool, ) { - let runtime_archive = + let (runtime_archive, runtime_files) = build_embedded_runtime_archive(runtime_package, platform, include_runtime); std::fs::write(out.join("copilot_cli.archive"), cli_archive) .expect("failed to write copilot_cli.archive"); @@ -213,6 +213,8 @@ fn emit_embedded( pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); pub(super) static RUNTIME_ARCHIVE: &[u8] = include_bytes!("copilot_runtime.archive"); pub(super) const CLI_BINARY_SIZE: u64 = {cli_binary_size}; +pub(super) static RUNTIME_FILES: &[super::RuntimeFile] = &[ +{runtime_files}]; "# ); @@ -223,12 +225,13 @@ fn build_embedded_runtime_archive( package: &[u8], platform: Platform, include_runtime: bool, -) -> Vec { +) -> (Vec, String) { let encoder = flate2::GzBuilder::new() .mtime(0) .write(Vec::new(), flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let runtime = append_hostless_runtime_tree(&mut archive, package, platform); + let mut files = String::new(); + let runtime = append_hostless_runtime_tree(&mut archive, package, platform, &mut files); if include_runtime { append_archive_file( &mut archive, @@ -236,19 +239,22 @@ fn build_embedded_runtime_archive( &runtime, 0o644, ); + append_runtime_manifest(&mut files, platform.runtime_library_name(), &runtime, 0o644); } let encoder = archive .into_inner() .expect("failed to finish minimal embedded CLI archive"); - encoder + let archive = encoder .finish() - .expect("failed to compress minimal embedded CLI archive") + .expect("failed to compress minimal embedded CLI archive"); + (archive, files) } fn append_hostless_runtime_tree( archive: &mut tar::Builder, package: &[u8], platform: Platform, + files: &mut String, ) -> Vec { let decoder = flate2::read::GzDecoder::new(package); let mut source = tar::Archive::new(decoder); @@ -284,6 +290,14 @@ fn append_hostless_runtime_tree( &bytes, mode, ); + append_runtime_manifest( + files, + destination + .to_str() + .expect("npm package paths are valid UTF-8"), + &bytes, + mode, + ); } runtime.unwrap_or_else(|| { panic!( @@ -293,6 +307,19 @@ fn append_hostless_runtime_tree( }) } +fn append_runtime_manifest(files: &mut String, path: &str, bytes: &[u8], mode: u32) { + use std::fmt::Write as _; + + let sha256: [u8; 32] = sha2::Sha256::digest(bytes).into(); + writeln!( + files, + " super::RuntimeFile {{ path: std::borrow::Cow::Borrowed({path:?}), size: {}, mode: {}, sha256: {sha256:?} }},", + bytes.len(), + mode & 0o777, + ) + .expect("write runtime manifest"); +} + fn hostless_runtime_path(source: &str, platform: Platform) -> Option { let relative = source.strip_prefix("package/")?; let parts: Vec<&str> = relative.split('/').collect(); @@ -966,5 +993,10 @@ fn archive_zip_entry_size(zip_bytes: &[u8], binary_name: &str) -> Option { fn verify_hash(data: &[u8], expected: &str) -> bool { let mut hasher = sha2::Sha256::new(); hasher.update(data); - format!("{:x}", hasher.finalize()) == expected + let actual: String = hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + actual == expected } diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 61a11c14a0..34265cbc53 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -21,11 +21,11 @@ //! header); anything that looks truncated or quarantined is re-extracted, and //! the whole publish is retried before surfacing a clear, actionable error. //! -//! Runtime assets are compared and extracted with bounded buffers instead of -//! retaining whole native images in memory. Matching files need only read -//! access; changed files are staged beside their targets and published only -//! after archive validation, including the gzip trailer. Installation is -//! atomic per file, not a transaction across the entire runtime bundle. +//! Runtime assets are hashed with bounded buffers against a manifest generated +//! into the consumer binary at build time. A valid warm cache needs only read +//! access and never decompresses the archive. Changed files are extracted in +//! one streaming pass and published after manifest and gzip validation. +//! Installation is atomic per file, not across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure // std-only logic that doesn't touch the embedded archive, so they compile @@ -33,7 +33,7 @@ // the standard `cargo test --no-default-features` job has `has_bundled_cli` // off but still needs to exercise them. #[cfg(has_bundled_cli)] -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; #[cfg(any(has_bundled_cli, test))] use std::fs; #[cfg(has_bundled_cli)] @@ -45,6 +45,8 @@ use std::sync::OnceLock; #[cfg(any(has_bundled_cli, test))] use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(has_bundled_cli)] +use sha2::{Digest, Sha256}; #[cfg(has_bundled_cli)] use tracing::{info, warn}; @@ -171,7 +173,7 @@ pub(crate) fn runtime_path() -> Option { #[cfg(has_bundled_cli)] { let dir = default_install_dir(CLI_VERSION); - match install_runtime(&dir, build_time::RUNTIME_ARCHIVE) { + match install_runtime(&dir, build_time::RUNTIME_ARCHIVE, build_time::RUNTIME_FILES) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -199,7 +201,11 @@ pub(crate) fn install_runtime_at(extract_dir: &Path) -> Option { return None; } }; - match install_runtime(&install_dir, build_time::RUNTIME_ARCHIVE) { + match install_runtime( + &install_dir, + build_time::RUNTIME_ARCHIVE, + build_time::RUNTIME_FILES, + ) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -281,7 +287,20 @@ const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.dylib"; const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; #[cfg(has_bundled_cli)] -fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { +#[derive(Clone, Debug)] +struct RuntimeFile { + path: std::borrow::Cow<'static, str>, + size: u64, + mode: u32, + sha256: [u8; 32], +} + +#[cfg(has_bundled_cli)] +fn install_runtime( + install_dir: &Path, + archive: impl Read, + files: &[RuntimeFile], +) -> Result { fs::create_dir_all(install_dir) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; let root = fs::canonicalize(install_dir) @@ -289,51 +308,22 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result Result { - if !runtime_entry_matches(&mut entry, &mut file)? { - changed.insert(path); - } - } - None => { - check_runtime_asset_parent(&root, parent, true)?; - pending.push(stage_runtime_entry(&mut entry, &target)?); - } - } + let valid = runtime_file_is_valid(asset, &target)?; + needs_install |= !valid; + assets.insert(path, (asset, valid)); } - - // tar stops at its end marker, before GzDecoder necessarily verifies the - // gzip CRC and length trailer. Validate those before publishing any files. - std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; if !required.is_empty() { return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); } - // A failed comparison has consumed part of the trusted entry. Rewind the - // archive once for all such files rather than buffering their prefixes. - // Cold installs and valid warm installs need only the first pass. - if !changed.is_empty() { - let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); - for entry in tar - .entries() + if !needs_install { + return Ok(install_dir.join(RUNTIME_BINARY_NAME)); + } + + let mut pending = Vec::new(); + seen.clear(); + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar + .entries() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + { + let mut entry = + entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry + .path() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + let path = runtime_asset_path(&path)?; + if !seen.insert(path.as_os_str().to_ascii_lowercase()) { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!("duplicate embedded runtime asset path: {}", path.display()), + )); + } + if !selected_runtime_asset(&path) { + continue; + } + let (asset, valid) = assets.remove(&path).ok_or_else(|| { + EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive entry is absent from manifest: {}", + path.display() + ), + ) + })?; + let mode = entry + .header() + .mode() .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - { - let mut entry = - entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - if !entry.header().entry_type().is_file() { - continue; - } - let path = entry - .path() + & 0o777; + if entry.size() != asset.size || mode != asset.mode { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive metadata differs from manifest: {}", + path.display() + ), + )); + } + if valid { + let digest = hash_runtime_file(&mut entry, asset.size) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - let path = runtime_asset_path(&path)?; - if changed.contains(&path) { - pending.push(stage_runtime_entry(&mut entry, &root.join(path))?); + if digest != asset.sha256 { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + format!( + "runtime archive content differs from manifest: {}", + asset.path + ), + )); } + } else { + let target = root.join(&path); + check_runtime_asset_parent(&root, target.parent().expect("checked asset path"), true)?; + pending.push(stage_runtime_entry(&mut entry, &target, asset)?); } - std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + } + // TAR's end marker can precede gzip's CRC and size trailer. + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !assets.is_empty() { + return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); } for staged in pending { publish(&staged.temporary, &staged.target)?; @@ -400,6 +428,23 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result bool { + if path == Path::new(CLI_BINARY_NAME) { + return false; + } + if matches!( + path.file_name().and_then(|name| name.to_str()), + Some("copilot_runtime.dll" | "libcopilot_runtime.dylib" | "libcopilot_runtime.so") + ) { + #[cfg(feature = "bundled-in-process")] + return path == Path::new(RUNTIME_LIBRARY_NAME); + #[cfg(not(feature = "bundled-in-process"))] + return false; + } + true +} + #[cfg(has_bundled_cli)] fn runtime_asset_path(path: &Path) -> Result { let invalid = || { @@ -512,10 +557,7 @@ impl Drop for StagedRuntimeFile { } #[cfg(has_bundled_cli)] -fn existing_runtime_file( - entry: &tar::Entry<'_, R>, - target: &Path, -) -> Result, EmbeddedCliError> { +fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result { let metadata = match fs::symlink_metadata(target) { Ok(metadata) if metadata.is_file() => Some(metadata), Ok(_) => { @@ -527,17 +569,12 @@ fn existing_runtime_file( Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, Err(e) => return Err(EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e)), }; - let size = entry.size(); let matches = metadata .as_ref() - .is_some_and(|metadata| metadata.len() == size); + .is_some_and(|metadata| metadata.len() == asset.size); // Immutable caches may strip write bits without invalidating their contents. #[cfg(unix)] - let mode = entry - .header() - .mode() - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o555; + let mode = asset.mode & 0o555; #[cfg(unix)] let matches = { use std::os::unix::fs::PermissionsExt; @@ -548,51 +585,47 @@ fn existing_runtime_file( }; if matches { match fs::File::open(target) { - Ok(file) => return Ok(Some(file)), + Ok(mut file) => match hash_runtime_file(&mut file, asset.size) { + Ok(digest) => return Ok(digest == asset.sha256), + Err(e) => { + tracing::debug!(path = %target.display(), error = %e, + "existing runtime asset cannot be verified; repairing"); + } + }, Err(e) => { tracing::debug!(path = %target.display(), error = %e, "existing runtime asset cannot be read; repairing"); } } } - Ok(None) + Ok(false) } #[cfg(has_bundled_cli)] -fn runtime_entry_matches( - entry: &mut tar::Entry<'_, R>, - existing: &mut fs::File, -) -> Result { +fn hash_runtime_file(reader: &mut impl Read, size: u64) -> std::io::Result<[u8; 32]> { let mut buffer = [0u8; 64 * 1024]; - let mut installed = [0u8; 64 * 1024]; - let mut remaining = entry.size(); + let mut hash = Sha256::new(); + let mut remaining = size; while remaining > 0 { let length = remaining.min(buffer.len() as u64) as usize; - entry - .read_exact(&mut buffer[..length]) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + reader.read_exact(&mut buffer[..length])?; remaining -= length as u64; - if let Err(e) = existing.read_exact(&mut installed[..length]) { - tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); - return Ok(false); - } - if installed[..length] != buffer[..length] { - return Ok(false); - } + hash.update(&buffer[..length]); } - match existing.read(&mut installed[..1]) { - Ok(read) => Ok(read == 0), - Err(e) => { - tracing::debug!(error = %e, "existing runtime asset cannot be read; repairing"); - Ok(false) - } + if reader.read(&mut buffer[..1])? != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "runtime file exceeds manifest size", + )); } + Ok(hash.finalize().into()) } #[cfg(has_bundled_cli)] fn stage_runtime_entry( entry: &mut tar::Entry<'_, R>, target: &Path, + asset: &RuntimeFile, ) -> Result { let parent = target.parent().expect("runtime asset has a checked parent"); let (temporary, mut file) = create_temp_file(parent)?; @@ -600,7 +633,7 @@ fn stage_runtime_entry( temporary, target: target.to_path_buf(), }; - let result = write_runtime_entry(entry, &mut file); + let result = write_runtime_entry(entry, &mut file, asset); // Close handles before cleanup or replacement, including on Windows. drop(file); result?; @@ -611,27 +644,35 @@ fn stage_runtime_entry( fn write_runtime_entry( entry: &mut tar::Entry<'_, R>, file: &mut fs::File, + asset: &RuntimeFile, ) -> Result<(), EmbeddedCliError> { - let size = entry.size(); - // Entry is already limited by tar, and take enforces that limit at the - // copy boundary too. A premature EOF must not publish a truncated file. - let written = std::io::copy(&mut (&mut *entry).take(size), &mut *file) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; - if written != size { + let mut buffer = [0u8; 64 * 1024]; + let mut remaining = asset.size; + let mut hash = Sha256::new(); + while remaining > 0 { + let length = remaining.min(buffer.len() as u64) as usize; + entry + .read_exact(&mut buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + remaining -= length as u64; + file.write_all(&buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + hash.update(&buffer[..length]); + } + let digest: [u8; 32] = hash.finalize().into(); + if digest != asset.sha256 { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!("runtime entry size mismatch: read {written} bytes, expected {size}"), + format!( + "runtime archive content differs from manifest: {}", + asset.path + ), )); } #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let mode = entry - .header() - .mode() - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o777; - file.set_permissions(fs::Permissions::from_mode(mode)) + file.set_permissions(fs::Permissions::from_mode(asset.mode)) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; } file.sync_all() @@ -1306,7 +1347,12 @@ mod tests { fs::write(dir.path().join(RUNTIME_NODE_NAME), b"stale runtime").expect("seed runtime"); fs::write(dir.path().join(RUNTIME_BINARY_NAME), b"stale wrapper").expect("seed wrapper"); - install_runtime(dir.path(), build_time::RUNTIME_ARCHIVE).expect("install runtime"); + super::install_runtime( + dir.path(), + build_time::RUNTIME_ARCHIVE, + build_time::RUNTIME_FILES, + ) + .expect("install runtime"); assert_eq!( fs::read(dir.path().join(RUNTIME_NODE_NAME)).expect("read runtime"), @@ -1340,7 +1386,19 @@ mod tests { } #[cfg(has_bundled_cli)] - fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> Vec { + #[derive(Clone)] + struct RuntimeFixture { + archive: Vec, + files: Vec, + } + + #[cfg(has_bundled_cli)] + fn install_runtime(dir: &Path, fixture: &RuntimeFixture) -> Result { + super::install_runtime(dir, fixture.archive.as_slice(), &fixture.files) + } + + #[cfg(has_bundled_cli)] + fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> RuntimeFixture { let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); let mut archive = tar::Builder::new(encoder); let mut entries = vec![ @@ -1350,6 +1408,7 @@ mod tests { #[cfg(feature = "bundled-in-process")] entries.push((RUNTIME_LIBRARY_NAME, b"library".as_slice(), 0o644)); entries.extend_from_slice(extra); + let mut files = Vec::new(); for (name, bytes, mode) in entries { let mut header = tar::Header::new_gnu(); // Raw names also let the installer see traversal fixtures which @@ -1359,8 +1418,198 @@ mod tests { header.set_mode(mode); header.set_cksum(); archive.append(&header, bytes).expect("append fixture"); + files.push(RuntimeFile { + path: name.to_owned().into(), + size: bytes.len() as u64, + mode, + sha256: Sha256::digest(bytes).into(), + }); + } + RuntimeFixture { + archive: archive.into_inner().unwrap().finish().unwrap(), + files, + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn generated_manifest_matches_every_embedded_runtime_file() { + let mut archive = + tar::Archive::new(flate2::read::GzDecoder::new(build_time::RUNTIME_ARCHIVE)); + let mut manifest: HashMap<_, _> = build_time::RUNTIME_FILES + .iter() + .map(|file| { + ( + runtime_asset_path(Path::new(file.path.as_ref())).unwrap(), + file, + ) + }) + .collect(); + assert_eq!(manifest.len(), build_time::RUNTIME_FILES.len()); + assert!(manifest.contains_key(Path::new(RUNTIME_NODE_NAME))); + assert!(manifest.contains_key(Path::new(RUNTIME_BINARY_NAME))); + #[cfg(feature = "bundled-in-process")] + assert!(manifest.contains_key(Path::new(RUNTIME_LIBRARY_NAME))); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + assert!(entry.header().entry_type().is_file()); + let path = runtime_asset_path(&entry.path().unwrap()).unwrap(); + let file = manifest.remove(&path).expect("manifest entry"); + assert_eq!(entry.size(), file.size); + assert_eq!(entry.header().mode().unwrap() & 0o777, file.mode); + assert_eq!( + hash_runtime_file(&mut entry, file.size).unwrap(), + file.sha256 + ); + } + assert!(manifest.is_empty()); + std::io::copy(&mut archive.into_inner(), &mut std::io::sink()).unwrap(); + } + + #[cfg(has_bundled_cli)] + struct UnreadableArchive; + + #[cfg(has_bundled_cli)] + impl Read for UnreadableArchive { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + panic!("valid warm installation must not read or decompress the archive") + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn warm_runtime_verifies_all_files_without_reading_archive() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[("nested/asset", &[0xAB; 65_537], 0o644)]); + install_runtime(dir.path(), &fixture).unwrap(); + let path = super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); + assert_eq!(path, dir.path().join(RUNTIME_BINARY_NAME)); + assert_no_runtime_temps(dir.path()); + + let mut changed_manifest = fixture.files.clone(); + changed_manifest[0].sha256[0] ^= 1; + // Same size and mode are insufficient: the hash must come from the + // trusted manifest, not any mutable cache-side metadata. + assert!( + super::install_runtime(dir.path(), b"invalid archive".as_slice(), &changed_manifest) + .is_err() + ); + } + + #[cfg(has_bundled_cli)] + #[test] + fn cold_and_repair_consume_one_forward_only_archive() { + struct Counted<'a> { + bytes: &'a [u8], + consumed: usize, + } + impl Read for Counted<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.bytes.read(buffer)?; + self.consumed += count; + Ok(count) + } + } + let dir = tempfile::tempdir().unwrap(); + let bytes = vec![0xAB; 65_537]; + let fixture = runtime_fixture(&[("nested/asset", &bytes, 0o644)]); + for corruption in [None, Some(b"changed".as_slice()), Some(b"run"), Some(b"")] { + if let Some(corruption) = corruption { + fs::write(dir.path().join(RUNTIME_NODE_NAME), corruption).unwrap(); + } + let mut reader = Counted { + bytes: &fixture.archive, + consumed: 0, + }; + super::install_runtime(dir.path(), &mut reader, &fixture.files).unwrap(); + assert_eq!(reader.consumed, fixture.archive.len()); + assert_eq!( + fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), + b"runtime" + ); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), bytes); + assert_no_runtime_temps(dir.path()); } - archive.into_inner().unwrap().finish().unwrap() + } + + #[cfg(has_bundled_cli)] + #[test] + fn manifest_mismatches_never_publish_staged_files() { + let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); + let wrong_content = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); + let wrong_size = runtime_fixture(&[("nested/asset", b"short", 0o644)]); + let wrong_mode = runtime_fixture(&[("nested/asset", b"trusted", 0o755)]); + let extra = runtime_fixture(&[ + ("nested/asset", b"trusted", 0o644), + ("extra", b"bad", 0o644), + ]); + let duplicate = runtime_fixture(&[ + ("nested/asset", b"trusted", 0o644), + ("./NESTED/ASSET", b"bad", 0o644), + ]); + let missing = runtime_fixture(&[]); + for other in [ + wrong_content, + wrong_size, + wrong_mode, + extra, + duplicate, + missing, + ] { + let dir = tempfile::tempdir().unwrap(); + assert!( + super::install_runtime(dir.path(), other.archive.as_slice(), &fixture.files) + .is_err() + ); + assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); + assert!(!dir.path().join("nested/asset").exists()); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(has_bundled_cli)] + #[test] + fn repair_verifies_archive_entries_even_when_cached_files_are_valid() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); + install_runtime(dir.path(), &fixture).unwrap(); + fs::remove_file(dir.path().join(RUNTIME_NODE_NAME)).unwrap(); + let mismatched = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); + + assert!( + super::install_runtime(dir.path(), mismatched.archive.as_slice(), &fixture.files) + .is_err() + ); + assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); + assert_eq!( + fs::read(dir.path().join("nested/asset")).unwrap(), + b"trusted" + ); + assert_no_runtime_temps(dir.path()); + } + + #[cfg(has_bundled_cli)] + #[test] + fn warm_runtime_rejects_same_size_corruption_despite_unchanged_metadata() { + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[]); + install_runtime(dir.path(), &fixture).unwrap(); + let runtime = dir.path().join(RUNTIME_NODE_NAME); + let original = fs::metadata(&runtime).unwrap(); + fs::write(&runtime, b"corrupt").unwrap(); + fs::File::options() + .write(true) + .open(&runtime) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(original.modified().unwrap())) + .unwrap(); + assert!( + super::install_runtime(dir.path(), b"invalid archive".as_slice(), &fixture.files) + .is_err() + ); + assert_eq!(fs::read(&runtime).unwrap(), b"corrupt"); + install_runtime(dir.path(), &fixture).unwrap(); + assert_eq!(fs::read(&runtime).unwrap(), b"runtime"); } #[cfg(has_bundled_cli)] @@ -1457,13 +1706,15 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let valid = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); let mut invalid_crc = valid.clone(); - let crc = invalid_crc.len() - 8; - invalid_crc[crc] ^= 0xFF; + let crc = invalid_crc.archive.len() - 8; + invalid_crc.archive[crc] ^= 0xFF; let mut invalid_length = valid.clone(); - let length = invalid_length.len() - 4; - invalid_length[length] ^= 0xFF; - let truncated_trailer = valid[..valid.len() - 1].to_vec(); - let truncated_body = valid[..valid.len() / 2].to_vec(); + let length = invalid_length.archive.len() - 4; + invalid_length.archive[length] ^= 0xFF; + let mut truncated_trailer = valid.clone(); + truncated_trailer.archive.truncate(valid.archive.len() - 1); + let mut truncated_body = valid.clone(); + truncated_body.archive.truncate(valid.archive.len() / 2); for archive in [ invalid_crc, invalid_length, @@ -1492,7 +1743,14 @@ mod tests { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(header.as_bytes()).unwrap(); encoder.write_all(b"short").unwrap(); - let archive = encoder.finish().unwrap(); + let mut archive = runtime_fixture(&[]); + archive.archive = encoder.finish().unwrap(); + archive + .files + .iter_mut() + .find(|file| file.path == RUNTIME_NODE_NAME) + .unwrap() + .size = 128 * 1024; assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); @@ -1512,7 +1770,10 @@ mod tests { archive .append_data(&mut header, RUNTIME_BINARY_NAME, b"wrapper".as_slice()) .unwrap(); - let archive = archive.into_inner().unwrap().finish().unwrap(); + let archive = RuntimeFixture { + archive: archive.into_inner().unwrap().finish().unwrap(), + files: runtime_fixture(&[]).files, + }; assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); From 523f7b55bbdf3e3c6a7c906997d0c77bbf9cf5e6 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Tue, 15 Sep 2026 22:11:52 -0700 Subject: [PATCH 16/18] fix(rust): reuse owner-only immutable runtime caches Verify readability through bounded hashing and executable access using effective process credentials rather than requiring every permission class to match the archive. Reuse locked rustix for a safe Unix access check and cover owner-only cache modes without re-extraction. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.lock | 1 + rust/Cargo.toml | 5 +++- rust/src/embeddedcli.rs | 65 +++++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5bbafdafef..df117a08c5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -477,6 +477,7 @@ dependencies = [ "regex", "reqwest", "rusqlite", + "rustix", "schemars", "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a6cdc1d320..19b2170d83 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2", "dep:rustix"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -69,6 +69,9 @@ futures-util = "0.3" reqwest = { version = "0.12", default-features = false, features = ["stream", "http2", "default-tls"] } tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "native-tls"] } +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", features = ["fs"], optional = true } + [target.'cfg(windows)'.dependencies] zip = { version = "2", default-features = false, features = ["deflate"], optional = true } windows-sys = { version = "0.61", default-features = false, features = [ diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 34265cbc53..cfd6717cfe 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -572,18 +572,19 @@ fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result match hash_runtime_file(&mut file, asset.size) { Ok(digest) => return Ok(digest == asset.sha256), @@ -1689,7 +1690,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let archive = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &archive).unwrap(); - for mode in [0o644, 0o655, 0o745, 0o754] { + for mode in [0o644, 0o400] { fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); install_runtime(dir.path(), &archive).unwrap(); assert_eq!( @@ -1700,6 +1701,25 @@ mod tests { assert_no_runtime_temps(dir.path()); } + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_reuses_executable_modes_without_group_or_other_access() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let fixture = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &fixture).unwrap(); + for mode in [0o745, 0o754, 0o700, 0o500] { + fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + mode + ); + } + assert_no_runtime_temps(dir.path()); + } + #[cfg(has_bundled_cli)] #[test] fn runtime_archive_errors_do_not_publish_and_clean_up_staging() { @@ -1974,6 +1994,17 @@ mod tests { #[cfg(all(has_bundled_cli, unix))] #[test] fn warm_runtime_install_needs_no_writable_cache() { + assert_read_only_runtime_cache(0o555); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn warm_runtime_reuses_owner_only_immutable_cache() { + assert_read_only_runtime_cache(0o500); + } + + #[cfg(all(has_bundled_cli, unix))] + fn assert_read_only_runtime_cache(permission_mask: u32) { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); @@ -1989,12 +2020,16 @@ mod tests { let mut read_only_files = Vec::new(); for name in files { let path = dir.path().join(name); - let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o555; + let mode = fs::metadata(&path).unwrap().permissions().mode() & permission_mask; fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); read_only_files.push((path, mode)); } - fs::set_permissions(dir.path().join("nested"), fs::Permissions::from_mode(0o555)).unwrap(); - fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + fs::set_permissions( + dir.path().join("nested"), + fs::Permissions::from_mode(permission_mask), + ) + .unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(permission_mask)).unwrap(); let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); let result = install_runtime(dir.path(), &archive); fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); From 85e7aea5a23ce99b67ab3486a52ed21b3a29a446 Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Wed, 16 Sep 2026 00:03:56 -0700 Subject: [PATCH 17/18] fix(rust): keep runtime memory fix dependency-free Remove the generated hash manifest and runtime sha2/rustix dependencies. Compare installed files directly against streamed archive entries with fixed-size buffers, and retain the original content-based warm-cache permission behavior. Keep atomic staging, archive integrity checks and regression coverage for immutable native libraries and multi-chunk corruption. Restore Cargo.toml, Cargo.lock and the build generator exactly to the PR base. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rust/Cargo.lock | 63 +---- rust/Cargo.toml | 8 +- rust/build/in_process.rs | 44 +--- rust/src/embeddedcli.rs | 524 ++++++++++----------------------------- 4 files changed, 150 insertions(+), 489 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index df117a08c5..b91eebd06c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -70,15 +70,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -138,15 +129,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -172,15 +154,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - [[package]] name = "data-encoding" version = "2.11.0" @@ -204,18 +177,8 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", - "crypto-common 0.1.7", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "crypto-common 0.2.2", + "block-buffer", + "crypto-common", ] [[package]] @@ -477,7 +440,6 @@ dependencies = [ "regex", "reqwest", "rusqlite", - "rustix", "schemars", "serde", "serde_json", @@ -584,15 +546,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "hybrid-array" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "1.10.1" @@ -1500,19 +1453,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", + "cpufeatures", + "digest", ] [[package]] name = "sha2" -version = "0.11.0" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", - "digest 0.11.3", + "cpufeatures", + "digest", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 19b2170d83..4495d3928c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip", "dep:sha2", "dep:rustix"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -59,7 +59,6 @@ getrandom = "0.2" uuid = { version = "1", default-features = false, features = ["v4"] } flate2 = { version = "1", optional = true } tar = { version = "0.4", optional = true } -sha2 = { version = "0.11", default-features = false, optional = true } # LLM inference callback transport: idiomatic HTTP/WebSocket forwarding for the # `CopilotRequestHandler`, plus base64/byte/stream plumbing for the chunk protocol. base64 = "0.22" @@ -69,9 +68,6 @@ futures-util = "0.3" reqwest = { version = "0.12", default-features = false, features = ["stream", "http2", "default-tls"] } tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "native-tls"] } -[target.'cfg(unix)'.dependencies] -rustix = { version = "1", features = ["fs"], optional = true } - [target.'cfg(windows)'.dependencies] zip = { version = "2", default-features = false, features = ["deflate"], optional = true } windows-sys = { version = "0.61", default-features = false, features = [ @@ -129,7 +125,7 @@ required-features = ["test-support"] dirs = "5" flate2 = "1" serde_json = "1" -sha2 = { version = "0.11", default-features = false } +sha2 = "0.10" tar = "0.4" ureq = { version = "2", default-features = false, features = ["native-tls"] } native-tls = "0.2" diff --git a/rust/build/in_process.rs b/rust/build/in_process.rs index f72f99cbdb..0a8bdb0fcf 100644 --- a/rust/build/in_process.rs +++ b/rust/build/in_process.rs @@ -201,7 +201,7 @@ fn emit_embedded( platform: Platform, include_runtime: bool, ) { - let (runtime_archive, runtime_files) = + let runtime_archive = build_embedded_runtime_archive(runtime_package, platform, include_runtime); std::fs::write(out.join("copilot_cli.archive"), cli_archive) .expect("failed to write copilot_cli.archive"); @@ -213,8 +213,6 @@ fn emit_embedded( pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); pub(super) static RUNTIME_ARCHIVE: &[u8] = include_bytes!("copilot_runtime.archive"); pub(super) const CLI_BINARY_SIZE: u64 = {cli_binary_size}; -pub(super) static RUNTIME_FILES: &[super::RuntimeFile] = &[ -{runtime_files}]; "# ); @@ -225,13 +223,12 @@ fn build_embedded_runtime_archive( package: &[u8], platform: Platform, include_runtime: bool, -) -> (Vec, String) { +) -> Vec { let encoder = flate2::GzBuilder::new() .mtime(0) .write(Vec::new(), flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let mut files = String::new(); - let runtime = append_hostless_runtime_tree(&mut archive, package, platform, &mut files); + let runtime = append_hostless_runtime_tree(&mut archive, package, platform); if include_runtime { append_archive_file( &mut archive, @@ -239,22 +236,19 @@ fn build_embedded_runtime_archive( &runtime, 0o644, ); - append_runtime_manifest(&mut files, platform.runtime_library_name(), &runtime, 0o644); } let encoder = archive .into_inner() .expect("failed to finish minimal embedded CLI archive"); - let archive = encoder + encoder .finish() - .expect("failed to compress minimal embedded CLI archive"); - (archive, files) + .expect("failed to compress minimal embedded CLI archive") } fn append_hostless_runtime_tree( archive: &mut tar::Builder, package: &[u8], platform: Platform, - files: &mut String, ) -> Vec { let decoder = flate2::read::GzDecoder::new(package); let mut source = tar::Archive::new(decoder); @@ -290,14 +284,6 @@ fn append_hostless_runtime_tree( &bytes, mode, ); - append_runtime_manifest( - files, - destination - .to_str() - .expect("npm package paths are valid UTF-8"), - &bytes, - mode, - ); } runtime.unwrap_or_else(|| { panic!( @@ -307,19 +293,6 @@ fn append_hostless_runtime_tree( }) } -fn append_runtime_manifest(files: &mut String, path: &str, bytes: &[u8], mode: u32) { - use std::fmt::Write as _; - - let sha256: [u8; 32] = sha2::Sha256::digest(bytes).into(); - writeln!( - files, - " super::RuntimeFile {{ path: std::borrow::Cow::Borrowed({path:?}), size: {}, mode: {}, sha256: {sha256:?} }},", - bytes.len(), - mode & 0o777, - ) - .expect("write runtime manifest"); -} - fn hostless_runtime_path(source: &str, platform: Platform) -> Option { let relative = source.strip_prefix("package/")?; let parts: Vec<&str> = relative.split('/').collect(); @@ -993,10 +966,5 @@ fn archive_zip_entry_size(zip_bytes: &[u8], binary_name: &str) -> Option { fn verify_hash(data: &[u8], expected: &str) -> bool { let mut hasher = sha2::Sha256::new(); hasher.update(data); - let actual: String = hasher - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect(); - actual == expected + format!("{:x}", hasher.finalize()) == expected } diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index cfd6717cfe..c1398fa0d4 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -21,10 +21,10 @@ //! header); anything that looks truncated or quarantined is re-extracted, and //! the whole publish is retried before surfacing a clear, actionable error. //! -//! Runtime assets are hashed with bounded buffers against a manifest generated -//! into the consumer binary at build time. A valid warm cache needs only read -//! access and never decompresses the archive. Changed files are extracted in -//! one streaming pass and published after manifest and gzip validation. +//! Runtime assets are compared and extracted with bounded buffers rather than +//! whole-file allocations. Matching files need only read access and retain +//! their installed permissions. Changed files are staged beside their targets +//! and published after archive validation, including the gzip trailer. //! Installation is atomic per file, not across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure @@ -33,7 +33,7 @@ // the standard `cargo test --no-default-features` job has `has_bundled_cli` // off but still needs to exercise them. #[cfg(has_bundled_cli)] -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; #[cfg(any(has_bundled_cli, test))] use std::fs; #[cfg(has_bundled_cli)] @@ -45,8 +45,6 @@ use std::sync::OnceLock; #[cfg(any(has_bundled_cli, test))] use std::sync::atomic::{AtomicU64, Ordering}; -#[cfg(has_bundled_cli)] -use sha2::{Digest, Sha256}; #[cfg(has_bundled_cli)] use tracing::{info, warn}; @@ -173,7 +171,7 @@ pub(crate) fn runtime_path() -> Option { #[cfg(has_bundled_cli)] { let dir = default_install_dir(CLI_VERSION); - match install_runtime(&dir, build_time::RUNTIME_ARCHIVE, build_time::RUNTIME_FILES) { + match install_runtime(&dir, build_time::RUNTIME_ARCHIVE) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -201,11 +199,7 @@ pub(crate) fn install_runtime_at(extract_dir: &Path) -> Option { return None; } }; - match install_runtime( - &install_dir, - build_time::RUNTIME_ARCHIVE, - build_time::RUNTIME_FILES, - ) { + match install_runtime(&install_dir, build_time::RUNTIME_ARCHIVE) { Ok(path) => { info!(path = %path.display(), version = CLI_VERSION, "embedded runtime installed"); return Some(path); @@ -287,20 +281,7 @@ const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.dylib"; const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; #[cfg(has_bundled_cli)] -#[derive(Clone, Debug)] -struct RuntimeFile { - path: std::borrow::Cow<'static, str>, - size: u64, - mode: u32, - sha256: [u8; 32], -} - -#[cfg(has_bundled_cli)] -fn install_runtime( - install_dir: &Path, - archive: impl Read, - files: &[RuntimeFile], -) -> Result { +fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { fs::create_dir_all(install_dir) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::CreateDir, e))?; let root = fs::canonicalize(install_dir) @@ -309,49 +290,8 @@ fn install_runtime( #[cfg(feature = "bundled-in-process")] required.push(RUNTIME_LIBRARY_NAME); let mut seen = HashSet::new(); - let mut assets = HashMap::new(); - let mut needs_install = false; - for asset in files { - let path = runtime_asset_path(Path::new(asset.path.as_ref()))?; - if !seen.insert(path.as_os_str().to_ascii_lowercase()) { - return Err(EmbeddedCliError::with_message( - EmbeddedCliErrorKind::Archive, - format!("duplicate embedded runtime asset path: {}", path.display()), - )); - } - if !selected_runtime_asset(&path) { - continue; - } - if let Some(index) = required.iter().position(|name| path == Path::new(name)) { - if asset.size == 0 { - return Err(EmbeddedCliError::with_message( - EmbeddedCliErrorKind::Verification, - format!("embedded runtime artifact is empty: {}", path.display()), - )); - } - required.remove(index); - } - let target = root.join(&path); - let parent = target.parent().ok_or_else(|| { - EmbeddedCliError::with_message( - EmbeddedCliErrorKind::Archive, - format!("embedded runtime asset has no parent: {}", path.display()), - ) - })?; - check_runtime_asset_parent(&root, parent, false)?; - let valid = runtime_file_is_valid(asset, &target)?; - needs_install |= !valid; - assets.insert(path, (asset, valid)); - } - if !required.is_empty() { - return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); - } - if !needs_install { - return Ok(install_dir.join(RUNTIME_BINARY_NAME)); - } - + let mut changed = HashSet::new(); let mut pending = Vec::new(); - seen.clear(); let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); for entry in tar .entries() @@ -375,53 +315,65 @@ fn install_runtime( if !selected_runtime_asset(&path) { continue; } - let (asset, valid) = assets.remove(&path).ok_or_else(|| { - EmbeddedCliError::with_message( - EmbeddedCliErrorKind::Verification, - format!( - "runtime archive entry is absent from manifest: {}", - path.display() - ), - ) - })?; - let mode = entry - .header() - .mode() - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? - & 0o777; - if entry.size() != asset.size || mode != asset.mode { - return Err(EmbeddedCliError::with_message( - EmbeddedCliErrorKind::Verification, - format!( - "runtime archive metadata differs from manifest: {}", - path.display() - ), - )); - } - if valid { - let digest = hash_runtime_file(&mut entry, asset.size) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - if digest != asset.sha256 { + if let Some(index) = required.iter().position(|name| path == Path::new(name)) { + if entry.size() == 0 { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!( - "runtime archive content differs from manifest: {}", - asset.path - ), + format!("embedded runtime artifact is empty: {}", path.display()), )); } - } else { - let target = root.join(&path); - check_runtime_asset_parent(&root, target.parent().expect("checked asset path"), true)?; - pending.push(stage_runtime_entry(&mut entry, &target, asset)?); + required.remove(index); + } + let target = root.join(&path); + let parent = target.parent().ok_or_else(|| { + EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Archive, + format!("embedded runtime asset has no parent: {}", path.display()), + ) + })?; + check_runtime_asset_parent(&root, parent, false)?; + match existing_runtime_file(&target, entry.size())? { + Some(mut installed) => { + if !runtime_entry_matches(&mut entry, &mut installed)? { + changed.insert(path); + } + } + None => { + check_runtime_asset_parent(&root, parent, true)?; + pending.push(stage_runtime_entry(&mut entry, &target)?); + } } } // TAR's end marker can precede gzip's CRC and size trailer. std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - if !assets.is_empty() { + if !required.is_empty() { return Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()); } + // Recover bytes consumed by failed comparisons without retaining their + // prefixes in memory. Cold installs and valid warm caches need one pass. + if !changed.is_empty() { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar + .entries() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + { + let mut entry = + entry.map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry + .path() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + let path = runtime_asset_path(&path)?; + if changed.contains(&path) { + pending.push(stage_runtime_entry(&mut entry, &root.join(path))?); + } + } + std::io::copy(&mut tar.into_inner(), &mut std::io::sink()) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; + } for staged in pending { publish(&staged.temporary, &staged.target)?; } @@ -557,7 +509,7 @@ impl Drop for StagedRuntimeFile { } #[cfg(has_bundled_cli)] -fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result { +fn existing_runtime_file(target: &Path, size: u64) -> Result, EmbeddedCliError> { let metadata = match fs::symlink_metadata(target) { Ok(metadata) if metadata.is_file() => Some(metadata), Ok(_) => { @@ -571,62 +523,54 @@ fn runtime_file_is_valid(asset: &RuntimeFile, target: &Path) -> Result match hash_runtime_file(&mut file, asset.size) { - Ok(digest) => return Ok(digest == asset.sha256), - Err(e) => { - tracing::debug!(path = %target.display(), error = %e, - "existing runtime asset cannot be verified; repairing"); - } - }, + Ok(file) => return Ok(Some(file)), Err(e) => { tracing::debug!(path = %target.display(), error = %e, "existing runtime asset cannot be read; repairing"); } } } - Ok(false) + Ok(None) } #[cfg(has_bundled_cli)] -fn hash_runtime_file(reader: &mut impl Read, size: u64) -> std::io::Result<[u8; 32]> { +fn runtime_entry_matches( + entry: &mut tar::Entry<'_, R>, + installed: &mut fs::File, +) -> Result { let mut buffer = [0u8; 64 * 1024]; - let mut hash = Sha256::new(); - let mut remaining = size; + let mut on_disk = [0u8; 64 * 1024]; + let mut remaining = entry.size(); while remaining > 0 { let length = remaining.min(buffer.len() as u64) as usize; - reader.read_exact(&mut buffer[..length])?; + entry + .read_exact(&mut buffer[..length]) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; remaining -= length as u64; - hash.update(&buffer[..length]); + if let Err(error) = installed.read_exact(&mut on_disk[..length]) { + tracing::debug!(%error, "existing runtime asset cannot be read; repairing"); + return Ok(false); + } + if on_disk[..length] != buffer[..length] { + return Ok(false); + } } - if reader.read(&mut buffer[..1])? != 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "runtime file exceeds manifest size", - )); + match installed.read(&mut on_disk[..1]) { + Ok(read) => Ok(read == 0), + Err(error) => { + tracing::debug!(%error, "existing runtime asset cannot be read; repairing"); + Ok(false) + } } - Ok(hash.finalize().into()) } #[cfg(has_bundled_cli)] fn stage_runtime_entry( entry: &mut tar::Entry<'_, R>, target: &Path, - asset: &RuntimeFile, ) -> Result { let parent = target.parent().expect("runtime asset has a checked parent"); let (temporary, mut file) = create_temp_file(parent)?; @@ -634,7 +578,7 @@ fn stage_runtime_entry( temporary, target: target.to_path_buf(), }; - let result = write_runtime_entry(entry, &mut file, asset); + let result = write_runtime_entry(entry, &mut file); // Close handles before cleanup or replacement, including on Windows. drop(file); result?; @@ -645,35 +589,25 @@ fn stage_runtime_entry( fn write_runtime_entry( entry: &mut tar::Entry<'_, R>, file: &mut fs::File, - asset: &RuntimeFile, ) -> Result<(), EmbeddedCliError> { - let mut buffer = [0u8; 64 * 1024]; - let mut remaining = asset.size; - let mut hash = Sha256::new(); - while remaining > 0 { - let length = remaining.min(buffer.len() as u64) as usize; - entry - .read_exact(&mut buffer[..length]) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))?; - remaining -= length as u64; - file.write_all(&buffer[..length]) - .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; - hash.update(&buffer[..length]); - } - let digest: [u8; 32] = hash.finalize().into(); - if digest != asset.sha256 { + let size = entry.size(); + let written = std::io::copy(&mut (&mut *entry).take(size), &mut *file) + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; + if written != size { return Err(EmbeddedCliError::with_message( EmbeddedCliErrorKind::Verification, - format!( - "runtime archive content differs from manifest: {}", - asset.path - ), + format!("runtime entry size mismatch: read {written} bytes, expected {size}"), )); } #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - file.set_permissions(fs::Permissions::from_mode(asset.mode)) + let mode = entry + .header() + .mode() + .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Archive, e))? + & 0o777; + file.set_permissions(fs::Permissions::from_mode(mode)) .map_err(|e| EmbeddedCliError::new(EmbeddedCliErrorKind::Io, e))?; } file.sync_all() @@ -1348,12 +1282,7 @@ mod tests { fs::write(dir.path().join(RUNTIME_NODE_NAME), b"stale runtime").expect("seed runtime"); fs::write(dir.path().join(RUNTIME_BINARY_NAME), b"stale wrapper").expect("seed wrapper"); - super::install_runtime( - dir.path(), - build_time::RUNTIME_ARCHIVE, - build_time::RUNTIME_FILES, - ) - .expect("install runtime"); + install_runtime(dir.path(), build_time::RUNTIME_ARCHIVE).expect("install runtime"); assert_eq!( fs::read(dir.path().join(RUNTIME_NODE_NAME)).expect("read runtime"), @@ -1387,19 +1316,7 @@ mod tests { } #[cfg(has_bundled_cli)] - #[derive(Clone)] - struct RuntimeFixture { - archive: Vec, - files: Vec, - } - - #[cfg(has_bundled_cli)] - fn install_runtime(dir: &Path, fixture: &RuntimeFixture) -> Result { - super::install_runtime(dir, fixture.archive.as_slice(), &fixture.files) - } - - #[cfg(has_bundled_cli)] - fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> RuntimeFixture { + fn runtime_fixture(extra: &[(&str, &[u8], u32)]) -> Vec { let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); let mut archive = tar::Builder::new(encoder); let mut entries = vec![ @@ -1409,7 +1326,6 @@ mod tests { #[cfg(feature = "bundled-in-process")] entries.push((RUNTIME_LIBRARY_NAME, b"library".as_slice(), 0o644)); entries.extend_from_slice(extra); - let mut files = Vec::new(); for (name, bytes, mode) in entries { let mut header = tar::Header::new_gnu(); // Raw names also let the installer see traversal fixtures which @@ -1419,174 +1335,8 @@ mod tests { header.set_mode(mode); header.set_cksum(); archive.append(&header, bytes).expect("append fixture"); - files.push(RuntimeFile { - path: name.to_owned().into(), - size: bytes.len() as u64, - mode, - sha256: Sha256::digest(bytes).into(), - }); - } - RuntimeFixture { - archive: archive.into_inner().unwrap().finish().unwrap(), - files, - } - } - - #[cfg(has_bundled_cli)] - #[test] - fn generated_manifest_matches_every_embedded_runtime_file() { - let mut archive = - tar::Archive::new(flate2::read::GzDecoder::new(build_time::RUNTIME_ARCHIVE)); - let mut manifest: HashMap<_, _> = build_time::RUNTIME_FILES - .iter() - .map(|file| { - ( - runtime_asset_path(Path::new(file.path.as_ref())).unwrap(), - file, - ) - }) - .collect(); - assert_eq!(manifest.len(), build_time::RUNTIME_FILES.len()); - assert!(manifest.contains_key(Path::new(RUNTIME_NODE_NAME))); - assert!(manifest.contains_key(Path::new(RUNTIME_BINARY_NAME))); - #[cfg(feature = "bundled-in-process")] - assert!(manifest.contains_key(Path::new(RUNTIME_LIBRARY_NAME))); - for entry in archive.entries().unwrap() { - let mut entry = entry.unwrap(); - assert!(entry.header().entry_type().is_file()); - let path = runtime_asset_path(&entry.path().unwrap()).unwrap(); - let file = manifest.remove(&path).expect("manifest entry"); - assert_eq!(entry.size(), file.size); - assert_eq!(entry.header().mode().unwrap() & 0o777, file.mode); - assert_eq!( - hash_runtime_file(&mut entry, file.size).unwrap(), - file.sha256 - ); - } - assert!(manifest.is_empty()); - std::io::copy(&mut archive.into_inner(), &mut std::io::sink()).unwrap(); - } - - #[cfg(has_bundled_cli)] - struct UnreadableArchive; - - #[cfg(has_bundled_cli)] - impl Read for UnreadableArchive { - fn read(&mut self, _: &mut [u8]) -> std::io::Result { - panic!("valid warm installation must not read or decompress the archive") - } - } - - #[cfg(has_bundled_cli)] - #[test] - fn warm_runtime_verifies_all_files_without_reading_archive() { - let dir = tempfile::tempdir().unwrap(); - let fixture = runtime_fixture(&[("nested/asset", &[0xAB; 65_537], 0o644)]); - install_runtime(dir.path(), &fixture).unwrap(); - let path = super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); - assert_eq!(path, dir.path().join(RUNTIME_BINARY_NAME)); - assert_no_runtime_temps(dir.path()); - - let mut changed_manifest = fixture.files.clone(); - changed_manifest[0].sha256[0] ^= 1; - // Same size and mode are insufficient: the hash must come from the - // trusted manifest, not any mutable cache-side metadata. - assert!( - super::install_runtime(dir.path(), b"invalid archive".as_slice(), &changed_manifest) - .is_err() - ); - } - - #[cfg(has_bundled_cli)] - #[test] - fn cold_and_repair_consume_one_forward_only_archive() { - struct Counted<'a> { - bytes: &'a [u8], - consumed: usize, - } - impl Read for Counted<'_> { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - let count = self.bytes.read(buffer)?; - self.consumed += count; - Ok(count) - } - } - let dir = tempfile::tempdir().unwrap(); - let bytes = vec![0xAB; 65_537]; - let fixture = runtime_fixture(&[("nested/asset", &bytes, 0o644)]); - for corruption in [None, Some(b"changed".as_slice()), Some(b"run"), Some(b"")] { - if let Some(corruption) = corruption { - fs::write(dir.path().join(RUNTIME_NODE_NAME), corruption).unwrap(); - } - let mut reader = Counted { - bytes: &fixture.archive, - consumed: 0, - }; - super::install_runtime(dir.path(), &mut reader, &fixture.files).unwrap(); - assert_eq!(reader.consumed, fixture.archive.len()); - assert_eq!( - fs::read(dir.path().join(RUNTIME_NODE_NAME)).unwrap(), - b"runtime" - ); - assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), bytes); - assert_no_runtime_temps(dir.path()); } - } - - #[cfg(has_bundled_cli)] - #[test] - fn manifest_mismatches_never_publish_staged_files() { - let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); - let wrong_content = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); - let wrong_size = runtime_fixture(&[("nested/asset", b"short", 0o644)]); - let wrong_mode = runtime_fixture(&[("nested/asset", b"trusted", 0o755)]); - let extra = runtime_fixture(&[ - ("nested/asset", b"trusted", 0o644), - ("extra", b"bad", 0o644), - ]); - let duplicate = runtime_fixture(&[ - ("nested/asset", b"trusted", 0o644), - ("./NESTED/ASSET", b"bad", 0o644), - ]); - let missing = runtime_fixture(&[]); - for other in [ - wrong_content, - wrong_size, - wrong_mode, - extra, - duplicate, - missing, - ] { - let dir = tempfile::tempdir().unwrap(); - assert!( - super::install_runtime(dir.path(), other.archive.as_slice(), &fixture.files) - .is_err() - ); - assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); - assert!(!dir.path().join("nested/asset").exists()); - assert_no_runtime_temps(dir.path()); - } - } - - #[cfg(has_bundled_cli)] - #[test] - fn repair_verifies_archive_entries_even_when_cached_files_are_valid() { - let dir = tempfile::tempdir().unwrap(); - let fixture = runtime_fixture(&[("nested/asset", b"trusted", 0o644)]); - install_runtime(dir.path(), &fixture).unwrap(); - fs::remove_file(dir.path().join(RUNTIME_NODE_NAME)).unwrap(); - let mismatched = runtime_fixture(&[("nested/asset", b"corrupt", 0o644)]); - - assert!( - super::install_runtime(dir.path(), mismatched.archive.as_slice(), &fixture.files) - .is_err() - ); - assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); - assert_eq!( - fs::read(dir.path().join("nested/asset")).unwrap(), - b"trusted" - ); - assert_no_runtime_temps(dir.path()); + archive.into_inner().unwrap().finish().unwrap() } #[cfg(has_bundled_cli)] @@ -1604,10 +1354,7 @@ mod tests { .unwrap() .set_times(fs::FileTimes::new().set_modified(original.modified().unwrap())) .unwrap(); - assert!( - super::install_runtime(dir.path(), b"invalid archive".as_slice(), &fixture.files) - .is_err() - ); + assert!(install_runtime(dir.path(), b"invalid archive").is_err()); assert_eq!(fs::read(&runtime).unwrap(), b"corrupt"); install_runtime(dir.path(), &fixture).unwrap(); assert_eq!(fs::read(&runtime).unwrap(), b"runtime"); @@ -1682,36 +1429,34 @@ mod tests { } } - #[cfg(all(has_bundled_cli, unix))] + #[cfg(has_bundled_cli)] #[test] - fn runtime_repairs_missing_execute_permission() { - use std::os::unix::fs::PermissionsExt; - + fn runtime_repairs_corruption_across_comparison_chunks() { let dir = tempfile::tempdir().unwrap(); - let archive = runtime_fixture(&[]); - let wrapper = install_runtime(dir.path(), &archive).unwrap(); - for mode in [0o644, 0o400] { - fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + let bytes = vec![0xAB; 3 * 64 * 1024 + 17]; + let archive = runtime_fixture(&[("nested/asset", &bytes, 0o644)]); + install_runtime(dir.path(), &archive).unwrap(); + for offset in [0, bytes.len() / 2, bytes.len() - 1] { + let mut corrupt = bytes.clone(); + corrupt[offset] ^= 1; + fs::write(dir.path().join("nested/asset"), &corrupt).unwrap(); install_runtime(dir.path(), &archive).unwrap(); - assert_eq!( - fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, - 0o755 - ); + assert_eq!(fs::read(dir.path().join("nested/asset")).unwrap(), bytes); + assert_no_runtime_temps(dir.path()); } - assert_no_runtime_temps(dir.path()); } #[cfg(all(has_bundled_cli, unix))] #[test] - fn runtime_reuses_executable_modes_without_group_or_other_access() { + fn runtime_reuse_preserves_caller_selected_permissions() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let fixture = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &fixture).unwrap(); - for mode in [0o745, 0o754, 0o700, 0o500] { + for mode in [0o745, 0o754, 0o700, 0o500, 0o400] { fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); - super::install_runtime(dir.path(), UnreadableArchive, &fixture.files).unwrap(); + install_runtime(dir.path(), &fixture).unwrap(); assert_eq!( fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, mode @@ -1726,15 +1471,15 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let valid = runtime_fixture(&[("nested/asset", b"asset", 0o644)]); let mut invalid_crc = valid.clone(); - let crc = invalid_crc.archive.len() - 8; - invalid_crc.archive[crc] ^= 0xFF; + let crc = invalid_crc.len() - 8; + invalid_crc[crc] ^= 0xFF; let mut invalid_length = valid.clone(); - let length = invalid_length.archive.len() - 4; - invalid_length.archive[length] ^= 0xFF; + let length = invalid_length.len() - 4; + invalid_length[length] ^= 0xFF; let mut truncated_trailer = valid.clone(); - truncated_trailer.archive.truncate(valid.archive.len() - 1); + truncated_trailer.truncate(valid.len() - 1); let mut truncated_body = valid.clone(); - truncated_body.archive.truncate(valid.archive.len() / 2); + truncated_body.truncate(valid.len() / 2); for archive in [ invalid_crc, invalid_length, @@ -1763,14 +1508,7 @@ mod tests { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(header.as_bytes()).unwrap(); encoder.write_all(b"short").unwrap(); - let mut archive = runtime_fixture(&[]); - archive.archive = encoder.finish().unwrap(); - archive - .files - .iter_mut() - .find(|file| file.path == RUNTIME_NODE_NAME) - .unwrap() - .size = 128 * 1024; + let archive = encoder.finish().unwrap(); assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_NODE_NAME).exists()); @@ -1790,10 +1528,7 @@ mod tests { archive .append_data(&mut header, RUNTIME_BINARY_NAME, b"wrapper".as_slice()) .unwrap(); - let archive = RuntimeFixture { - archive: archive.into_inner().unwrap().finish().unwrap(), - files: runtime_fixture(&[]).files, - }; + let archive = archive.into_inner().unwrap().finish().unwrap(); assert!(install_runtime(dir.path(), &archive).is_err()); assert!(!dir.path().join(RUNTIME_BINARY_NAME).exists()); @@ -1994,17 +1729,23 @@ mod tests { #[cfg(all(has_bundled_cli, unix))] #[test] fn warm_runtime_install_needs_no_writable_cache() { - assert_read_only_runtime_cache(0o555); + assert_read_only_runtime_cache(0o555, false); } #[cfg(all(has_bundled_cli, unix))] #[test] fn warm_runtime_reuses_owner_only_immutable_cache() { - assert_read_only_runtime_cache(0o500); + assert_read_only_runtime_cache(0o500, false); + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn warm_runtime_reuses_nonexecutable_native_library() { + assert_read_only_runtime_cache(0o555, true); } #[cfg(all(has_bundled_cli, unix))] - fn assert_read_only_runtime_cache(permission_mask: u32) { + fn assert_read_only_runtime_cache(permission_mask: u32, readonly_native_library: bool) { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); @@ -2020,7 +1761,10 @@ mod tests { let mut read_only_files = Vec::new(); for name in files { let path = dir.path().join(name); - let mode = fs::metadata(&path).unwrap().permissions().mode() & permission_mask; + let mut mode = fs::metadata(&path).unwrap().permissions().mode() & permission_mask; + if readonly_native_library && name == RUNTIME_NODE_NAME { + mode &= !0o111; + } fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); read_only_files.push((path, mode)); } From 1dfcddeb3f46d74090997d5bb0e1bbd2f54921fe Mon Sep 17 00:00:00 2001 From: Mohamed Mansour Date: Wed, 16 Sep 2026 09:45:45 -0700 Subject: [PATCH 18/18] fix(rust): validate runtime wrapper execute access Check only the runtime wrapper with POSIX faccessat and effective credentials, preserving readable native libraries and owner-only executable caches. Repair inaccessible wrappers through the existing staged publish path and reject persistent execute denial before returning. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/src/embeddedcli.rs | 109 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index c1398fa0d4..59dddd0ed2 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -23,8 +23,9 @@ //! //! Runtime assets are compared and extracted with bounded buffers rather than //! whole-file allocations. Matching files need only read access and retain -//! their installed permissions. Changed files are staged beside their targets -//! and published after archive validation, including the gzip trailer. +//! their installed permissions, provided the wrapper remains executable by +//! the current process. Changed files are staged beside their targets and +//! published after archive validation, including the gzip trailer. //! Installation is atomic per file, not across the entire runtime bundle. // The atomic-publish + verify helpers (and their unit tests) are pure @@ -334,7 +335,12 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result { - if !runtime_entry_matches(&mut entry, &mut installed)? { + let matches = runtime_entry_matches(&mut entry, &mut installed)?; + #[cfg(unix)] + let matches = matches + && (path != Path::new(RUNTIME_BINARY_NAME) + || check_runtime_wrapper_execute_access(&target).is_ok()); + if !matches { changed.insert(path); } } @@ -377,9 +383,45 @@ fn install_runtime(install_dir: &Path, archive: &[u8]) -> Result std::io::Result<()> { + use std::ffi::{CString, c_char, c_int}; + use std::os::unix::ffi::OsStrExt; + + // std has no effective-credentials access check. Bind the POSIX libc API, + // not a raw syscall: libc handles Linux kernel differences (glibc/musl). + // These / constants are shared by each OS's supported + // x86_64 and aarch64 targets; macOS uses different AT_* values than Linux. + #[cfg(target_os = "linux")] + const AT_FDCWD: c_int = -100; + #[cfg(target_os = "macos")] + const AT_FDCWD: c_int = -2; + #[cfg(target_os = "linux")] + const AT_EACCESS: c_int = 0x200; + #[cfg(target_os = "macos")] + const AT_EACCESS: c_int = 0x10; + const X_OK: c_int = 1; + unsafe extern "C" { + fn faccessat(dirfd: c_int, path: *const c_char, mode: c_int, flags: c_int) -> c_int; + } + + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + // SAFETY: path is NUL-terminated and lives through the call. faccessat + // only reads it and uses the platform's C integer ABI and flag values. + if unsafe { faccessat(AT_FDCWD, path.as_ptr(), X_OK, AT_EACCESS) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + #[cfg(has_bundled_cli)] fn selected_runtime_asset(path: &Path) -> bool { if path == Path::new(CLI_BINARY_NAME) { @@ -1448,13 +1490,13 @@ mod tests { #[cfg(all(has_bundled_cli, unix))] #[test] - fn runtime_reuse_preserves_caller_selected_permissions() { + fn runtime_reuse_preserves_executable_caller_selected_permissions() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let fixture = runtime_fixture(&[]); let wrapper = install_runtime(dir.path(), &fixture).unwrap(); - for mode in [0o745, 0o754, 0o700, 0o500, 0o400] { + for mode in [0o745, 0o754, 0o700, 0o500] { fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); install_runtime(dir.path(), &fixture).unwrap(); assert_eq!( @@ -1465,6 +1507,60 @@ mod tests { assert_no_runtime_temps(dir.path()); } + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_repairs_nonexecutable_wrapper() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + for mode in [0o400, 0o600] { + fs::set_permissions(&wrapper, fs::Permissions::from_mode(mode)).unwrap(); + install_runtime(dir.path(), &archive).unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + assert_eq!(fs::read(&wrapper).unwrap(), b"wrapper"); + assert_no_runtime_temps(dir.path()); + } + } + + #[cfg(all(has_bundled_cli, unix))] + #[test] + fn runtime_rejects_nonexecutable_wrapper_in_readonly_cache() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let archive = runtime_fixture(&[]); + let wrapper = install_runtime(dir.path(), &archive).unwrap(); + fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o400)).unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + let write_denied = fs::File::create(dir.path().join("write-probe")).is_err(); + let result = install_runtime(dir.path(), &archive); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + if write_denied { + assert!( + result.is_err(), + "returned a nonexecutable wrapper: {result:?}" + ); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + 0o400 + ); + } else { + eprintln!("read-only permission enforcement unavailable (e.g. privileged user)"); + fs::remove_file(dir.path().join("write-probe")).unwrap(); + result.unwrap(); + assert_eq!( + fs::metadata(&wrapper).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + assert_no_runtime_temps(dir.path()); + } + #[cfg(has_bundled_cli)] #[test] fn runtime_archive_errors_do_not_publish_and_clean_up_staging() { @@ -1742,6 +1838,7 @@ mod tests { #[test] fn warm_runtime_reuses_nonexecutable_native_library() { assert_read_only_runtime_cache(0o555, true); + assert_read_only_runtime_cache(0o500, true); } #[cfg(all(has_bundled_cli, unix))] @@ -1762,7 +1859,7 @@ mod tests { for name in files { let path = dir.path().join(name); let mut mode = fs::metadata(&path).unwrap().permissions().mode() & permission_mask; - if readonly_native_library && name == RUNTIME_NODE_NAME { + if readonly_native_library && name != RUNTIME_BINARY_NAME { mode &= !0o111; } fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap();