diff --git a/Cargo.lock b/Cargo.lock index 54fabbbeb3..cd4061fbd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,6 +1571,7 @@ version = "0.1.0" dependencies = [ "clap", "getrandom 0.3.4", + "libc", "litebox_broker_core", "litebox_broker_host", "litebox_broker_local", diff --git a/litebox/src/broker/mod.rs b/litebox/src/broker/mod.rs index aba801e952..20d58a54c0 100644 --- a/litebox/src/broker/mod.rs +++ b/litebox/src/broker/mod.rs @@ -19,8 +19,8 @@ use litebox_broker_protocol::fs::{ }; use litebox_broker_protocol::pipe::{CreatePipeResponse, MAX_PIPE_TRANSFER_SIZE}; use litebox_broker_protocol::process::{ - CreatedProcess, MAX_CHILD_OBJECT_DUPLICATES, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessExitStatus, - ProcessTermination, + CreatedProcess, MAX_CHILD_MEMORY_WRITE_SIZE, MAX_CHILD_OBJECT_DUPLICATES, + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessExitStatus, ProcessTermination, }; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::readiness::ReadinessFlags; @@ -62,6 +62,13 @@ pub(crate) trait BrokerControl: Send + Sync { payload: &[u8], ) -> core::result::Result<(), BrokerControlError>; + fn write_child_memory( + &self, + child_process_id: litebox_broker_protocol::ProcessId, + offset: u64, + data: &[u8], + ) -> core::result::Result<(), BrokerControlError>; + fn exit_child_process( &self, child_process_id: litebox_broker_protocol::ProcessId, @@ -523,6 +530,21 @@ where }) } + fn write_child_memory( + &self, + child_process_id: litebox_broker_protocol::ProcessId, + offset: u64, + data: &[u8], + ) -> core::result::Result<(), BrokerControlError> { + if data.is_empty() || data.len() > MAX_CHILD_MEMORY_WRITE_SIZE as usize { + return Err(BrokerControlError::Broker(ErrorCode::ResourceExhausted)); + } + let lease = self.acquire_shared_buffer(data.len())?; + self.request(|local| { + local.write_child_memory(child_process_id, offset, lease.sequence(), data) + }) + } + fn exit_child_process( &self, child_process_id: litebox_broker_protocol::ProcessId, diff --git a/litebox/src/event/wait.rs b/litebox/src/event/wait.rs index 9470728167..3b5e018351 100644 --- a/litebox/src/event/wait.rs +++ b/litebox/src/event/wait.rs @@ -412,15 +412,20 @@ impl<'a, Platform: RawSyncPrimitivesProvider + TimeProvider> WaitContext<'a, Pla // Check for timeout before checking for an interrupt. This is important // for things like sleep(), where we want to return `TimedOut` rather than // `Interrupted` if the deadline has already passed. + if self.deadline.is_some() && self.remaining_timeout().is_none() { + return Err(WaitError::TimedOut); + } + if self.check_interrupt.check_for_interrupt() { + return Err(WaitError::Interrupted); + } + + // The interrupt check may block (for example, while another thread + // pauses this one), so compute the remaining timeout after it. let timeout = if self.deadline.is_some() { Some(self.remaining_timeout().ok_or(WaitError::TimedOut)?) } else { None }; - if self.check_interrupt.check_for_interrupt() { - return Err(WaitError::Interrupted); - } - if let Some(timeout) = timeout { let r = self .waker diff --git a/litebox/src/platform/page_mgmt.rs b/litebox/src/platform/page_mgmt.rs index 81a3a08a54..c08acf7d7a 100644 --- a/litebox/src/platform/page_mgmt.rs +++ b/litebox/src/platform/page_mgmt.rs @@ -124,6 +124,14 @@ pub trait PageManagementProvider: RawPointerProvider { /// /// Note it must be aligned to `ALIGN`. const TASK_ADDR_MAX: usize; + /// The upper bound (exclusive) for addresses that the memory manager chooses itself. + /// + /// Mappings at caller-specified fixed addresses may still extend up to + /// [`Self::TASK_ADDR_MAX`]. Platforms whose host allocates in the upper part of the task + /// range can lower this to keep self-placed task memory clear of host memory. + /// + /// Note it must be aligned to `ALIGN` and greater than [`Self::TASK_ADDR_MIN`]. + const PLACEMENT_ADDR_MAX: usize = Self::TASK_ADDR_MAX; /// Alignment of native reservation base addresses, in bytes. /// diff --git a/litebox/src/process.rs b/litebox/src/process.rs index 1cec6ceca1..b63a989d30 100644 --- a/litebox/src/process.rs +++ b/litebox/src/process.rs @@ -8,7 +8,9 @@ use alloc::sync::Arc; use alloc::vec::Vec; use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::process::{ProcessExitStatus, ProcessIdentity, ProcessTermination}; +use litebox_broker_protocol::process::{ + MAX_CHILD_MEMORY_WRITE_SIZE, ProcessExitStatus, ProcessIdentity, ProcessTermination, +}; use litebox_broker_protocol::signal::PendingSignal; use litebox_broker_protocol::{ObjectHandle, ProcessId}; use litebox_platform::time::TimeProvider; @@ -231,6 +233,22 @@ impl Process { .collect()) } + /// Writes `data` at `offset` of this pending child process's memory image, + /// which the child's runner receives when the child starts. + /// + /// The image is zero-filled where nothing was written. + pub fn write_memory(&self, offset: u64, data: &[u8]) -> Result<(), ProcessError> { + let mut offset = offset; + for chunk in data.chunks(MAX_CHILD_MEMORY_WRITE_SIZE as usize) { + self.broker + .write_child_memory(self.identity.process_id, offset, chunk)?; + offset = offset + .checked_add(chunk.len() as u64) + .ok_or(ProcessError::ResourceExhausted)?; + } + Ok(()) + } + /// Records that this pending child process exited without starting a /// runner, leaving it a zombie reporting `exit_status`. pub fn exit(&self, exit_status: ProcessExitStatus) -> Result<(), ProcessError> { diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 66481e734e..e564fb7c2b 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -39,7 +39,7 @@ mod test_platform; pub mod test_support; use alloc::sync::{Arc, Weak}; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use hashbrown::HashMap; use litebox_broker_protocol::{ObjectHandle, ProcessId}; @@ -55,7 +55,8 @@ pub use policy::{ PolicyProfile, SocketPolicy, SocketPolicyError, }; pub use process::{ - AssociationCancellation, BrokerProcess, CallerCredential, ProcessLifecycleSink, ProcessShutdown, + AssociationCancellation, BrokerProcess, CallerCredential, ChildImage, ProcessImage, + ProcessLifecycleSink, ProcessShutdown, }; use random::RandomProvider; use socket::{BrokerSocketPorts, SocketProvider}; @@ -87,6 +88,10 @@ pub struct BrokerCoreLimits { pub max_threads: usize, /// Maximum live broker-allocated thread IDs owned by one process. pub max_threads_per_process: usize, + /// Maximum size in bytes of one pending child's memory image. + pub max_child_image_size: u64, + /// Maximum total size in bytes of child memory images held by the broker. + pub max_total_child_image_size: u64, } impl BrokerCoreLimits { @@ -101,6 +106,8 @@ impl BrokerCoreLimits { max_sockets_per_process: 256, max_threads: 4096, max_threads_per_process: 1024, + max_child_image_size: 1024 * 1024 * 1024, + max_total_child_image_size: 4 * 1024 * 1024 * 1024, }; /// Creates a broker core limit set. @@ -118,6 +125,8 @@ impl BrokerCoreLimits { max_sockets_per_process: Self::DEFAULT.max_sockets_per_process, max_threads: Self::DEFAULT.max_threads, max_threads_per_process: Self::DEFAULT.max_threads_per_process, + max_child_image_size: Self::DEFAULT.max_child_image_size, + max_total_child_image_size: Self::DEFAULT.max_total_child_image_size, } } @@ -141,6 +150,8 @@ impl BrokerCoreLimits { max_sockets_per_process, max_threads: Self::DEFAULT.max_threads, max_threads_per_process: Self::DEFAULT.max_threads_per_process, + max_child_image_size: Self::DEFAULT.max_child_image_size, + max_total_child_image_size: Self::DEFAULT.max_total_child_image_size, } } @@ -186,6 +197,24 @@ impl BrokerCoreLimits { ..self } } + + /// Returns these limits with explicit per-child and broker-wide child + /// memory image size limits. + /// + /// A per-child limit above the broker-wide limit is accepted; the + /// broker-wide limit still applies. + #[must_use] + pub const fn with_child_image_size_limits( + self, + max_child_image_size: u64, + max_total_child_image_size: u64, + ) -> Self { + Self { + max_child_image_size, + max_total_child_image_size, + ..self + } + } } impl Default for BrokerCoreLimits { @@ -212,6 +241,8 @@ pub struct BrokerCore { pub(crate) pending_references: Arc, pub(crate) reserved_pipe_capacity: Arc, pub(crate) reserved_sockets: Arc, + /// Bytes of child memory images held by the broker. + pub(crate) reserved_child_image_size: Arc, pub(crate) random_provider: Arc, pub(crate) socket_provider: Arc, pub(crate) timer_provider: Arc, @@ -272,6 +303,7 @@ impl BrokerCore { pending_references: Arc::new(AtomicUsize::new(0)), reserved_pipe_capacity: Arc::new(AtomicUsize::new(0)), reserved_sockets: Arc::new(AtomicUsize::new(0)), + reserved_child_image_size: Arc::new(AtomicU64::new(0)), random_provider, socket_provider, timer_provider, diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index e23ddc6737..b6adc11ba4 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -2,10 +2,13 @@ // Licensed under the MIT license. use alloc::{ + boxed::Box, sync::{Arc, Weak}, vec::Vec, }; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::any::Any; +use core::ops::Range; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use crate::object::{self, ObjectEntry, ObjectReference, ObjectRights}; use crate::readiness::{ReadinessRegistration, ReadinessSink}; @@ -29,6 +32,88 @@ pub trait ProcessLifecycleSink: Send + Sync { /// Host runner shutdown action installed into a broker process. pub type ProcessShutdown = Arc; +/// Platform storage for a pending child's memory image. +/// +/// The image is a byte array, zero wherever it was not written, whose layout +/// only the parent and the child's runner understand. +pub trait ProcessImage: Send { + /// Writes `data` at `offset`, extending the image as needed. + fn write(&mut self, offset: u64, data: &[u8]) -> Result<()>; + + /// Copies the bytes at `range` in the shared memory `memory` to `offset`, + /// extending the image as needed. + /// + /// Returns `None` if the image cannot read `memory`'s type directly, so + /// the caller must copy the bytes through [`Self::write`] instead. + fn write_from_shared( + &mut self, + _offset: u64, + _memory: &dyn Any, + _range: Range, + ) -> Option> { + None + } + + /// Returns the image so the platform that created it can recover its + /// concrete type when starting the child. + fn as_any(&self) -> &dyn Any; +} + +/// A pending child's memory image. +/// +/// The image holds its share of the broker-wide child image budget until it is +/// dropped. +pub struct ChildImage { + image: Box, + /// Image bytes counted in `budget`. + reserved: u64, + budget: Arc, +} + +impl ChildImage { + fn new(image: Box, budget: Arc) -> Self { + Self { + image, + reserved: 0, + budget, + } + } + + /// Returns the platform image. + #[must_use] + pub fn image(&self) -> &dyn ProcessImage { + &*self.image + } + + /// Counts image bytes up to `end` in the budget, which must stay within + /// `limit`. + fn reserve(&mut self, end: u64, limit: u64) -> Result<()> { + let Some(growth) = end.checked_sub(self.reserved).filter(|growth| *growth > 0) else { + return Ok(()); + }; + self.budget + .try_update(Ordering::Relaxed, Ordering::Relaxed, |reserved| { + reserved.checked_add(growth).filter(|total| *total <= limit) + }) + .map_err(|_| BrokerError::ResourceExhausted)?; + self.reserved = end; + Ok(()) + } +} + +impl Drop for ChildImage { + fn drop(&mut self) { + self.budget + .try_update(Ordering::Relaxed, Ordering::Relaxed, |reserved| { + reserved.checked_sub(self.reserved) + }) + .expect("reserved child image size must include every live image"); + } +} + +/// A pending child taken for startup, with its memory image if one was written. +type ChildWithImage = (Arc, Option); + /// Caller identity information supplied by the broker entry layer. /// /// The first userland proof of concept does not authenticate Unix-socket peers, @@ -159,7 +244,7 @@ struct BrokerProcessState { /// Parent handle readiness published once this process terminates. exit_readiness: Option, /// Child retained until this process requests startup. - pending_child_process: Option>, + pending_child_process: Option, /// Whether a starting child continues after its parent dies. continue_startup_on_parent_death: bool, retirement: ProcessRetirement, @@ -167,6 +252,13 @@ struct BrokerProcessState { shutdown: Option, } +/// A child retained until its parent requests startup. +struct PendingChild { + process: Arc, + /// Memory image the child starts from, once written. + image: Option, +} + /// Broker-visible status of one process. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ProcessStatus { @@ -383,7 +475,10 @@ impl BrokerProcess { Err(BrokerError::Internal) } else { drop(child_state); - state.pending_child_process = Some(Arc::clone(&child)); + state.pending_child_process = Some(PendingChild { + process: Arc::clone(&child), + image: None, + }); Ok(CreatedProcess { identity: ProcessIdentity { process_id: child.id(), @@ -442,22 +537,80 @@ impl BrokerProcess { /// Takes the pending child selected for startup. pub fn take_child_process(&self, child_process_id: ProcessId) -> Result> { + self.take_child_process_with_image(child_process_id) + .map(|(child, _)| child) + } + + /// Takes the pending child selected for startup with its memory image, if + /// one was written. + pub fn take_child_process_with_image( + &self, + child_process_id: ProcessId, + ) -> Result { let mut state = self.state.lock(); - if !self.accepts_operations(&state) { + self.pending_child(&mut state, child_process_id)?; + let PendingChild { process, image } = state + .pending_child_process + .take() + .ok_or(BrokerError::Internal)?; + Ok((process, image)) + } + + /// Lets `write` store `length` bytes at `offset` in the memory image of + /// the pending child selected by `child_process_id`. + /// + /// The first write creates the image with `create`. The image ends no + /// later than the broker's child image size limit, and all child images + /// together stay within the broker's total child image size limit. + pub fn write_child_memory>( + &self, + child_process_id: ProcessId, + offset: u64, + length: u64, + create: impl FnOnce() -> Result>, + write: impl FnOnce(&mut dyn ProcessImage) -> core::result::Result<(), E>, + ) -> core::result::Result<(), E> { + let limits = self.core.limits; + let end = offset + .checked_add(length) + .filter(|end| *end <= limits.max_child_image_size) + .ok_or(BrokerError::ResourceExhausted)?; + // The state lock keeps the child pending while its image is written. + let mut state = self.state.lock(); + let pending = self.pending_child(&mut state, child_process_id)?; + let image = match &mut pending.image { + Some(image) => image, + None => pending.image.insert(ChildImage::new( + create()?, + Arc::clone(&self.core.reserved_child_image_size), + )), + }; + image.reserve(end, limits.max_total_child_image_size)?; + write(&mut *image.image) + } + + /// Returns the pending child selected by `child_process_id` while it + /// awaits startup. + fn pending_child<'a>( + &self, + state: &'a mut BrokerProcessState, + child_process_id: ProcessId, + ) -> Result<&'a mut PendingChild> { + if !self.accepts_operations(state) { return Err(BrokerError::PeerClosed); } - let child = state + let pending = state .pending_child_process - .as_ref() - .filter(|child| child.id() == child_process_id) + .as_mut() + .filter(|pending| pending.process.id() == child_process_id) .ok_or(BrokerError::UnknownObject)?; - if !child.awaits_startup(&child.state.lock()) { + if !pending + .process + .awaits_startup(&pending.process.state.lock()) + { return Err(BrokerError::PeerClosed); } - state - .pending_child_process - .take() - .ok_or(BrokerError::Internal) + Ok(pending) } /// Records the exit of the pending child selected by `child_process_id`. @@ -544,11 +697,12 @@ impl BrokerProcess { if !self.accepts_operations(&state) { return Err(BrokerError::PeerClosed); } - let child = state + let child = &state .pending_child_process .as_ref() - .filter(|child| child.id() == child_process_id) - .ok_or(BrokerError::UnknownObject)?; + .filter(|pending| pending.process.id() == child_process_id) + .ok_or(BrokerError::UnknownObject)? + .process; if !child.awaits_startup(&child.state.lock()) { return Err(BrokerError::PeerClosed); } @@ -780,7 +934,7 @@ impl BrokerProcess { for shutdown in shutdowns { shutdown(); } - if let Some(child) = pending_child_process { + if let Some(PendingChild { process: child, .. }) = pending_child_process { let _ = child.fail_start(BrokerError::PeerClosed, false, true); child.retire(true); } @@ -1574,11 +1728,12 @@ impl Drop for BrokerProcess { #[cfg(test)] mod tests { + use core::any::Any; use core::sync::atomic::{AtomicUsize, Ordering}; use super::{ - BrokerProcess, ProcessLifecycleSink, ProcessReferences, ProcessStatus, - release_pending_reference, + BrokerProcess, ProcessImage, ProcessLifecycleSink, ProcessReferences, ProcessStatus, + Result, release_pending_reference, }; use crate::readiness::ReadinessSink; use crate::stdio::StdioOutputStream; @@ -1595,7 +1750,7 @@ mod tests { use litebox_broker_protocol::process::{CreatedProcess, ProcessExitStatus, ProcessTermination}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::{ObjectHandle, ProcessId}; - use std::{sync::Arc, vec, vec::Vec}; + use std::{boxed::Box, sync::Arc, vec, vec::Vec}; const TEST_MAX_REFERENCES: usize = 4; const TEST_MAX_PIPE_CAPACITY: usize = 8; @@ -2181,6 +2336,152 @@ mod tests { assert!(!broker.processes.read().contains_key(&process_id)); } + #[test] + fn pending_child_memory_image_is_created_once_and_taken_with_child() { + type Writes = Arc)>>>; + struct TestImage(Writes); + + impl ProcessImage for TestImage { + fn write(&mut self, offset: u64, data: &[u8]) -> Result<()> { + self.0.lock().unwrap().push((offset, data.to_vec())); + Ok(()) + } + + fn as_any(&self) -> &dyn Any { + self + } + } + + let broker = TestBrokerCoreBuilder::new( + PolicyEngine::with_unauthenticated_rights(ObjectRights::all()) + .with_process_duplication_enabled(true), + ) + .with_limits(BrokerCoreLimits::DEFAULT.with_child_image_size_limits(8, 8)) + .build() + .unwrap(); + let parent = broker + .allocate_process(CallerCredential::Unauthenticated, None) + .unwrap(); + parent.complete_start().unwrap(); + let process_id = parent + .allocate_child_process(readiness_sink()) + .unwrap() + .identity + .process_id; + let writes = Arc::new(std::sync::Mutex::new(Vec::new())); + let created = AtomicUsize::new(0); + let create = || { + created.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(TestImage(Arc::clone(&writes))) as Box) + }; + let write = |process_id, offset, data: &[u8]| { + parent.write_child_memory(process_id, offset, data.len() as u64, create, |image| { + image.write(offset, data) + }) + }; + + assert_eq!( + write(ProcessId(process_id.0 + 1), 0, &[1]), + Err(BrokerError::UnknownObject) + ); + assert_eq!( + write(process_id, 7, &[1, 2]), + Err(BrokerError::ResourceExhausted) + ); + assert_eq!( + write(process_id, u64::MAX, &[1]), + Err(BrokerError::ResourceExhausted) + ); + assert_eq!(created.load(Ordering::Relaxed), 0); + assert_eq!(write(process_id, 6, &[1, 2]), Ok(())); + assert_eq!(write(process_id, 0, &[3]), Ok(())); + assert_eq!(created.load(Ordering::Relaxed), 1); + assert_eq!(broker.reserved_child_image_size.load(Ordering::Relaxed), 8); + + let (child, image) = parent.take_child_process_with_image(process_id).unwrap(); + assert_eq!(child.id(), process_id); + let image = image.unwrap(); + assert!(image.image().as_any().downcast_ref::().is_some()); + drop(image); + assert_eq!(*writes.lock().unwrap(), [(6, vec![1, 2]), (0, vec![3])]); + assert_eq!(Arc::strong_count(&writes), 1); + assert_eq!(broker.reserved_child_image_size.load(Ordering::Relaxed), 0); + assert_eq!(write(process_id, 0, &[1]), Err(BrokerError::UnknownObject)); + child.complete_start().unwrap(); + } + + #[test] + fn child_images_share_the_broker_image_budget() { + struct TestImage; + + impl ProcessImage for TestImage { + fn write(&mut self, _offset: u64, _data: &[u8]) -> Result<()> { + Ok(()) + } + + fn as_any(&self) -> &dyn Any { + self + } + } + + let broker = TestBrokerCoreBuilder::new( + PolicyEngine::with_unauthenticated_rights(ObjectRights::all()) + .with_process_duplication_enabled(true), + ) + .with_limits(BrokerCoreLimits::DEFAULT.with_child_image_size_limits(8, 12)) + .build() + .unwrap(); + let pending_child = || { + let parent = broker + .allocate_process(CallerCredential::Unauthenticated, None) + .unwrap(); + parent.complete_start().unwrap(); + let child = parent + .allocate_child_process(readiness_sink()) + .unwrap() + .identity + .process_id; + (parent, child) + }; + let write = |parent: &BrokerProcess, child, offset, length| { + parent.write_child_memory::( + child, + offset, + length, + || Ok(Box::new(TestImage)), + |_| Ok(()), + ) + }; + let reserved = || broker.reserved_child_image_size.load(Ordering::Relaxed); + let (first, first_child) = pending_child(); + let (second, second_child) = pending_child(); + + assert_eq!(write(&first, first_child, 0, 8), Ok(())); + // Rewriting bytes the image already holds takes no more of the budget. + assert_eq!(write(&first, first_child, 2, 4), Ok(())); + assert_eq!(write(&second, second_child, 0, 4), Ok(())); + assert_eq!( + write(&second, second_child, 4, 1), + Err(BrokerError::ResourceExhausted) + ); + assert_eq!(reserved(), 12); + + // A taken image keeps its share until it is dropped. + let (child, image) = first.take_child_process_with_image(first_child).unwrap(); + assert_eq!( + write(&second, second_child, 4, 1), + Err(BrokerError::ResourceExhausted) + ); + drop(image); + assert_eq!(reserved(), 4); + assert_eq!(write(&second, second_child, 4, 4), Ok(())); + assert_eq!(reserved(), 8); + child.complete_start().unwrap(); + + second.exit_child_process(second_child, EXITED).unwrap(); + assert_eq!(reserved(), 0); + } + #[test] fn pending_child_exit_after_owner_death_is_rejected() { let broker = TestBrokerCoreBuilder::new( diff --git a/litebox_broker_core/src/socket/tests.rs b/litebox_broker_core/src/socket/tests.rs index 3f23bc2b69..bd9b1f2bd2 100644 --- a/litebox_broker_core/src/socket/tests.rs +++ b/litebox_broker_core/src/socket/tests.rs @@ -1335,6 +1335,7 @@ fn test_broker_with_policy( pending_references: Arc::new(AtomicUsize::new(0)), reserved_pipe_capacity: Arc::new(AtomicUsize::new(0)), reserved_sockets: Arc::new(AtomicUsize::new(0)), + reserved_child_image_size: Arc::new(core::sync::atomic::AtomicU64::new(0)), random_provider: Arc::new(crate::random::TestRandomProvider), socket_provider, timer_provider: Arc::new(crate::timer::UnsupportedTimerProvider), diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index eb677d9756..560a457164 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -21,10 +21,12 @@ extern crate alloc; #[cfg(test)] extern crate std; -use alloc::{sync::Arc, vec::Vec}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; use litebox_broker_core::readiness::ReadinessSink; -use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess, CallerCredential}; +use litebox_broker_core::{ + BrokerCore, BrokerError, BrokerProcess, CallerCredential, ChildImage, ProcessImage, +}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::event::{AddEventResponse, CreateEventResponse}; use litebox_broker_protocol::fs::{ @@ -45,9 +47,9 @@ use litebox_broker_protocol::pipe::{ }; use litebox_broker_protocol::process::{ CreateThreadRequest, CreateThreadResponse, DuplicateObjectsToChildRequest, - ExitChildProcessRequest, MAX_CHILD_OBJECT_DUPLICATES, MAX_PROCESS_BOOTSTRAP_SIZE, - ProcessStartupData, ProcessStartupDescriptor, StartChildProcessRequest, - StartChildProcessSource, + ExitChildProcessRequest, MAX_CHILD_MEMORY_WRITE_SIZE, MAX_CHILD_OBJECT_DUPLICATES, + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessStartupData, ProcessStartupDescriptor, + StartChildProcessRequest, WriteChildMemoryRequest, }; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ @@ -575,7 +577,7 @@ fn handle_request( handle_file_request(process, request, shared_buffers, readiness_sink) .map(BrokerResult::File) } - BrokerOperation::StartChildProcess(_) => { + BrokerOperation::StartChildProcess(_) | BrokerOperation::WriteChildMemory(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) } } @@ -662,7 +664,7 @@ fn handle_file_request( offset, }) => { validate_shared_buffer(buffer, MAX_FILE_TRANSFER_SIZE)?; - let mut data = allocate_zeroed(buffer.length())?; + let mut data = allocate_zeroed(buffer.length() as usize)?; match litebox_broker_core::fs::read(process, handle, &mut data, offset) .map_err(RequestFailure::from)? { @@ -858,11 +860,11 @@ fn validate_shared_buffer(buffer: SharedBufferSequence, max_length: u32) -> Requ Ok(()) } -fn allocate_zeroed(length: u32) -> RequestResult> { +fn allocate_zeroed(length: usize) -> RequestResult> { let mut data = Vec::new(); - data.try_reserve_exact(length as usize) + data.try_reserve_exact(length) .map_err(|_| RequestFailure::Respond(ErrorCode::OutOfMemory))?; - data.resize(length as usize, 0); + data.resize(length, 0); Ok(data) } @@ -873,7 +875,7 @@ pub fn read_shared_buffer( max_length: u32, ) -> RequestResult> { validate_shared_buffer(buffer, max_length)?; - let mut data = allocate_zeroed(buffer.length())?; + let mut data = allocate_zeroed(buffer.length() as usize)?; shared_buffers .read_sequence(buffer, &mut data) .map_err(shared_buffer_access_failure)?; @@ -882,15 +884,26 @@ pub fn read_shared_buffer( /// Platform implementation that starts execution for a broker-created process. /// -/// Once called, the launcher owns final process retirement even when launch -/// fails. Success means process startup reached `Running`. +/// Once called, [`Self::launch`] owns final process retirement even when +/// launch fails. Success means process startup reached `Running`. pub trait ProcessLauncher: Send + Sync { /// Starts one process and waits for startup to commit or fail. + /// + /// The process's runner receives `image`, the memory image its parent + /// wrote, if any. fn launch( self: Arc, process: Arc, startup: ProcessStartupData, + image: Option, ) -> core::result::Result<(), BrokerError>; + + /// Creates an empty memory image for a pending child. + /// + /// Platforms that cannot pass images to runners reject the request. + fn create_image(&self) -> core::result::Result, BrokerError> { + Err(BrokerError::UnsupportedOperation) + } } /// Handles a process operation using the configured platform launcher. @@ -907,7 +920,7 @@ where match operation { BrokerOperation::StartChildProcess(StartChildProcessRequest { child_process_id, - source: StartChildProcessSource::Bootstrap(startup), + startup, }) => Some( read_shared_buffer(shared_buffers, startup.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) .and_then(|payload| { @@ -920,10 +933,69 @@ where }) .map(|()| BrokerResult::ProcessStarted), ), + BrokerOperation::WriteChildMemory(WriteChildMemoryRequest { + child_process_id, + offset, + data, + }) => Some( + write_child_memory( + launcher.as_ref(), + parent, + *child_process_id, + *offset, + *data, + shared_buffers, + ) + .map(|()| BrokerResult::ChildMemoryWritten), + ), _ => None, } } +/// Writes `data` at `offset` in the pending child's memory image, copying +/// straight from shared memory when the image supports it. +fn write_child_memory( + launcher: &Launcher, + parent: &BrokerProcess, + child_process_id: ProcessId, + offset: u64, + data: SharedBufferSequence, + shared_buffers: &SharedBufferPool, +) -> RequestResult<()> +where + Memory: SharedMemory, + Launcher: ProcessLauncher + ?Sized, +{ + validate_shared_buffer(data, MAX_CHILD_MEMORY_WRITE_SIZE)?; + let ranges = shared_buffers + .sequence_ranges(data, data.length() as usize) + .map_err(shared_buffer_access_failure)?; + let memory = shared_buffers.memory(); + parent.write_child_memory( + child_process_id, + offset, + u64::from(data.length()), + || launcher.create_image(), + |image| { + let mut image_offset = offset; + for range in ranges { + let length = range.len(); + if let Some(result) = image.write_from_shared(image_offset, memory, range.clone()) { + result?; + } else { + let mut buffer = allocate_zeroed(length)?; + memory + .read(range.start, &mut buffer) + .map_err(|error| shared_buffer_access_failure(error.into()))?; + image.write(image_offset, &buffer)?; + } + image_offset += length as u64; + } + Ok(()) + }, + ) +} + fn start_child_process( launcher: Arc, parent: &BrokerProcess, @@ -933,8 +1005,8 @@ fn start_child_process( if !parent.is_running() { return Err(RequestFailure::Abort(ErrorCode::ProtocolState)); } - let process = parent - .take_child_process(child_process_id) + let (process, image) = parent + .take_child_process_with_image(child_process_id) .map_err(RequestFailure::from)?; if parent.is_cancellation_requested() { let _ = process.fail_start(BrokerError::PeerClosed, false, true); @@ -947,6 +1019,7 @@ fn start_child_process( ProcessStartupData { payload: startup.payload, }, + image, ) .map_err(RequestFailure::from) } diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index d13bb739e1..e15d7396bf 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -43,9 +43,9 @@ use litebox_broker_protocol::message::{ }; use litebox_broker_protocol::process::{ CreateThreadRequest, CreateThreadResponse, CreatedProcess, DuplicateObjectsToChildRequest, - ExitChildProcessRequest, MAX_CHILD_OBJECT_DUPLICATES, MAX_PROCESS_BOOTSTRAP_SIZE, - ProcessExitStatus, ProcessStartupData, ProcessStartupDescriptor, ProcessTermination, - StartChildProcessRequest, StartChildProcessSource, + ExitChildProcessRequest, MAX_CHILD_MEMORY_WRITE_SIZE, MAX_CHILD_OBJECT_DUPLICATES, + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessExitStatus, ProcessStartupData, ProcessStartupDescriptor, + ProcessTermination, StartChildProcessRequest, WriteChildMemoryRequest, }; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; @@ -211,7 +211,7 @@ impl BrokerLocal { match self.request(BrokerOperation::StartChildProcess( StartChildProcessRequest { child_process_id, - source: StartChildProcessSource::Bootstrap(ProcessStartupDescriptor { buffer }), + startup: ProcessStartupDescriptor { buffer }, }, ))? { BrokerResult::ProcessStarted => Ok(()), @@ -220,6 +220,41 @@ impl BrokerLocal { } } + /// Writes `data` at `offset` of the memory image of a pending child + /// created by [`Self::allocate_child_process`]. + /// + /// The caller must retain exclusive ownership of the data sequence until + /// this method returns. + /// + /// # Panics + /// + /// Panics if the data length differs from the shared-buffer sequence or the + /// broker returns a response for another operation. + pub fn write_child_memory( + &self, + child_process_id: ProcessId, + offset: u64, + buffer: SharedBufferSequence, + data: &[u8], + ) -> Result<(), Channel::Error> { + if buffer.length() > MAX_CHILD_MEMORY_WRITE_SIZE { + return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); + } + + self.write_shared_buffer(buffer, data); + match self.request(BrokerOperation::WriteChildMemory(WriteChildMemoryRequest { + child_process_id, + offset, + data: buffer, + }))? { + BrokerResult::ChildMemoryWritten => Ok(()), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => { + panic!("broker returned unexpected child-memory-write response: {response:?}") + } + } + } + /// Records the exit of a pending child created by /// [`Self::allocate_child_process`] that never started. /// diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index b29ebc87c9..8a92c94b79 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -2,7 +2,7 @@ // Licensed under the MIT license. use std::{ - os::fd::{AsFd, AsRawFd, RawFd}, + os::fd::{AsFd, AsRawFd, OwnedFd, RawFd}, path::Path, sync::{ Arc, Mutex, @@ -37,6 +37,9 @@ pub struct BrokerConnection { pub positional_io_fds: [RawFd; 2], /// Association-shutdown descriptor that a runner's syscall filter must permit. pub shutdown_fd: RawFd, + /// Sealed memfd holding the memory image this process starts from, if + /// its parent wrote one. + pub process_image: Option, } /// Connects to and negotiates an association with a Linux-userland broker. @@ -57,11 +60,12 @@ pub fn connect( ) })?; let association_coordinator = Arc::new(BrokerAssociationFailureCoordinator::new()); - let (local, startup, (notification_channel, positional_io_fds, shutdown_fd)) = + let (local, startup, (notification_channel, positional_io_fds, shutdown_fd, process_image)) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; let control_memory = setup.receive_control_ring(Some(setup_deadline))?; + let process_image = setup.receive_process_image(Some(setup_deadline))?; let positional_io_fds = [ shared_memory.as_fd().as_raw_fd(), control_memory.as_fd().as_raw_fd(), @@ -84,7 +88,12 @@ pub fn connect( Ok(( call_channel, Arc::new(shared_memory), - (notification_channel, positional_io_fds, shutdown_fd), + ( + notification_channel, + positional_io_fds, + shutdown_fd, + process_image, + ), )) }) .context("broker negotiation failed")?; @@ -95,6 +104,7 @@ pub fn connect( coordinator: association_coordinator, positional_io_fds, shutdown_fd, + process_image, }, startup, )) diff --git a/litebox_broker_protocol/src/fs.rs b/litebox_broker_protocol/src/fs.rs index da7bcb7822..c00539bd37 100644 --- a/litebox_broker_protocol/src/fs.rs +++ b/litebox_broker_protocol/src/fs.rs @@ -14,7 +14,7 @@ use crate::ObjectHandle; use crate::shared_buffer::{SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence}; /// Maximum number of shared-buffer slots used by one file read or write. -pub const MAX_FILE_TRANSFER_SLOT_COUNT: u32 = 8; +pub const MAX_FILE_TRANSFER_SLOT_COUNT: u32 = 16; /// Maximum bytes transferred by one logical file read or write. pub const MAX_FILE_TRANSFER_SIZE: u32 = SHARED_BUFFER_SLOT_SIZE * MAX_FILE_TRANSFER_SLOT_COUNT; diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index dcfc3edf2a..c3a8054aba 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -21,7 +21,7 @@ use crate::pipe::{ use crate::process::{ CreateThreadRequest, CreateThreadResponse, DuplicateObjectsToChildRequest, ExitChildProcessRequest, ProcessExitStatus, ProcessStartupDescriptor, ProcessTermination, - StartChildProcessRequest, StartChildProcessSource, + StartChildProcessRequest, WriteChildMemoryRequest, }; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; @@ -95,6 +95,8 @@ pub enum BrokerOperation { SetChildReaping(bool), /// Duplicate object references into this process's pending child. DuplicateObjectsToChild(DuplicateObjectsToChildRequest), + /// Write bytes into this process's pending child's memory image. + WriteChildMemory(WriteChildMemoryRequest), /// Timer object request family. Timer(TimerRequest), /// Signal request family. @@ -130,15 +132,14 @@ impl BrokerOperation { | FileRequest::Rmdir(RmdirFileRequest { path: buffer, .. }), ) | Self::StartChildProcess(StartChildProcessRequest { - source: - StartChildProcessSource::Bootstrap(ProcessStartupDescriptor { buffer, .. }) - | StartChildProcessSource::Duplicate(buffer), + startup: ProcessStartupDescriptor { buffer }, .. }) | Self::DuplicateObjectsToChild(DuplicateObjectsToChildRequest { handles: buffer, .. - }) => Some(*buffer), + }) + | Self::WriteChildMemory(WriteChildMemoryRequest { data: buffer, .. }) => Some(*buffer), Self::CreateThread(_) | Self::ExitThread(_) | Self::CloseObject(_) @@ -336,6 +337,8 @@ pub enum BrokerResult { /// Object references were duplicated into a pending child, whose handles /// replaced the request's handles in its shared buffer. ObjectsDuplicated, + /// Bytes were written into a pending child's memory image. + ChildMemoryWritten, /// Timer object response family. Timer(TimerResponse), /// Signal response family. diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index e0186d1da5..9d9cdf456b 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -3,7 +3,9 @@ use alloc::vec::Vec; -use crate::shared_buffer::{SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence}; +use crate::shared_buffer::{ + MAX_SHARED_BUFFER_SEQUENCE_SLOTS, SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence, +}; use crate::{ObjectHandle, ProcessId, ThreadId}; /// Maximum size of one process bootstrap carried through the broker. @@ -88,23 +90,16 @@ pub enum CreateThreadResponse { Process(CreatedProcess), } -/// Source used to start one child process. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum StartChildProcessSource { - /// Starts a child from an opaque platform bootstrap. - Bootstrap(ProcessStartupDescriptor), - /// Starts a child by duplicating the calling process from an encoded, - /// input-only image whose exact length is the buffer sequence length. - Duplicate(SharedBufferSequence), -} - /// Starts a pending child created earlier. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartChildProcessRequest { /// Pending child to start. pub child_process_id: ProcessId, - /// Process startup source. - pub source: StartChildProcessSource, + /// Opaque platform bootstrap for the child. + /// + /// The child's runner also receives the memory image written by + /// [`WriteChildMemoryRequest`]s, if any. + pub startup: ProcessStartupDescriptor, } /// Maximum number of object references one [`DuplicateObjectsToChildRequest`] @@ -126,6 +121,33 @@ pub struct DuplicateObjectsToChildRequest { pub handles: SharedBufferSequence, } +/// Number of shared-buffer slots one [`WriteChildMemoryRequest`] fills at most. +const MAX_CHILD_MEMORY_WRITE_SLOT_COUNT: u32 = 16; + +const _: () = + assert!(MAX_CHILD_MEMORY_WRITE_SLOT_COUNT as usize <= MAX_SHARED_BUFFER_SEQUENCE_SLOTS); + +/// Maximum number of bytes one [`WriteChildMemoryRequest`] writes. +pub const MAX_CHILD_MEMORY_WRITE_SIZE: u32 = + SHARED_BUFFER_SLOT_SIZE * MAX_CHILD_MEMORY_WRITE_SLOT_COUNT; + +/// Writes bytes into the memory image of the caller's pending child, as a +/// Linux `fork` child starts from a copy of its parent's memory. +/// +/// The image is a byte array, zero wherever it was not written, that the +/// child's runner receives when the child starts. Its layout is opaque to the +/// broker. A pending child has no image until it is first written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WriteChildMemoryRequest { + /// Pending child whose image receives the bytes. + pub child_process_id: ProcessId, + /// Image offset of the first byte. + pub offset: u64, + /// Operation-scoped shared-buffer sequence holding between one and + /// [`MAX_CHILD_MEMORY_WRITE_SIZE`] bytes. + pub data: SharedBufferSequence, +} + /// Records the exit of a pending child that ran without starting its own /// runner, as a Linux `vfork` child does before `execve`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/litebox_broker_protocol/src/shared_buffer.rs b/litebox_broker_protocol/src/shared_buffer.rs index b83b59f912..72195aaf19 100644 --- a/litebox_broker_protocol/src/shared_buffer.rs +++ b/litebox_broker_protocol/src/shared_buffer.rs @@ -20,7 +20,7 @@ pub const SHARED_BUFFER_SLOT_SIZE: u32 = 64 * 1024; pub const SHARED_BUFFER_SLOT_COUNT: u32 = 256; /// Maximum number of slots named by one operation-scoped sequence. -pub const MAX_SHARED_BUFFER_SEQUENCE_SLOTS: usize = 8; +pub const MAX_SHARED_BUFFER_SEQUENCE_SLOTS: usize = 16; const _: () = assert!(MAX_SHARED_BUFFER_SEQUENCE_SLOTS <= SHARED_BUFFER_SLOT_COUNT as usize); @@ -233,7 +233,7 @@ mod tests { #[test] fn larger_slots_do_not_change_existing_transfer_limits() { assert_eq!(crate::pipe::MAX_PIPE_TRANSFER_SIZE, 32 * 1024); - assert_eq!(crate::fs::MAX_FILE_TRANSFER_SIZE, 512 * 1024); + assert_eq!(crate::fs::MAX_FILE_TRANSFER_SIZE, 1024 * 1024); assert_eq!(crate::socket::MAX_SOCKET_TRANSFER_SIZE, 32 * 1024); assert_eq!(crate::socket::MAX_UDP_DATAGRAM_SIZE, 65_507); } diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 358ffe4aeb..ade0745239 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -25,7 +25,7 @@ use crate::message::{ use crate::process::{ CreateThreadRequest, CreateThreadResponse, CreatedProcess, DuplicateObjectsToChildRequest, ExitChildProcessRequest, ProcessExitStatus, ProcessIdentity, ProcessStartupDescriptor, - ProcessTermination, StartChildProcessRequest, StartChildProcessSource, + ProcessTermination, StartChildProcessRequest, WriteChildMemoryRequest, }; use crate::readiness::ReadinessFlags; @@ -59,11 +59,10 @@ const REQUEST_TAG_TIMER: u8 = 17; const REQUEST_TAG_GET_STATUS_FLAGS: u8 = 18; const REQUEST_TAG_SET_STATUS_FLAGS: u8 = 19; const REQUEST_TAG_SIGNAL: u8 = 20; +const REQUEST_TAG_WRITE_CHILD_MEMORY: u8 = 21; const CREATE_THREAD_TAG_THREAD: u8 = 0; const CREATE_THREAD_TAG_PROCESS: u8 = 1; -const START_CHILD_PROCESS_TAG_BOOTSTRAP: u8 = 0; -const START_CHILD_PROCESS_TAG_DUPLICATE: u8 = 1; const PROCESS_EXIT_STATUS_TAG_EXITED: u8 = 0; const PROCESS_EXIT_STATUS_TAG_SIGNALED: u8 = 1; const PROCESS_EXIT_STATUS_TAG_UNKNOWN: u8 = 2; @@ -89,6 +88,7 @@ const RESPONSE_TAG_TIMER: u8 = 17; const RESPONSE_TAG_STATUS_FLAGS: u8 = 18; const RESPONSE_TAG_STATUS_FLAGS_SET: u8 = 19; const RESPONSE_TAG_SIGNAL: u8 = 20; +const RESPONSE_TAG_CHILD_MEMORY_WRITTEN: u8 = 21; // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; @@ -98,7 +98,7 @@ const RESPONSE_TAG_VERSION_MISMATCH: u8 = 255; const NOTIFICATION_TAG_READINESS: u8 = 0; /// Maximum byte length of any encoded active request or response. -pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 67; +pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 99; /// Maximum byte length of any encoded broker notification. pub const MAX_ENCODED_NOTIFICATION_SIZE: usize = 13; @@ -156,7 +156,8 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_SIGNAL + | REQUEST_TAG_WRITE_CHILD_MEMORY => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -238,16 +239,7 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.u8(REQUEST_TAG_START_CHILD_PROCESS); encoder.request_id(request_id); encoder.process_id(request.child_process_id); - match request.source { - StartChildProcessSource::Bootstrap(ProcessStartupDescriptor { buffer }) => { - encoder.u8(START_CHILD_PROCESS_TAG_BOOTSTRAP); - encoder.shared_buffer_sequence(buffer); - } - StartChildProcessSource::Duplicate(buffer) => { - encoder.u8(START_CHILD_PROCESS_TAG_DUPLICATE); - encoder.shared_buffer_sequence(buffer); - } - } + encoder.shared_buffer_sequence(request.startup.buffer); } BrokerOperation::GetProcessExitStatus(handle) => { encoder.u8(REQUEST_TAG_GET_PROCESS_EXIT_STATUS); @@ -282,6 +274,17 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.process_id(child_process_id); encoder.shared_buffer_sequence(handles); } + BrokerOperation::WriteChildMemory(WriteChildMemoryRequest { + child_process_id, + offset, + data, + }) => { + encoder.u8(REQUEST_TAG_WRITE_CHILD_MEMORY); + encoder.request_id(request_id); + encoder.process_id(child_process_id); + encoder.u64(offset); + encoder.shared_buffer_sequence(data); + } BrokerOperation::Timer(request) => { encoder.u8(REQUEST_TAG_TIMER); encoder.request_id(request_id); @@ -320,7 +323,8 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_TIMER | REQUEST_TAG_GET_STATUS_FLAGS | REQUEST_TAG_SET_STATUS_FLAGS - | REQUEST_TAG_SIGNAL => {} + | REQUEST_TAG_SIGNAL + | REQUEST_TAG_WRITE_CHILD_MEMORY => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -343,21 +347,11 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_FILL_RANDOM => BrokerOperation::FillRandom(decoder.shared_buffer_sequence()?), REQUEST_TAG_FILE => BrokerOperation::File(fs::decode_fs_request(&mut decoder)?), REQUEST_TAG_START_CHILD_PROCESS => { - let child_process_id = decoder.process_id()?; - let source = match decoder.u8()? { - START_CHILD_PROCESS_TAG_BOOTSTRAP => { - StartChildProcessSource::Bootstrap(ProcessStartupDescriptor { - buffer: decoder.shared_buffer_sequence()?, - }) - } - START_CHILD_PROCESS_TAG_DUPLICATE => { - StartChildProcessSource::Duplicate(decoder.shared_buffer_sequence()?) - } - _ => return Err(WireError::InvalidTag), - }; BrokerOperation::StartChildProcess(StartChildProcessRequest { - child_process_id, - source, + child_process_id: decoder.process_id()?, + startup: ProcessStartupDescriptor { + buffer: decoder.shared_buffer_sequence()?, + }, }) } REQUEST_TAG_GET_PROCESS_EXIT_STATUS => { @@ -383,6 +377,13 @@ pub fn decode_request(frame: &[u8]) -> Result { handles: decoder.shared_buffer_sequence()?, }) } + REQUEST_TAG_WRITE_CHILD_MEMORY => { + BrokerOperation::WriteChildMemory(WriteChildMemoryRequest { + child_process_id: decoder.process_id()?, + offset: decoder.u64()?, + data: decoder.shared_buffer_sequence()?, + }) + } REQUEST_TAG_TIMER => BrokerOperation::Timer(timer::decode_timer_request(&mut decoder)?), REQUEST_TAG_SIGNAL => BrokerOperation::Signal(signal::decode_signal_request(&mut decoder)?), _ => unreachable!("active request tag was validated"), @@ -469,7 +470,8 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { + | RESPONSE_TAG_SIGNAL + | RESPONSE_TAG_CHILD_MEMORY_WRITTEN => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_VERSION_MISMATCH => BrokerHandshakeResponse::VersionMismatch { @@ -590,6 +592,10 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.u8(RESPONSE_TAG_OBJECTS_DUPLICATED); encoder.request_id(request_id); } + BrokerResult::ChildMemoryWritten => { + encoder.u8(RESPONSE_TAG_CHILD_MEMORY_WRITTEN); + encoder.request_id(request_id); + } BrokerResult::Timer(response) => { encoder.u8(RESPONSE_TAG_TIMER); encoder.request_id(request_id); @@ -636,7 +642,8 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_TIMER | RESPONSE_TAG_STATUS_FLAGS | RESPONSE_TAG_STATUS_FLAGS_SET - | RESPONSE_TAG_SIGNAL => {} + | RESPONSE_TAG_SIGNAL + | RESPONSE_TAG_CHILD_MEMORY_WRITTEN => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -678,6 +685,7 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_EXIT_STATUS_REPORTED => BrokerResult::ExitStatusReported, RESPONSE_TAG_CHILD_REAPING_SET => BrokerResult::ChildReapingSet, RESPONSE_TAG_OBJECTS_DUPLICATED => BrokerResult::ObjectsDuplicated, + RESPONSE_TAG_CHILD_MEMORY_WRITTEN => BrokerResult::ChildMemoryWritten, RESPONSE_TAG_TIMER => BrokerResult::Timer(timer::decode_timer_response(&mut decoder)?), RESPONSE_TAG_SIGNAL => BrokerResult::Signal(signal::decode_signal_response(&mut decoder)?), _ => unreachable!("active response tag was validated"), @@ -808,9 +816,12 @@ mod tests { use crate::process::{ CreateThreadRequest, CreateThreadResponse, CreatedProcess, DuplicateObjectsToChildRequest, ExitChildProcessRequest, ProcessExitStatus, ProcessIdentity, ProcessStartupDescriptor, - ProcessTermination, StartChildProcessRequest, StartChildProcessSource, + ProcessTermination, StartChildProcessRequest, WriteChildMemoryRequest, + }; + use crate::shared_buffer::{ + MAX_SHARED_BUFFER_SEQUENCE_SLOTS, SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence, + SharedBufferSlotIndex, }; - use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::signal::{OpenSignalsResponse, PendingSignal, SendSignalRequest, TakeSignalRequest}; use crate::socket::{ AcceptSocketRequest, AcceptSocketResponse, AddressFamily, BindSocketRequest, @@ -877,6 +888,7 @@ mod tests { RESPONSE_TAG_STATUS_FLAGS, RESPONSE_TAG_STATUS_FLAGS_SET, RESPONSE_TAG_SIGNAL, + RESPONSE_TAG_CHILD_MEMORY_WRITTEN, ], [ REQUEST_TAG_NEGOTIATE, @@ -899,6 +911,7 @@ mod tests { REQUEST_TAG_GET_STATUS_FLAGS, REQUEST_TAG_SET_STATUS_FLAGS, REQUEST_TAG_SIGNAL, + REQUEST_TAG_WRITE_CHILD_MEMORY, ] ); assert_eq!( @@ -930,18 +943,13 @@ mod tests { let handle = ObjectHandle(13); // The wire bound covers encodable requests before operation-specific // transfer limits are validated. + let largest_slots: [SharedBufferSlotIndex; MAX_SHARED_BUFFER_SEQUENCE_SLOTS] = + core::array::from_fn(|index| { + SharedBufferSlotIndex(u32::try_from(2 * index + 1).unwrap()) + }); let largest_sequence = SharedBufferSequence::new( - &[ - SharedBufferSlotIndex(2), - SharedBufferSlotIndex(5), - SharedBufferSlotIndex(7), - SharedBufferSlotIndex(9), - SharedBufferSlotIndex(10), - SharedBufferSlotIndex(11), - SharedBufferSlotIndex(13), - SharedBufferSlotIndex(15), - ], - 512 * 1024, + &largest_slots, + u32::try_from(MAX_SHARED_BUFFER_SEQUENCE_SLOTS).unwrap() * SHARED_BUFFER_SLOT_SIZE, ) .unwrap(); let operations = [ @@ -1174,13 +1182,9 @@ mod tests { BrokerOperation::Socket(SocketRequest::Status(SocketStatusRequest { handle })), BrokerOperation::StartChildProcess(StartChildProcessRequest { child_process_id: process_id(u32::MAX), - source: StartChildProcessSource::Bootstrap(ProcessStartupDescriptor { + startup: ProcessStartupDescriptor { buffer: largest_sequence, - }), - }), - BrokerOperation::StartChildProcess(StartChildProcessRequest { - child_process_id: process_id(1), - source: StartChildProcessSource::Duplicate(sequence(0, 2)), + }, }), BrokerOperation::GetProcessExitStatus(ObjectHandle(u64::MAX)), BrokerOperation::ExitChildProcess(ExitChildProcessRequest { @@ -1204,6 +1208,11 @@ mod tests { child_process_id: process_id(u32::MAX), handles: largest_sequence, }), + BrokerOperation::WriteChildMemory(WriteChildMemoryRequest { + child_process_id: process_id(u32::MAX), + offset: u64::MAX, + data: largest_sequence, + }), ]; let mut maximum_encoded_size = 0; @@ -1588,6 +1597,7 @@ mod tests { BrokerResult::ExitStatusReported, BrokerResult::ChildReapingSet, BrokerResult::ObjectsDuplicated, + BrokerResult::ChildMemoryWritten, BrokerResult::Error(ErrorCode::PolicyDenied), BrokerResult::Error(ErrorCode::WouldBlock), BrokerResult::Error(ErrorCode::NonBlockingWouldBlock), @@ -1795,18 +1805,6 @@ mod tests { decode_request(&unknown_create_thread), Err(WireError::InvalidTag) ); - let mut unknown_child_start = encode_request(BrokerRequest { - request_id: TEST_REQUEST_ID, - operation: BrokerOperation::StartChildProcess(StartChildProcessRequest { - child_process_id: process_id(1), - source: StartChildProcessSource::Duplicate(sequence(0, 2)), - }), - }); - unknown_child_start[13] = 0xff; - assert_eq!( - decode_request(&unknown_child_start), - Err(WireError::InvalidTag) - ); let mut unknown_exit_status = encode_request(BrokerRequest { request_id: TEST_REQUEST_ID, operation: BrokerOperation::ExitChildProcess(ExitChildProcessRequest { diff --git a/litebox_broker_transport/src/shared_memory.rs b/litebox_broker_transport/src/shared_memory.rs index 8e08e7ba2e..ed116125bc 100644 --- a/litebox_broker_transport/src/shared_memory.rs +++ b/litebox_broker_transport/src/shared_memory.rs @@ -9,11 +9,13 @@ //! access. use alloc::sync::Arc; +use core::ops::Range; use thiserror::Error; use litebox_broker_protocol::shared_buffer::{ - SharedBufferLayout, SharedBufferLayoutError, SharedBufferSequence, SharedBufferSlotIndex, + MAX_SHARED_BUFFER_SEQUENCE_SLOTS, SharedBufferLayout, SharedBufferLayoutError, + SharedBufferSequence, SharedBufferSlotIndex, }; /// Error accessing a shared-memory resource. @@ -200,29 +202,57 @@ impl SharedBufferPool { Ok(()) } - /// Copies a sequence prefix into `destination`. - pub fn read_sequence( + /// Returns the shared-memory ranges holding the first `length` bytes of + /// `sequence`, in transfer order. + /// + /// Slots adjacent in memory share one range. + pub fn sequence_ranges( &self, sequence: SharedBufferSequence, - destination: &mut [u8], - ) -> Result<(), SharedBufferError> { + length: usize, + ) -> Result> + use, SharedBufferError> { let descriptors = sequence.descriptors(self.layout)?; - if destination.len() > sequence.length() as usize { + if length > sequence.length() as usize { return Err(SharedBufferError::TransferExceedsSequence); } - let mut offset = 0; + let mut ranges: [Range; MAX_SHARED_BUFFER_SEQUENCE_SLOTS] = + core::array::from_fn(|_| 0..0); + let mut count = 0_usize; + let mut remaining = length; for descriptor in descriptors { - if offset == destination.len() { + if remaining == 0 { break; } - let length = (destination.len() - offset).min(descriptor.length as usize); - let end = offset + length; - self.read(descriptor.slot_index, &mut destination[offset..end])?; - offset = end; + let slot_length = remaining.min(descriptor.length as usize); + let range = self.layout.range(descriptor.slot_index, slot_length)?; + remaining -= slot_length; + match count.checked_sub(1).map(|last| &mut ranges[last]) { + Some(last) if last.end == range.start => last.end = range.end, + _ => { + ranges[count] = range; + count += 1; + } + } } - if offset != destination.len() { + if remaining != 0 { return Err(SharedBufferError::TransferExceedsSequence); } + Ok(ranges.into_iter().take(count)) + } + + /// Copies a sequence prefix into `destination`. + pub fn read_sequence( + &self, + sequence: SharedBufferSequence, + destination: &mut [u8], + ) -> Result<(), SharedBufferError> { + let mut offset = 0; + for range in self.sequence_ranges(sequence, destination.len())? { + let end = offset + range.len(); + self.memory + .read(range.start, &mut destination[offset..end])?; + offset = end; + } Ok(()) } @@ -232,23 +262,12 @@ impl SharedBufferPool { sequence: SharedBufferSequence, source: &[u8], ) -> Result<(), SharedBufferError> { - let descriptors = sequence.descriptors(self.layout)?; - if source.len() > sequence.length() as usize { - return Err(SharedBufferError::TransferExceedsSequence); - } let mut offset = 0; - for descriptor in descriptors { - if offset == source.len() { - break; - } - let length = (source.len() - offset).min(descriptor.length as usize); - let end = offset + length; - self.write(descriptor.slot_index, &source[offset..end])?; + for range in self.sequence_ranges(sequence, source.len())? { + let end = offset + range.len(); + self.memory.write(range.start, &source[offset..end])?; offset = end; } - if offset != source.len() { - return Err(SharedBufferError::TransferExceedsSequence); - } Ok(()) } } @@ -309,6 +328,32 @@ mod tests { ); } + #[test] + fn pool_merges_slots_adjacent_in_memory() { + let layout = SharedBufferLayout::new(4, 4).unwrap(); + let pool = + SharedBufferPool::new(TestSharedMemory::new(layout.total_len()), layout).unwrap(); + let sequence = SharedBufferSequence::new( + &[ + SharedBufferSlotIndex(1), + SharedBufferSlotIndex(2), + SharedBufferSlotIndex(0), + SharedBufferSlotIndex(3), + ], + 14, + ) + .unwrap(); + + let ranges = |length| { + pool.sequence_ranges(sequence, length) + .unwrap() + .collect::>() + }; + assert_eq!(ranges(14), [4..12, 0..4, 12..14]); + assert_eq!(ranges(6), core::iter::once(4..10).collect::>()); + assert_eq!(ranges(0), []); + } + struct TestSharedMemory(Mutex>); impl TestSharedMemory { diff --git a/litebox_broker_transport_linux_userland/src/memfd.rs b/litebox_broker_transport_linux_userland/src/memfd.rs index 8a1f29bb7b..881904ca0e 100644 --- a/litebox_broker_transport_linux_userland/src/memfd.rs +++ b/litebox_broker_transport_linux_userland/src/memfd.rs @@ -10,12 +10,14 @@ //! //! Rust never dereferences the peer-writable mapping. Byte and word access uses //! positional descriptor I/O into private buffers; the mapping exists only to -//! provide checked addresses to the kernel's futex operations. +//! provide checked addresses to the kernel's futex operations and to process +//! image writes. use std::io::{Error, Result as IoResult}; use std::io::{ErrorKind, IoSlice, IoSliceMut}; use std::mem::{align_of, size_of}; -use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; +use std::ops::Range; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::ptr::NonNull; use std::time::Instant; @@ -46,6 +48,8 @@ use crate::unix_io::{ const REQUIRED_MEMFD_SEALS: SealFlags = SealFlags::from_bits_retain( SealFlags::GROW.bits() | SealFlags::SHRINK.bits() | SealFlags::SEAL.bits(), ); +const PROCESS_IMAGE_SEALS: SealFlags = + SealFlags::from_bits_retain(REQUIRED_MEMFD_SEALS.bits() | SealFlags::WRITE.bits()); /// Linux memfd-backed shared memory usable by broker transports. pub struct MemfdSharedMemory { fd: OwnedFd, @@ -478,17 +482,163 @@ pub fn receive_control_ring_memfd( MemfdSharedMemory::control_ring_from_received_fd(fd) } +/// Sparse memfd holding a process memory image for a runner being started. +/// +/// The broker writes the image, then [`send_process_image`] seals it against +/// all changes before passing it to the runner. +pub struct MemfdProcessImage { + fd: OwnedFd, +} + +impl MemfdProcessImage { + /// Creates an empty image. + pub fn create() -> IoResult { + let fd = memfd_create( + "litebox-process-image", + MemfdFlags::CLOEXEC | MemfdFlags::ALLOW_SEALING, + )?; + Ok(Self { fd }) + } + + /// Copies the bytes at `range` in `memory` to `offset`, extending the image + /// as needed. + /// + /// The kernel copies straight from the shared mapping, so a peer writing + /// the range concurrently changes only the copied bytes. + pub fn write_from_shared( + &self, + mut offset: u64, + memory: &MemfdSharedMemory, + range: Range, + ) -> IoResult<()> { + let mut source = shared_address(&memory.mapping, range.start, range.len(), 1) + .map_err(MemfdSharedMemory::wait_access_error)?; + if !memory.policy.permits_byte_range(range.start, range.len()) { + return Err(MemfdSharedMemory::wait_access_error( + SharedMemoryError::InvalidRange, + )); + } + let mut remaining = range.len(); + while remaining > 0 { + let file_offset = libc::off_t::try_from(offset) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "process image is too large"))?; + // SAFETY: `source..source + remaining` lies within the live shared + // mapping, which `memory` keeps mapped. The kernel copies from it + // without Rust forming references to peer-writable memory. + let written = + unsafe { libc::pwrite(self.fd.as_raw_fd(), source.cast(), remaining, file_offset) }; + match written { + 0 => { + return Err(Error::new( + ErrorKind::WriteZero, + "failed to write process image", + )); + } + 1.. => { + let written = written.cast_unsigned(); + source = source.wrapping_add(written); + remaining -= written; + offset += written as u64; + } + _ => { + let error = Error::last_os_error(); + if error.kind() != ErrorKind::Interrupted { + return Err(error); + } + } + } + } + Ok(()) + } + + /// Writes `data` at `offset`, extending the image as needed. + pub fn write(&self, mut offset: u64, mut data: &[u8]) -> IoResult<()> { + while !data.is_empty() { + match pwrite(&self.fd, data, offset) { + Ok(0) => { + return Err(Error::new( + ErrorKind::WriteZero, + "failed to write process image", + )); + } + Ok(written) => { + data = &data[written..]; + offset += written as u64; + } + Err(Errno::INTR) => {} + Err(error) => return Err(error.into()), + } + } + Ok(()) + } +} + +/// Seals `image`, if any, and sends it over an exclusively owned connected +/// Unix stream. +/// +/// The peer must call [`receive_process_image`] at the same setup step, even +/// when there is no image. +pub fn send_process_image( + stream: &mut UnixStream, + image: Option<&MemfdProcessImage>, + deadline: Option, +) -> IoResult<()> { + let fd = match image { + Some(image) => { + fcntl_add_seals(&image.fd, PROCESS_IMAGE_SEALS)?; + Some(image.fd.as_fd()) + } + None => None, + }; + with_write_deadline(stream, deadline, |stream, deadline| { + send_frame(stream, fd, deadline) + }) +} + +/// Receives the optional process image sent by [`send_process_image`]. +/// +/// A received image is sealed against all changes, so its size and contents +/// stay as validated. +pub fn receive_process_image( + stream: &mut UnixStream, + deadline: Option, +) -> IoResult> { + let (carrier, fd) = with_read_deadline(stream, deadline, receive_frame)?; + match (carrier, fd) { + (0, None) => Ok(None), + (1, Some(fd)) => { + if !fcntl_get_seals(&fd)?.contains(PROCESS_IMAGE_SEALS) { + return Err(invalid_data("process image is not sealed")); + } + Ok(Some(fd)) + } + _ => Err(invalid_data("invalid process image setup data")), + } +} + fn send_fd(stream: &mut UnixStream, fd: BorrowedFd<'_>, deadline: Option) -> IoResult<()> { + send_frame(stream, Some(fd), deadline) +} + +/// Sends one setup frame: a carrier byte that is 1 when `fd` is attached and +/// 0 otherwise. +fn send_frame( + stream: &mut UnixStream, + fd: Option>, + deadline: Option, +) -> IoResult<()> { // Unix streams require an ordinary data byte to carry ancillary data. - let carrier = [0]; + let carrier = [u8::from(fd.is_some())]; let io = [IoSlice::new(&carrier)]; - let fds = [fd]; + let fds = fd.as_slice(); let mut control_space = [std::mem::MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(1))]; let mut control = SendAncillaryBuffer::new(&mut control_space); - assert!( - control.push(SendAncillaryMessage::ScmRights(&fds)), - "SCM_RIGHTS control buffer is correctly sized" - ); + if !fds.is_empty() { + assert!( + control.push(SendAncillaryMessage::ScmRights(fds)), + "SCM_RIGHTS control buffer is correctly sized" + ); + } loop { refresh_write_deadline(stream, deadline)?; match rustix::net::sendmsg(stream.as_fd(), &io, &mut control, SendFlags::NOSIGNAL) { @@ -507,6 +657,19 @@ fn send_fd(stream: &mut UnixStream, fd: BorrowedFd<'_>, deadline: Option) -> IoResult { + match receive_frame(stream, deadline)? { + (_, Some(fd)) => Ok(fd), + (_, None) => Err(invalid_data( + "shared-memory setup contained invalid descriptor data", + )), + } +} + +/// Receives one setup frame's carrier byte and at most one descriptor. +fn receive_frame( + stream: &mut UnixStream, + deadline: Option, +) -> IoResult<(u8, Option)> { let mut carrier = [0]; let mut io = [IoSliceMut::new(&mut carrier)]; let mut control_space = [std::mem::MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(4))]; @@ -545,15 +708,13 @@ fn receive_fd(stream: &mut UnixStream, deadline: Option) -> IoResult 1 { return Err(invalid_data( "shared-memory setup contained invalid descriptor data", )); } - Ok(received_fds - .pop() - .expect("exactly one received descriptor was validated")) + Ok((carrier[0], received_fds.pop())) } impl Drop for MappedRegion { @@ -1045,6 +1206,76 @@ mod tests { assert_eq!(sender.write_timeout().unwrap(), previous_timeout); } + #[test] + fn process_image_copies_from_shared_memory() { + let memory = MemfdSharedMemory::create(16).unwrap(); + memory.write(4, b"shared").unwrap(); + let image = MemfdProcessImage::create().unwrap(); + + image.write_from_shared(0x1000, &memory, 4..10).unwrap(); + assert_eq!( + image + .write_from_shared(0, &memory, 12..17) + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + let control = MemfdSharedMemory::create_control_ring().unwrap(); + assert_eq!( + image + .write_from_shared(0, &control, 0..1) + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + + assert_eq!(fstat(&image.fd).unwrap().st_size, 0x1006); + let mut bytes = [0; 6]; + assert_eq!(pread(&image.fd, &mut bytes, 0x1000).unwrap(), 6); + assert_eq!(&bytes, b"shared"); + } + + #[test] + fn rejects_mismatched_and_unsealed_process_images() { + let image = MemfdProcessImage::create().unwrap(); + + let (mut receiver, mut sender) = UnixStream::pair().unwrap(); + sender.write_all(&[1]).unwrap(); + assert_eq!( + receive_process_image(&mut receiver, None) + .unwrap_err() + .kind(), + ErrorKind::InvalidData + ); + + let (mut receiver, mut sender) = UnixStream::pair().unwrap(); + sender.write_all(&[2]).unwrap(); + assert_eq!( + receive_process_image(&mut receiver, None) + .unwrap_err() + .kind(), + ErrorKind::InvalidData + ); + + let (mut receiver, mut sender) = UnixStream::pair().unwrap(); + send_test_fds(&mut sender, &[image.fd.as_fd()]); + assert_eq!( + receive_process_image(&mut receiver, None) + .unwrap_err() + .kind(), + ErrorKind::InvalidData + ); + + let (mut receiver, mut sender) = UnixStream::pair().unwrap(); + send_frame(&mut sender, Some(image.fd.as_fd()), None).unwrap(); + assert_eq!( + receive_process_image(&mut receiver, None) + .unwrap_err() + .kind(), + ErrorKind::InvalidData + ); + } + fn send_test_fds(stream: &mut UnixStream, fds: &[BorrowedFd<'_>]) { let carrier = [0]; let io = [IoSlice::new(&carrier)]; diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 671386f1f5..20c6764883 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -32,7 +32,7 @@ use litebox_broker_transport::control_ring::{ ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, }; -use crate::memfd::MemfdSharedMemory; +use crate::memfd::{MemfdProcessImage, MemfdSharedMemory}; use crate::setup::{ copy_io_error, invalid_data, read_setup_frame, ring_error, shutdown_socket, wire_error, write_setup_frame, @@ -171,6 +171,16 @@ impl UnixStreamHostSetupChannel { crate::memfd::send_memfd(&mut self.stream, shared_memory, deadline) } + /// Sends the optional process image during association setup, after the + /// control ring. + pub fn send_process_image( + &mut self, + image: Option<&MemfdProcessImage>, + deadline: Option, + ) -> IoResult<()> { + crate::memfd::send_process_image(&mut self.stream, image, deadline) + } + /// Consumes a negotiated setup channel into independently usable active /// request, response, notification, and shutdown handles. pub fn into_active( diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index d4b48f296a..78caf1e076 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -129,6 +129,15 @@ impl UnixStreamLocalSetupChannel { crate::memfd::receive_control_ring_memfd(&mut self.stream, deadline) } + /// Receives the optional process image offered by the broker during + /// setup, after the control ring. + pub fn receive_process_image( + &mut self, + deadline: Option, + ) -> IoResult> { + crate::memfd::receive_process_image(&mut self.stream, deadline) + } + /// Consumes a negotiated setup channel into independently usable active /// call, notification, and shutdown handles, starting the response /// dispatcher and liveness monitor. diff --git a/litebox_broker_userland/Cargo.toml b/litebox_broker_userland/Cargo.toml index 6855ddec83..f197d68daa 100644 --- a/litebox_broker_userland/Cargo.toml +++ b/litebox_broker_userland/Cargo.toml @@ -13,6 +13,7 @@ litebox_broker_transport = { path = "../litebox_broker_transport", version = "0. litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(target_os = "linux")'.dependencies] +libc = { version = "0.2.177", default-features = false } litebox_broker_platform_linux_userland = { path = "../litebox_broker_platform_linux_userland", version = "0.1.0" } litebox_broker_transport_linux_userland = { path = "../litebox_broker_transport_linux_userland", version = "0.1.0" } litebox_platform_linux_userland = { path = "../litebox_platform_linux_userland", version = "0.1.0" } diff --git a/litebox_broker_userland/src/linux.rs b/litebox_broker_userland/src/linux.rs index ff85f033c9..770aedfcee 100644 --- a/litebox_broker_userland/src/linux.rs +++ b/litebox_broker_userland/src/linux.rs @@ -239,7 +239,7 @@ fn serve_control_stream( |channel, shared_memory, control_memory| { channel.send_memfd(shared_memory, Some(setup_deadline))?; channel.send_memfd(control_memory, Some(setup_deadline))?; - Ok(()) + channel.send_process_image(None, Some(setup_deadline)) }, UnixStreamHostSetupChannel::into_active, ) diff --git a/litebox_broker_userland/src/main.rs b/litebox_broker_userland/src/main.rs index b1a58b1c77..a00b15ea7b 100644 --- a/litebox_broker_userland/src/main.rs +++ b/litebox_broker_userland/src/main.rs @@ -222,6 +222,10 @@ fn run_runner_instance( if let Some(proxy_url) = proxy_url { config = config.with_proxy_url(proxy_url.to_owned()); } + #[cfg(target_os = "linux")] + if args.allow_process_duplication { + config = config.with_aslr_disabled(); + } let runner_status = litebox_broker_userland::runner::run_to_completion(config, broker)?; if !runner_status.success() { return Err(IoError::other(format!("runner exited with {runner_status}")).into()); diff --git a/litebox_broker_userland/src/process_launcher.rs b/litebox_broker_userland/src/process_launcher.rs index 041e7db931..2e5401e5e7 100644 --- a/litebox_broker_userland/src/process_launcher.rs +++ b/litebox_broker_userland/src/process_launcher.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Condvar, Mutex}; use std::time::Instant; use litebox_broker_core::{ - BrokerCore, BrokerError, BrokerProcess, CallerCredential, ProcessLifecycleSink, + BrokerCore, BrokerError, BrokerProcess, CallerCredential, ChildImage, ProcessLifecycleSink, }; use litebox_broker_host::ProcessLauncher; use litebox_broker_protocol::process::{ProcessExitStatus, ProcessStartupData}; @@ -28,11 +28,25 @@ pub(crate) struct UserlandProcessLauncher { pub(crate) struct PendingRunnerAssociation { pub(super) process: Arc, data: Option, + image: Option, } impl PendingRunnerAssociation { - fn new(process: Arc, data: Option) -> Self { - Self { process, data } + fn new( + process: Arc, + data: Option, + image: Option, + ) -> Self { + Self { + process, + data, + image, + } + } + + /// Takes the memory image to pass to the runner during setup. + pub(crate) fn take_image(&mut self) -> Option { + self.image.take() } pub(crate) fn into_process_and_startup( @@ -119,7 +133,7 @@ impl UserlandProcessLauncher { .broker .create_process(CallerCredential::HostGuaranteed, None) .map_err(broker_io_error)?; - let association = PendingRunnerAssociation::new(Arc::clone(&process), None); + let association = PendingRunnerAssociation::new(Arc::clone(&process), None, None); let (completion_sender, completion_receiver) = sync_channel(1); let startup = Arc::clone(&launcher).launch_runner(association, config, Some(completion_sender)); @@ -199,14 +213,20 @@ impl ProcessLauncher for UserlandProcessLauncher { self: Arc, process: Arc, data: ProcessStartupData, + image: Option, ) -> Result<(), BrokerError> { let config = self.started_runner_config.clone(); self.launch_runner( - PendingRunnerAssociation::new(process, Some(data)), + PendingRunnerAssociation::new(process, Some(data), image), config, None, ) } + + #[cfg(target_os = "linux")] + fn create_image(&self) -> Result, BrokerError> { + crate::runner::create_image() + } } fn broker_io_error(error: BrokerError) -> IoError { diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index af2acde91f..9f05c1d485 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -24,12 +24,20 @@ mod windows; use crate::process_launcher::{PendingRunnerAssociation, UserlandProcessLauncher}; #[cfg(target_os = "linux")] use linux::PlatformRunnerEndpoint; +#[cfg(target_os = "linux")] +pub(crate) use linux::create_image; +#[cfg(target_os = "linux")] +use linux::wait_for_runner_event; #[cfg(all(windows, target_arch = "x86_64"))] use windows::PlatformRunnerEndpoint; +#[cfg(all(windows, target_arch = "x86_64"))] +use windows::wait_for_runner_event; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); const PROCESS_EXIT_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); -const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); +/// How often runner connections and exits are rechecked where readiness waits +/// are unavailable (Windows, and Linux kernels without `pidfd_open`). +const RUNNER_POLL_INTERVAL: Duration = Duration::from_millis(1); /// Configuration for starting one out-of-process runner. /// @@ -40,6 +48,8 @@ pub struct RunnerConfig { executable: PathBuf, arguments: Vec, proxy_url: Option, + #[cfg(target_os = "linux")] + disable_aslr: bool, } impl RunnerConfig { @@ -51,9 +61,22 @@ impl RunnerConfig { executable, arguments, proxy_url: None, + #[cfg(target_os = "linux")] + disable_aslr: false, } } + /// Starts runners with address-space layout randomization disabled. + /// + /// Runners started from a parent's memory image rely on every runner + /// sharing the same host layout. + #[cfg(target_os = "linux")] + #[must_use] + pub fn with_aslr_disabled(mut self) -> Self { + self.disable_aslr = true; + self + } + /// Configures the HTTP proxy URL passed to the runner. #[must_use] pub fn with_proxy_url(mut self, proxy_url: String) -> Self { @@ -80,6 +103,8 @@ impl RunnerConfig { executable: self.executable.clone(), arguments: Vec::new(), proxy_url: self.proxy_url.clone(), + #[cfg(target_os = "linux")] + disable_aslr: self.disable_aslr, } } } @@ -226,7 +251,7 @@ impl RunnerShutdown { if remaining.is_zero() { return Ok(false); } - std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); + wait_for_runner_event(&self.runner, None, Some(remaining))?; } } } @@ -236,11 +261,13 @@ impl RunnerInstance { pub(crate) fn start(config: RunnerConfig) -> IoResult { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let endpoint = PlatformRunnerEndpoint::create()?; - let runner = Arc::new(Mutex::new( - Command::new(&config.executable) - .args(config.arguments(endpoint.control_channel())) - .spawn()?, - )); + let mut command = Command::new(&config.executable); + command.args(config.arguments(endpoint.control_channel())); + #[cfg(target_os = "linux")] + if config.disable_aslr { + linux::disable_aslr(&mut command); + } + let runner = Arc::new(Mutex::new(command.spawn()?)); let shutdown = Arc::new(RunnerShutdown { runner: Arc::clone(&runner), state: Mutex::new(RunnerShutdownState::Active), @@ -393,6 +420,7 @@ fn accept_runner_channel( channel_name: &'static str, mut runner_status: impl FnMut() -> IoResult>, mut try_accept: impl FnMut() -> IoResult, + mut wait: impl FnMut(Duration) -> IoResult<()>, ) -> IoResult { loop { if let Some(status) = runner_status()? { @@ -413,7 +441,7 @@ fn accept_runner_channel( Err(error) if error.kind() == ErrorKind::WouldBlock => {} Err(error) => return Err(error), } - std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); + wait(remaining)?; } } @@ -436,7 +464,7 @@ fn wait_for_runner_exit(runner: &Arc>) -> IoResult { { return Ok(status); } - std::thread::sleep(ACCEPT_RETRY_DELAY); + wait_for_runner_event(runner, None, None)?; } } diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index b61635b6b3..6e651c9f38 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -1,26 +1,86 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +use std::any::Any; use std::ffi::OsStr; use std::io::{Error as IoError, ErrorKind, Result as IoResult}; +use std::ops::Range; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::os::unix::net::UnixListener; +use std::os::unix::process::CommandExt; use std::path::PathBuf; -use std::process::Child; +use std::process::{Child, Command}; use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::time::{Duration, Instant}; -use litebox_broker_core::BrokerCore; +use litebox_broker_core::{BrokerCore, BrokerError, ProcessImage}; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; -use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; +use litebox_broker_transport_linux_userland::memfd::{MemfdProcessImage, MemfdSharedMemory}; use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; use super::{ - PendingRunnerAssociation, UserlandProcessLauncher, accept_runner_channel, runner_has_exited, + PendingRunnerAssociation, RUNNER_POLL_INTERVAL, UserlandProcessLauncher, accept_runner_channel, + runner_has_exited, }; use crate::runtime::{AssociationOutcome, is_peer_closed_error}; +/// Memory image passed to a runner as a sealed memfd. +struct RunnerProcessImage(MemfdProcessImage); + +impl ProcessImage for RunnerProcessImage { + fn write(&mut self, offset: u64, data: &[u8]) -> Result<(), BrokerError> { + self.0 + .write(offset, data) + .map_err(|_| BrokerError::OutOfMemory) + } + + fn write_from_shared( + &mut self, + offset: u64, + memory: &dyn Any, + range: Range, + ) -> Option> { + let memory = memory.downcast_ref::()?; + Some( + self.0 + .write_from_shared(offset, memory, range) + .map_err(|_| BrokerError::OutOfMemory), + ) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub(crate) fn create_image() -> Result, BrokerError> { + MemfdProcessImage::create() + .map(|image| Box::new(RunnerProcessImage(image)) as Box) + .map_err(|_| BrokerError::OutOfMemory) +} + +/// Makes `command` start its process with address-space layout randomization +/// disabled. +pub(super) fn disable_aslr(command: &mut Command) { + // SAFETY: The hook runs in the forked child before exec and only makes + // async-signal-safe `personality` system calls. + unsafe { + command.pre_exec(|| { + let persona = libc::personality(0xffff_ffff); + if persona == -1 + || libc::personality( + (persona as libc::c_ulong) | libc::ADDR_NO_RANDOMIZE as libc::c_ulong, + ) == -1 + { + return Err(IoError::last_os_error()); + } + Ok(()) + }); + } +} + pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, listener: Option, @@ -75,11 +135,12 @@ impl PlatformRunnerEndpoint { fn serve_association( control_listener: &UnixListener, runner: &Arc>, - startup: PendingRunnerAssociation, + mut startup: PendingRunnerAssociation, setup_deadline: Instant, broker: BrokerCore, launcher: Arc, ) -> AssociationOutcome { + let image = startup.take_image(); let shutdown_was_expected = startup.process.shutdown_was_expected(); let control_channel = match accept_control_channel(control_listener, runner, setup_deadline) { Ok(connection) => connection, @@ -98,10 +159,20 @@ fn serve_association( control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), MemfdSharedMemory::create_control_ring, - |channel, shared_memory, control_memory| { + // Moving `image` into this one-shot closure releases the broker's + // snapshot as soon as setup ends; the runner owns its copy after that. + move |channel, shared_memory, control_memory| { channel.send_memfd(shared_memory, Some(setup_deadline))?; channel.send_memfd(control_memory, Some(setup_deadline))?; - Ok(()) + let image = image.as_ref().map(|image| { + &image + .image() + .as_any() + .downcast_ref::() + .expect("the userland launcher creates every process image") + .0 + }); + channel.send_process_image(image, Some(setup_deadline)) }, UnixStreamHostSetupChannel::into_active, launcher, @@ -118,6 +189,7 @@ fn accept_control_channel( "control", || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), || control_listener.accept().map(|(stream, _)| stream), + |remaining| wait_for_runner_event(runner, Some(control_listener), Some(remaining)), )?; { let mut runner = runner.lock().expect("runner process mutex poisoned"); @@ -134,3 +206,57 @@ fn accept_control_channel( setup_deadline, )) } + +/// Waits until the runner exits, `listener` becomes readable, or `timeout` +/// elapses. Callers recheck their own condition, so early returns are harmless. +pub(super) fn wait_for_runner_event( + runner: &Arc>, + listener: Option<&UnixListener>, + timeout: Option, +) -> IoResult<()> { + let pidfd = { + let mut runner = runner.lock().expect("runner process mutex poisoned"); + if runner.try_wait()?.is_some() { + return Ok(()); + } + // Holding the lock keeps the runner unreaped, so its PID cannot be + // reused before the pidfd pins the process. + let pid = libc::pid_t::try_from(runner.id()).map_err(|_| ErrorKind::InvalidInput)?; + // SAFETY: `pidfd_open` takes no pointers. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + RawFd::try_from(fd) + .ok() + .filter(|fd| *fd >= 0) + // SAFETY: A successful `pidfd_open` returns a new descriptor + // that nothing else owns. + .map(|fd| unsafe { OwnedFd::from_raw_fd(fd) }) + }; + // Without a pidfd (kernels before 5.3), fall back to polling for exit. + let timeout = match pidfd { + Some(_) => timeout, + None => Some(timeout.map_or(RUNNER_POLL_INTERVAL, |t| t.min(RUNNER_POLL_INTERVAL))), + }; + let timeout_ms = timeout.map_or(-1, |timeout| { + i32::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(i32::MAX) + }); + // `poll` ignores entries with negative descriptors. + let mut fds = [ + pidfd.as_ref().map_or(-1, AsRawFd::as_raw_fd), + listener.map_or(-1, AsRawFd::as_raw_fd), + ] + .map(|fd| libc::pollfd { + fd, + events: libc::POLLIN, + revents: 0, + }); + // SAFETY: `fds` is a valid, writable array of `fds.len()` entries whose + // descriptors stay open for the duration of the call. + let result = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, timeout_ms) }; + if result < 0 { + let error = IoError::last_os_error(); + if error.kind() != ErrorKind::Interrupted { + return Err(error); + } + } + Ok(()) +} diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 360efbb203..34431534be 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -6,7 +6,7 @@ use std::io::Result as IoResult; use std::os::windows::io::AsRawHandle; use std::process::Child; use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; @@ -16,7 +16,8 @@ use litebox_broker_transport_windows_userland::named_pipe::{ use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; use super::{ - PendingRunnerAssociation, UserlandProcessLauncher, accept_runner_channel, runner_has_exited, + PendingRunnerAssociation, RUNNER_POLL_INTERVAL, UserlandProcessLauncher, accept_runner_channel, + runner_has_exited, }; use crate::runtime::{AssociationOutcome, is_peer_closed_error}; @@ -67,11 +68,16 @@ impl PlatformRunnerEndpoint { fn serve_association( control_listener: &mut WindowsNamedPipeListener, runner: &Arc>, - startup: PendingRunnerAssociation, + mut startup: PendingRunnerAssociation, setup_deadline: Instant, broker: BrokerCore, launcher: Arc, ) -> AssociationOutcome { + let image = startup.take_image(); + debug_assert!( + image.is_none(), + "Windows runners do not support process images" + ); let shutdown_was_expected = startup.process.shutdown_was_expected(); let control_channel = match accept_control_channel(control_listener, runner, setup_deadline) { Ok(connection) => connection, @@ -113,6 +119,7 @@ fn accept_control_channel( "control", || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), || control_listener.try_accept(), + |remaining| wait_for_runner_event(runner, None, Some(remaining)), )?; let runner_id = runner.lock().expect("runner process mutex poisoned").id(); validate_client_process(&control_stream, runner_id)?; @@ -122,6 +129,21 @@ fn accept_control_channel( )) } +/// Sleeps for at most [`RUNNER_POLL_INTERVAL`] before callers recheck the +/// runner and listener. +#[expect( + clippy::unnecessary_wraps, + reason = "callers share the fallible Linux signature" +)] +pub(super) fn wait_for_runner_event( + _runner: &Arc>, + _listener: Option<&WindowsNamedPipeListener>, + timeout: Option, +) -> IoResult<()> { + std::thread::sleep(timeout.map_or(RUNNER_POLL_INTERVAL, |t| t.min(RUNNER_POLL_INTERVAL))); + Ok(()) +} + fn unique_control_pipe_name() -> OsString { let process_id = std::process::id(); let nonce = std::time::SystemTime::now() diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index acb44b9a03..6669174de2 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -138,6 +138,11 @@ fn run_fake_runner(args: &[OsString]) { )?; let control_memory = setup.receive_control_ring(Some(Instant::now() + Duration::from_secs(5)))?; + assert!( + setup + .receive_process_image(Some(Instant::now() + Duration::from_secs(5)))? + .is_none() + ); let control_ring = ControlRing::new(control_memory).map_err(|error| { std::io::Error::new( ErrorKind::InvalidData, diff --git a/litebox_common_linux/src/errno/generated.rs b/litebox_common_linux/src/errno/generated.rs index 12bc76aca2..d825f8ad5c 100644 --- a/litebox_common_linux/src/errno/generated.rs +++ b/litebox_common_linux/src/errno/generated.rs @@ -153,6 +153,7 @@ impl super::Errno { 132 => "ERFKILL: Operation not possible due to RF-kill", 133 => "EHWPOISON: Memory page has hardware error", 512 => "ERESTARTSYS: Interrupted system call, restartable with SA_RESTART", + 513 => "ERESTARTNOINTR: Interrupted system call, always restartable", 514 => "ERESTARTNOHAND: Interrupted system call, restartable if unhandled", _ => unreachable!(), } diff --git a/litebox_common_linux/src/errno/mod.rs b/litebox_common_linux/src/errno/mod.rs index f9b419e169..c90ce4c090 100644 --- a/litebox_common_linux/src/errno/mod.rs +++ b/litebox_common_linux/src/errno/mod.rs @@ -58,6 +58,8 @@ impl Errno { /// Restart the syscall unless a handler without `SA_RESTART` runs. Interrupted waits convert /// to this, as in Linux. pub const ERESTARTSYS: Self = Self::from_const(512); + /// Restart the syscall, even after a handler runs. + pub const ERESTARTNOINTR: Self = Self::from_const(513); /// Restart the syscall only if no handler runs. pub const ERESTARTNOHAND: Self = Self::from_const(514); @@ -77,7 +79,8 @@ impl Errno { /// restart. #[must_use] pub fn without_restart(self) -> Self { - if self == Self::ERESTARTSYS || self == Self::ERESTARTNOHAND { + if self == Self::ERESTARTSYS || self == Self::ERESTARTNOINTR || self == Self::ERESTARTNOHAND + { Self::EINTR } else { self diff --git a/litebox_common_linux/src/lib.rs b/litebox_common_linux/src/lib.rs index a92015ae79..5125938907 100644 --- a/litebox_common_linux/src/lib.rs +++ b/litebox_common_linux/src/lib.rs @@ -2150,6 +2150,7 @@ impl ShutdownHow { #[non_exhaustive] #[derive(Debug)] pub enum SyscallRequest { + Fork, Vfork, Wait4 { pid: i32, @@ -2916,6 +2917,8 @@ impl SyscallRequest { Sysno::exit => sys_req!(Exit { status }), Sysno::exit_group => sys_req!(ExitGroup { status }), #[cfg(target_arch = "x86_64")] + Sysno::fork => SyscallRequest::Fork, + #[cfg(target_arch = "x86_64")] Sysno::vfork => SyscallRequest::Vfork, Sysno::wait4 => sys_req!(Wait4 { pid, wstatus:*, options, rusage:* }), Sysno::uname => sys_req!(Uname { buf:* }), @@ -3334,7 +3337,7 @@ impl TimeParam { /// pt_regs from [Linux](https://elixir.bootlin.com/linux/v5.19.17/source/arch/x86/include/asm/ptrace.h#L59) #[cfg(target_arch = "x86_64")] #[repr(C)] -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, FromBytes, IntoBytes, Immutable)] pub struct PtRegs { /* * C ABI says these regs are callee-preserved. They aren't saved on kernel entry @@ -3376,7 +3379,7 @@ pub struct PtRegs { /// pt_regs from [Linux](https://elixir.bootlin.com/linux/v5.19.17/source/arch/arm64/include/asm/ptrace.h#L178) #[cfg(target_arch = "aarch64")] #[repr(C, align(16))] -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, FromBytes, IntoBytes, Immutable)] pub struct PtRegs { /// General-purpose registers x0-x30. pub regs: [usize; AARCH64_GENERAL_REGISTER_COUNT], diff --git a/litebox_common_linux/src/program_startup.rs b/litebox_common_linux/src/program_startup.rs index 11d543d5bd..40a2fa0fdb 100644 --- a/litebox_common_linux/src/program_startup.rs +++ b/litebox_common_linux/src/program_startup.rs @@ -1,23 +1,55 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Bounded Linux program startup payload used by constrained `vfork` transfer. +//! Bounded Linux process startup payloads, which start a child in a fresh runner after +//! constrained `vfork` transfer or `fork`. +use alloc::boxed::Box; use alloc::ffi::CString; use alloc::string::String; use alloc::vec::Vec; +use core::ops::Range; use litebox::pipes::HalfPipeType; use litebox_broker_protocol::ObjectHandle; use litebox_broker_protocol::process::MAX_PROCESS_BOOTSTRAP_SIZE; +use zerocopy::{FromBytes, IntoBytes}; -use crate::signal::SigSet; +use crate::signal::{NSIG, SigAction, SigAltStack, SigSet}; +use crate::vmem::VmFlags; +use crate::{PtRegs, TASK_COMM_LEN}; -const HEADER_SIZE: usize = size_of::<[u32; 11]>() + size_of::<[u64; 2]>(); +const HEADER_SIZE: usize = size_of::() + size_of::<[u32; 11]>() + size_of::<[u64; 2]>(); /// Size of an inherited descriptor's number, handle, and kind tag, which precede its kind's fields. const INHERITED_FD_HEADER_SIZE: usize = size_of::() + size_of::() + size_of::(); +/// Size of a memory region's start, end, and flags. +const FORK_REGION_SIZE: usize = size_of::<[u64; 2]>() + size_of::(); const FILE_TAG: u8 = 0; const PIPE_TAG: u8 = 1; +const PROGRAM_STARTUP_TAG: u8 = 0; +const FORK_STARTUP_TAG: u8 = 1; + +/// Linux startup of a child process in a fresh runner. +pub enum LinuxProcessStartup { + /// Load a program, as `execve` does. + Program(LinuxProgramStartup), + /// Continue a process duplicated by `fork`. + Fork(Box), +} + +impl LinuxProcessStartup { + /// Decodes and validates a bounded broker payload produced by + /// [`LinuxProgramStartup::encode`] or [`LinuxForkStartup::encode`]. + pub fn decode(payload: &[u8]) -> Result { + match payload.first() { + Some(&PROGRAM_STARTUP_TAG) => LinuxProgramStartup::decode(payload).map(Self::Program), + Some(&FORK_STARTUP_TAG) => { + LinuxForkStartup::decode(payload).map(|startup| Self::Fork(Box::new(startup))) + } + _ => Err(LinuxProgramStartupError::Malformed), + } + } +} /// Linux program state needed to load a child in a fresh runner. /// @@ -85,6 +117,83 @@ pub enum InheritedFdKind { }, } +/// Linux process state needed to continue a child duplicated by `fork` in a fresh runner. +/// +/// The child's memory contents travel separately, in a process image holding the contents of +/// each region that [has contents](ForkMemoryRegion::has_contents) back to back, in region order. +#[derive(Clone)] +pub struct LinuxForkStartup { + /// Parent process ID visible to the child. + pub parent_process_id: i32, + /// Real user ID. + pub uid: u32, + /// Effective user ID. + pub euid: u32, + /// Real group ID. + pub gid: u32, + /// Effective group ID. + pub egid: u32, + /// File mode creation mask, with only permission bits set. + pub umask: u32, + /// Absolute working directory. + pub cwd: String, + /// Command name. + pub comm: [u8; TASK_COMM_LEN], + /// Blocked signals. + pub blocked_signals: SigSet, + /// Signal dispositions, indexed by signal number minus one. + pub signal_actions: [SigAction; NSIG], + /// Alternate signal stack. + pub alternate_signal_stack: SigAltStack, + /// Registers at the `fork` system call, which the child returns from. + pub registers: PtRegs, + /// Thread pointer, such as the FS base on x86-64. + pub thread_pointer: usize, + /// The parent's system call entry point, which the child's must match because the + /// duplicated code calls it. + pub syscall_entry_point: usize, + /// Address where the child stores its thread ID, or zero. + pub set_child_tid: usize, + /// Address the child clears and wakes when it exits, or zero. + pub clear_child_tid: usize, + /// Initial program break. + pub initial_program_break: usize, + /// Current program break. + pub program_break: usize, + /// Memory regions, in strictly ascending address order. + pub regions: Vec, + /// Descriptors the child inherits, in strictly ascending descriptor order. + pub fds: Vec, +} + +/// A memory region of a process duplicated by `fork`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ForkMemoryRegion { + /// Page-aligned addresses. + pub range: Range, + /// Region flags. + pub flags: VmFlags, +} + +impl ForkMemoryRegion { + /// Returns whether the region's contents are in the process image, which holds those of + /// every region with any access permission or [`VmFlags::VM_HAS_CONTENTS`]. Any other + /// region starts zero-filled. + pub fn has_contents(&self) -> bool { + self.flags + .intersects(VmFlags::VM_ACCESS_FLAGS | VmFlags::VM_HAS_CONTENTS) + } +} + +/// A descriptor a child duplicated by `fork` inherits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ForkedFd { + /// The descriptor and the child's broker handle to its object. + pub inherited: InheritedFd, + /// Whether the descriptor is closed on `execve`. + pub close_on_exec: bool, +} + /// Invalid or unsupported Linux program startup data. #[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)] pub enum LinuxProgramStartupError { @@ -120,6 +229,7 @@ impl LinuxProgramStartup { output .try_reserve_exact(encoded_len) .map_err(|_| LinuxProgramStartupError::TooLarge)?; + output.push(PROGRAM_STARTUP_TAG); push_u32(&mut output, self.parent_process_id.cast_unsigned()); push_u32(&mut output, self.uid); push_u32(&mut output, self.euid); @@ -167,11 +277,14 @@ impl LinuxProgramStartup { } /// Decodes and validates a bounded broker payload. - pub fn decode(payload: &[u8]) -> Result { + fn decode(payload: &[u8]) -> Result { if payload.len() < HEADER_SIZE || payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { return Err(LinuxProgramStartupError::Malformed); } let mut input = payload; + if read_u8(&mut input)? != PROGRAM_STARTUP_TAG { + return Err(LinuxProgramStartupError::Malformed); + } let parent_process_id = read_u32(&mut input)?.cast_signed(); let real_user_id = read_u32(&mut input)?; let effective_user_id = read_u32(&mut input)?; @@ -250,6 +363,156 @@ impl LinuxProgramStartup { } } +impl LinuxForkStartup { + /// Encodes this startup into the bounded broker payload. + pub fn encode(&self) -> Result, LinuxProgramStartupError> { + validate_fork(self)?; + let mut output = Vec::new(); + output.push(FORK_STARTUP_TAG); + push_u32(&mut output, self.parent_process_id.cast_unsigned()); + push_u32(&mut output, self.uid); + push_u32(&mut output, self.euid); + push_u32(&mut output, self.gid); + push_u32(&mut output, self.egid); + push_u32(&mut output, self.umask); + output.extend_from_slice(&self.comm); + push_u64(&mut output, self.blocked_signals.as_u64()); + output.extend_from_slice(self.signal_actions.as_bytes()); + output.extend_from_slice(self.alternate_signal_stack.as_bytes()); + output.extend_from_slice(self.registers.as_bytes()); + for value in [ + self.thread_pointer, + self.syscall_entry_point, + self.set_child_tid, + self.clear_child_tid, + self.initial_program_break, + self.program_break, + ] { + push_u64(&mut output, value as u64); + } + for count in [self.cwd.len(), self.regions.len(), self.fds.len()] { + push_u32( + &mut output, + u32::try_from(count).map_err(|_| LinuxProgramStartupError::TooLarge)?, + ); + } + output.extend_from_slice(self.cwd.as_bytes()); + for region in &self.regions { + push_u64(&mut output, region.range.start as u64); + push_u64(&mut output, region.range.end as u64); + push_u32(&mut output, region.flags.bits()); + } + for fd in &self.fds { + fd.inherited.encode(&mut output); + output.push(u8::from(fd.close_on_exec)); + } + if output.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { + return Err(LinuxProgramStartupError::TooLarge); + } + Ok(output) + } + + /// Decodes and validates a bounded broker payload. + #[expect( + clippy::similar_names, + reason = "uid/euid and gid/egid are Linux credential names" + )] + fn decode(payload: &[u8]) -> Result { + if payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { + return Err(LinuxProgramStartupError::Malformed); + } + let mut input = payload; + if read_u8(&mut input)? != FORK_STARTUP_TAG { + return Err(LinuxProgramStartupError::Malformed); + } + let parent_process_id = read_u32(&mut input)?.cast_signed(); + let uid = read_u32(&mut input)?; + let euid = read_u32(&mut input)?; + let gid = read_u32(&mut input)?; + let egid = read_u32(&mut input)?; + let umask = read_u32(&mut input)?; + let comm = take_bytes(&mut input, TASK_COMM_LEN)? + .try_into() + .map_err(|_| LinuxProgramStartupError::Malformed)?; + let blocked_signals = SigSet::from_u64(read_u64(&mut input)?); + let signal_actions = read_value(&mut input)?; + let alternate_signal_stack = read_value(&mut input)?; + let registers = read_value(&mut input)?; + let thread_pointer = read_usize(&mut input)?; + let syscall_entry_point = read_usize(&mut input)?; + let set_child_tid = read_usize(&mut input)?; + let clear_child_tid = read_usize(&mut input)?; + let initial_program_break = read_usize(&mut input)?; + let program_break = read_usize(&mut input)?; + let cwd_length = usize::try_from(read_u32(&mut input)?) + .map_err(|_| LinuxProgramStartupError::Malformed)?; + let region_count = usize::try_from(read_u32(&mut input)?) + .map_err(|_| LinuxProgramStartupError::Malformed)?; + let fd_count = usize::try_from(read_u32(&mut input)?) + .map_err(|_| LinuxProgramStartupError::Malformed)?; + let cwd = core::str::from_utf8(take_bytes(&mut input, cwd_length)?) + .map_err(|_| LinuxProgramStartupError::InvalidWorkingDirectory)? + .into(); + if region_count > input.len() / FORK_REGION_SIZE { + return Err(LinuxProgramStartupError::Malformed); + } + let mut regions = Vec::with_capacity(region_count); + for _ in 0..region_count { + let start = read_usize(&mut input)?; + let end = read_usize(&mut input)?; + let flags = VmFlags::from_bits(read_u32(&mut input)?) + .ok_or(LinuxProgramStartupError::Malformed)?; + regions.push(ForkMemoryRegion { + range: start..end, + flags, + }); + } + if fd_count > input.len() / (INHERITED_FD_HEADER_SIZE + size_of::()) { + return Err(LinuxProgramStartupError::Malformed); + } + let mut fds = Vec::with_capacity(fd_count); + for _ in 0..fd_count { + let inherited = InheritedFd::decode(&mut input)?; + let close_on_exec = match read_u8(&mut input)? { + 0 => false, + 1 => true, + _ => return Err(LinuxProgramStartupError::Malformed), + }; + fds.push(ForkedFd { + inherited, + close_on_exec, + }); + } + if !input.is_empty() { + return Err(LinuxProgramStartupError::Malformed); + } + let startup = Self { + parent_process_id, + uid, + euid, + gid, + egid, + umask, + cwd, + comm, + blocked_signals, + signal_actions, + alternate_signal_stack, + registers, + thread_pointer, + syscall_entry_point, + set_child_tid, + clear_child_tid, + initial_program_break, + program_break, + regions, + fds, + }; + validate_fork(&startup)?; + Ok(startup) + } +} + impl InheritedFd { fn encoded_len(&self) -> usize { INHERITED_FD_HEADER_SIZE @@ -308,6 +571,33 @@ fn validate(startup: &LinuxProgramStartup) -> Result<(), LinuxProgramStartupErro Ok(()) } +fn validate_fork(startup: &LinuxForkStartup) -> Result<(), LinuxProgramStartupError> { + if startup.parent_process_id <= 0 { + return Err(LinuxProgramStartupError::InvalidParentProcess); + } + if !startup.cwd.starts_with('/') || startup.cwd.as_bytes().contains(&0) { + return Err(LinuxProgramStartupError::InvalidWorkingDirectory); + } + if startup.umask & !0o777 != 0 { + return Err(LinuxProgramStartupError::InvalidUmask); + } + let mut previous_end = 0; + for region in &startup.regions { + if region.range.start < previous_end || region.range.start >= region.range.end { + return Err(LinuxProgramStartupError::Malformed); + } + previous_end = region.range.end; + } + if startup + .fds + .windows(2) + .any(|pair| pair[0].inherited.fd >= pair[1].inherited.fd) + { + return Err(LinuxProgramStartupError::Malformed); + } + Ok(()) +} + fn encoded_len(startup: &LinuxProgramStartup) -> Result { let mut length = HEADER_SIZE .checked_add(startup.path.len()) @@ -366,6 +656,15 @@ fn read_u64(input: &mut &[u8]) -> Result { )) } +fn read_usize(input: &mut &[u8]) -> Result { + usize::try_from(read_u64(input)?).map_err(|_| LinuxProgramStartupError::Malformed) +} + +fn read_value(input: &mut &[u8]) -> Result { + T::read_from_bytes(take_bytes(input, size_of::())?) + .map_err(|_| LinuxProgramStartupError::Malformed) +} + fn take_bytes<'a>( input: &mut &'a [u8], length: usize, @@ -460,6 +759,128 @@ mod tests { ); } + fn fork_startup() -> LinuxForkStartup { + let mut signal_actions = [SigAction { + sigaction: crate::signal::SIG_DFL, + flags: crate::signal::SaFlags::empty(), + __pad: 0, + restorer: 0, + mask: SigSet::empty(), + }; NSIG]; + signal_actions[0] = SigAction { + sigaction: 0x1234, + flags: crate::signal::SaFlags::RESTORER, + __pad: 0, + restorer: 0x5678, + mask: SigSet::empty().with(Signal::SIGUSR2), + }; + LinuxForkStartup { + parent_process_id: 17, + uid: 1000, + euid: 1001, + gid: 1002, + egid: 1003, + umask: 0o027, + cwd: "/home/user".into(), + comm: *b"python3\0\0\0\0\0\0\0\0\0", + blocked_signals: SigSet::empty().with(Signal::SIGUSR1), + signal_actions, + alternate_signal_stack: SigAltStack { + sp: 0x7000_0000, + flags: crate::signal::SsFlags::empty(), + __pad: 0, + size: 0x4000, + }, + registers: PtRegs::default(), + thread_pointer: 0x7fff_0000, + syscall_entry_point: 0x5555_0000, + set_child_tid: 0x7fff_1000, + clear_child_tid: 0, + initial_program_break: 0x40_0000, + program_break: 0x40_2000, + regions: vec![ + ForkMemoryRegion { + range: 0x40_0000..0x40_2000, + flags: VmFlags::VM_READ | VmFlags::VM_WRITE, + }, + ForkMemoryRegion { + range: 0x7000_0000..0x7000_4000, + flags: VmFlags::empty(), + }, + ], + fds: vec![ + ForkedFd { + inherited: InheritedFd { + fd: 0, + handle: ObjectHandle(7), + kind: InheritedFdKind::File, + }, + close_on_exec: false, + }, + ForkedFd { + inherited: InheritedFd { + fd: 3, + handle: ObjectHandle(8), + kind: InheritedFdKind::Pipe { + endpoint: HalfPipeType::SenderHalf, + }, + }, + close_on_exec: true, + }, + ], + } + } + + #[test] + fn fork_startup_round_trips() { + let startup = fork_startup(); + let encoded = startup.encode().unwrap(); + let Ok(LinuxProcessStartup::Fork(decoded)) = LinuxProcessStartup::decode(&encoded) else { + panic!("fork startup must decode as a fork"); + }; + assert_eq!(decoded.regions, startup.regions); + assert_eq!(decoded.fds, startup.fds); + assert_eq!(decoded.comm, startup.comm); + assert_eq!(decoded.signal_actions[0].restorer, 0x5678); + assert_eq!(decoded.encode().unwrap(), encoded); + } + + #[test] + fn fork_startup_rejects_malformed_payloads() { + let startup = fork_startup(); + let encoded = startup.encode().unwrap(); + + let mut trailing = encoded.clone(); + trailing.push(0); + assert!(matches!( + LinuxProcessStartup::decode(&trailing), + Err(LinuxProgramStartupError::Malformed) + )); + assert!(matches!( + LinuxProcessStartup::decode(&encoded[..encoded.len() - 1]), + Err(LinuxProgramStartupError::Malformed) + )); + let mut unknown = encoded; + unknown[0] = 2; + assert!(matches!( + LinuxProcessStartup::decode(&unknown), + Err(LinuxProgramStartupError::Malformed) + )); + + let mut overlapping = startup.clone(); + overlapping.regions[1].range = 0x40_1000..0x40_3000; + assert!(matches!( + overlapping.encode(), + Err(LinuxProgramStartupError::Malformed) + )); + let mut unordered = startup; + unordered.fds.swap(0, 1); + assert!(matches!( + unordered.encode(), + Err(LinuxProgramStartupError::Malformed) + )); + } + #[test] fn program_startup_rejects_trailing_and_invalid_strings() { let startup = LinuxProgramStartup { diff --git a/litebox_common_linux/src/signal/mod.rs b/litebox_common_linux/src/signal/mod.rs index 55c29c158e..e40a016fb8 100644 --- a/litebox_common_linux/src/signal/mod.rs +++ b/litebox_common_linux/src/signal/mod.rs @@ -15,7 +15,7 @@ use x86_64::Sigcontext; use int_enum::IntEnum; use litebox::utils::ReinterpretSignedExt as _; -use zerocopy::{FromBytes, IntoBytes}; +use zerocopy::{FromBytes, Immutable, IntoBytes}; use crate::errno::Errno; @@ -119,7 +119,7 @@ pub enum SignalDisposition { Continue, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, Immutable)] #[repr(transparent)] pub struct SigSet(u64); @@ -228,7 +228,7 @@ impl core::ops::Not for SigSet { } /// Signal action flags for `rt_sigaction` syscall. -#[derive(Copy, Clone, FromBytes, IntoBytes)] +#[derive(Copy, Clone, FromBytes, IntoBytes, Immutable)] #[repr(transparent)] pub struct SaFlags(u32); @@ -249,7 +249,7 @@ bitflags::bitflags! { /// Linux's `sigaction` struct used by the `rt_sigaction` syscall. #[repr(C)] -#[derive(Copy, Clone, FromBytes, IntoBytes)] +#[derive(Copy, Clone, FromBytes, IntoBytes, Immutable)] pub struct SigAction { pub sigaction: usize, pub flags: SaFlags, @@ -272,7 +272,7 @@ pub enum SigmaskHow { } #[repr(C)] -#[derive(Copy, Clone, FromBytes, IntoBytes)] +#[derive(Copy, Clone, FromBytes, IntoBytes, Immutable)] pub struct SigAltStack { pub sp: usize, pub flags: SsFlags, @@ -283,7 +283,7 @@ pub struct SigAltStack { } /// Signal stack flags. -#[derive(Debug, Clone, Copy, FromBytes, IntoBytes)] +#[derive(Debug, Clone, Copy, FromBytes, IntoBytes, Immutable)] #[repr(transparent)] pub struct SsFlags(u32); diff --git a/litebox_common_linux/src/vmem.rs b/litebox_common_linux/src/vmem.rs index 199d720e20..5d36821a57 100644 --- a/litebox_common_linux/src/vmem.rs +++ b/litebox_common_linux/src/vmem.rs @@ -53,6 +53,10 @@ bitflags::bitflags! { /// The area can grow downward upon page fault. const VM_GROWSDOWN = 1 << 8; + /// The area may hold data even while it has no access: it is file-backed or had access + /// at some point. Inaccessible areas without it, such as reservations and guard pages, + /// hold only zeros. + const VM_HAS_CONTENTS = 1 << 9; const VM_ACCESS_FLAGS = Self::VM_READ.bits() | Self::VM_WRITE.bits() @@ -312,9 +316,13 @@ impl VmArea { self.is_file_backed } - /// Create a new [`VmArea`] with the given flags. + /// Create a new [`VmArea`] with the given flags, adding [`VmFlags::VM_HAS_CONTENTS`] if the + /// area is file-backed or has any access. #[inline] - pub(super) fn new(flags: VmFlags, is_file_backed: bool) -> Self { + pub(super) fn new(mut flags: VmFlags, is_file_backed: bool) -> Self { + if is_file_backed || flags.intersects(VmFlags::VM_ACCESS_FLAGS) { + flags |= VmFlags::VM_HAS_CONTENTS; + } Self { flags, is_file_backed, @@ -831,7 +839,8 @@ where /// next mapping after the expansion. /// /// It fails if it resizes more than one mapping or needs to split the current mapping - /// (due to enlarging). + /// (due to enlarging). When enlarging, adjacent mappings that differ only in + /// [`VmFlags::VM_HAS_CONTENTS`] count as one. /// /// See for reference. /// @@ -848,45 +857,47 @@ where { let range = range.start..range.end; // `cur_range` contains `range.start` - let (cur_range, cur_vma) = self + let cur_range = self .vmas .get_key_value(&range.start) + .map(|(cur_range, _)| cur_range.clone()) .ok_or(VmemResizeError::NotExist(range.start))?; - let new_end = range - .start - .checked_add(new_size.as_usize()) - .ok_or(VmemResizeError::OutOfMemory)?; - match new_end.cmp(&range.end) { - core::cmp::Ordering::Equal => { - // no change - return Ok(()); - } - core::cmp::Ordering::Less => { + let new_end = range.start.checked_add(new_size.as_usize()); + if let Some(new_end) = new_end + && new_end <= range.end + { + if new_end < range.end { // shrink let range = PageRange::new(new_end, range.end).unwrap(); unsafe { self.remove_mapping(range) }.unwrap(); - return Ok(()); } - core::cmp::Ordering::Greater => {} + return Ok(()); } // grow - if range.end > cur_range.end { + if self.merged_area(range.clone()).is_none() { // we can't remap across vm area boundaries return Err(VmemResizeError::InvalidAddr { - range: cur_range.clone(), + range: cur_range, addr: range.end, }); } + let new_end = new_end.ok_or(VmemResizeError::OutOfMemory)?; + // `last_range` contains the last page of `range` + let (last_range, last_vma) = self + .vmas + .get_key_value(&(range.end - 1)) + .map(|(last_range, last_vma)| (last_range.clone(), *last_vma)) + .unwrap(); - if range.end == cur_range.end { - // expand the current range + if range.end == last_range.end { + // expand the last range let r = range.end..new_end; if self.vmas.overlaps(&r) { return Err(VmemResizeError::RangeOccupied(r)); } - if cur_vma.is_file_backed() { + if last_vma.is_file_backed() { unimplemented!("file-backed mapping expansion is not supported yet"); } let range = PageRange::new(range.end, new_end).unwrap(); @@ -894,7 +905,7 @@ where // litebox mappings in this range, this may fail if there are // platform mappings in the way. match unsafe { - self.insert_mapping(range, *cur_vma, false, FixedAddressBehavior::NoReplace) + self.insert_mapping(range, last_vma, false, FixedAddressBehavior::NoReplace) } { Ok(_) => {} Err(AllocationError::PermissionDenied) => { @@ -914,8 +925,30 @@ where return Ok(()); } - // has to split the current range and move it to somewhere else - Err(VmemResizeError::RangeOccupied(range.end..cur_range.end)) + // has to split the last range and move it to somewhere else + Err(VmemResizeError::RangeOccupied(range.end..last_range.end)) + } + + /// Returns the one area Linux sees over `range`, if areas that differ only in + /// [`VmFlags::VM_HAS_CONTENTS`] cover it without gaps. + /// + /// Such areas, as left by making part of an inaccessible area accessible and back, are one + /// area to Linux. The returned area may hold data if any of them may. + fn merged_area(&self, range: Range) -> Option { + let (first_range, first) = self.vmas.get_key_value(&range.start)?; + let mut area = *first; + let mut end = first_range.end; + while end < range.end { + let (next_range, next) = self.vmas.get_key_value(&end)?; + if next.is_file_backed() != first.is_file_backed() + || !((next.flags() ^ first.flags()) - VmFlags::VM_HAS_CONTENTS).is_empty() + { + return None; + } + area.flags |= next.flags(); + end = next_range.end; + } + Some(area) } /// Move a range from `old_range` to `suggested_new_range`. @@ -934,7 +967,8 @@ where /// # Panics /// /// Panics if the size of `suggested_new_range` is smaller than the size of `old_range`. - /// Panics if the `old_range` is not covered by exactly one mapping. + /// Panics if the `old_range` is not covered by exactly one mapping, counting adjacent mappings + /// that differ only in [`VmFlags::VM_HAS_CONTENTS`] as one. pub(super) unsafe fn move_mappings( &mut self, old_range: PageRange, @@ -946,13 +980,9 @@ where { assert!(new_size.as_usize() >= old_range.len()); - // Check if the given range is covered by exactly one mapping - let (cur_range, vma) = self - .vmas - .get_key_value(&old_range.start) - .expect("VMEM: range not found"); - assert!(cur_range.contains(&(old_range.end - 1))); - let vma = *vma; + let vma = self + .merged_area(old_range.into()) + .expect("VMEM: range not covered by one mapping"); if vma.is_file_backed() { unimplemented!("file-backed mapping move is not supported yet"); @@ -977,7 +1007,8 @@ where } { Ok(new_addr) => new_addr, Err(RemapError::UnsupportedByPlatform) => { - // SAFETY: Native remapping left the source unchanged, and the destination is free. + // SAFETY: The caller excludes source users, and the copy only places pages with + // hints, which never replace existing mappings. return unsafe { self.remap_fallback_with_copy(old_range, new_range, vma) }; } Err(error) => return Err(VmemMoveError::RemapError(error)), @@ -1015,7 +1046,7 @@ where { const COPY_CHUNK_SIZE: usize = 1 << 16; - let permissions = MemoryRegionPermissions::from(vma.flags()); + let permissions = MemoryRegionPermissions::from(vma.flags() & VmFlags::VM_ACCESS_FLAGS); let temporary = VmArea::new( (vma.flags() | VmFlags::VM_READ | VmFlags::VM_WRITE) - VmFlags::VM_EXEC, false, @@ -1122,13 +1153,8 @@ where .map_err(VmemProtectError::ProtectError)?; for (intersection, vma) in mappings_to_change { let new_flags = (vma.flags & !VmFlags::VM_ACCESS_FLAGS) | flags; - self.vmas.insert( - intersection, - VmArea { - flags: new_flags, - is_file_backed: vma.is_file_backed, - }, - ); + self.vmas + .insert(intersection, VmArea::new(new_flags, vma.is_file_backed)); } Ok(()) @@ -1307,17 +1333,21 @@ where length: NonZeroPageSize, behavior: FixedAddressBehavior, ) -> FindAreaRequest { - let (address_range_end, alignment) = if suggested_address.is_none() { - ( + // Only fixed-address requests may go above the placement limit; addresses chosen here, + // including fallbacks for unusable hints, stay below it. + let (address_range_end, alignment) = match (suggested_address, behavior) { + (None, _) => ( // Some platform may allocate more than requested to satisfy alignment requirements, // so we restrict the maximum address to avoid exceeding the platform's addressable range. - Platform::TASK_ADDR_MAX & !(Platform::RESERVATION_ALIGNMENT - 1), + Platform::PLACEMENT_ADDR_MAX & !(Platform::RESERVATION_ALIGNMENT - 1), // When no specific address is suggested, use the platform's reservation alignment // to minimize fragmentation and number of system calls. Platform::RESERVATION_ALIGNMENT, - ) - } else { - (Platform::TASK_ADDR_MAX, ALIGN) + ), + (Some(_), FixedAddressBehavior::Hint(_)) => (Platform::PLACEMENT_ADDR_MAX, ALIGN), + (Some(_), FixedAddressBehavior::NoReplace | FixedAddressBehavior::Replace) => { + (Platform::TASK_ADDR_MAX, ALIGN) + } }; FindAreaRequest { suggested_address, @@ -1571,14 +1601,17 @@ mod tests { use super::*; type AllocationCall = (Range, FixedAddressBehavior); + type PermissionUpdate = (Range, MemoryRegionPermissions); litebox::define_page_reservation!(DummyReservation); /// A configurable dummy page-management backend. struct DummyVmemBackend { rejected_address: Option, + remap_unsupported: bool, calls: Mutex>, releases: Mutex>>, + permission_updates: Mutex>, } impl litebox::platform::RawPointerProvider for DummyVmemBackend { @@ -1646,6 +1679,9 @@ mod tests { new_range: Range, permissions: MemoryRegionPermissions, ) -> Result, litebox::platform::page_mgmt::RemapError> { + if self.remap_unsupported { + return Err(litebox::platform::page_mgmt::RemapError::UnsupportedByPlatform); + } Ok(TransparentMutPtr::from_usize(new_range.start)) } @@ -1654,6 +1690,9 @@ mod tests { range: Range, new_permissions: MemoryRegionPermissions, ) -> Result<(), litebox::platform::page_mgmt::PermissionUpdateError> { + self.permission_updates + .lock() + .push((range, new_permissions)); Ok(()) } @@ -1667,8 +1706,10 @@ mod tests { ) -> &'static DummyVmemBackend { Box::leak(Box::new(DummyVmemBackend { rejected_address, + remap_unsupported: false, calls: Mutex::new(Vec::new()), releases: Mutex::new(Vec::new()), + permission_updates: Mutex::new(Vec::new()), })) } @@ -2067,4 +2108,197 @@ mod tests { Err(AllocationError::AboveMaxAddress) )); } + + #[test] + fn inaccessible_areas_remember_whether_they_had_access() { + let start = DummyVmemBackend::::TASK_ADDR_MIN; + let mut vmm = Vmem::new(dummy_backend::(None)); + unsafe { + vmm.create_pages( + Some(NonZeroAddress::new(start).unwrap()), + NonZeroPageSize::new(3 * PAGE_SIZE).unwrap(), + CreatePagesFlags::FIXED_ADDR, + MemoryRegionPermissions::empty(), + ) + } + .unwrap(); + let accessed = PageRange::new(start + PAGE_SIZE, start + 2 * PAGE_SIZE).unwrap(); + for permissions in [ + MemoryRegionPermissions::READ | MemoryRegionPermissions::WRITE, + MemoryRegionPermissions::empty(), + ] { + unsafe { vmm.protect_mapping(accessed, permissions) }.unwrap(); + } + assert_eq!( + contents_by_area(&vmm), + [ + (start..start + PAGE_SIZE, false), + (start + PAGE_SIZE..start + 2 * PAGE_SIZE, true), + (start + 2 * PAGE_SIZE..start + 3 * PAGE_SIZE, false), + ] + ); + + // Growing in place treats areas that differ only in whether they may hold data as one, + // without merging them. + unsafe { + vmm.resize_mapping( + PageRange::new(start, start + 3 * PAGE_SIZE).unwrap(), + NonZeroPageSize::new(4 * PAGE_SIZE).unwrap(), + ) + } + .unwrap(); + let blocker = start + 5 * PAGE_SIZE..start + 6 * PAGE_SIZE; + unsafe { + vmm.create_mapping( + NonZeroAddress::new(blocker.start), + NonZeroPageSize::new(PAGE_SIZE).unwrap(), + VmArea::new(VmFlags::VM_READ | VmFlags::VM_MAYREAD, false), + CreatePagesFlags::FIXED_ADDR, + ) + } + .unwrap(); + let split = [ + (start..start + PAGE_SIZE, false), + (start + PAGE_SIZE..start + 2 * PAGE_SIZE, true), + (start + 2 * PAGE_SIZE..start + 4 * PAGE_SIZE, false), + (blocker, true), + ]; + assert_eq!(contents_by_area(&vmm), split); + + // Failed growth and moves leave them apart. + let grown = PageRange::new(start, start + 4 * PAGE_SIZE).unwrap(); + let huge = NonZeroPageSize::new(usize::MAX & !(PAGE_SIZE - 1)).unwrap(); + assert!(matches!( + unsafe { vmm.resize_mapping(grown, NonZeroPageSize::new(6 * PAGE_SIZE).unwrap()) }, + Err(VmemResizeError::RangeOccupied(_)) + )); + assert!(matches!( + unsafe { vmm.resize_mapping(grown, huge) }, + Err(VmemResizeError::OutOfMemory) + )); + assert!(matches!( + unsafe { vmm.move_mappings(grown, None, huge) }, + Err(VmemMoveError::OutOfMemory) + )); + assert_eq!(contents_by_area(&vmm), split); + + // Moving them makes one area that may hold data. + let moved = + unsafe { vmm.move_mappings(grown, None, NonZeroPageSize::new(4 * PAGE_SIZE).unwrap()) } + .unwrap() + .as_usize(); + let areas = vmm + .iter() + .map(|(range, vma)| (range.clone(), vma.flags())) + .collect::>(); + assert_eq!(areas.len(), 2); + assert_eq!(areas[1].0, moved..moved + 4 * PAGE_SIZE); + assert!(areas[1].1.contains(VmFlags::VM_HAS_CONTENTS)); + assert!(!areas[1].1.intersects(VmFlags::VM_ACCESS_FLAGS)); + } + + #[test] + fn growing_across_distinct_areas_fails_before_size_overflows() { + let start = DummyVmemBackend::::TASK_ADDR_MIN; + let mut vmm = Vmem::new(dummy_backend::(None)); + for (offset, flags) in [ + (0, VmFlags::VM_READ | VmFlags::VM_MAYREAD), + (PAGE_SIZE, VmFlags::VM_MAYREAD), + ] { + unsafe { + vmm.create_mapping( + NonZeroAddress::new(start + offset), + NonZeroPageSize::new(PAGE_SIZE).unwrap(), + VmArea::new(flags, false), + CreatePagesFlags::FIXED_ADDR, + ) + } + .unwrap(); + } + + assert!(matches!( + unsafe { + vmm.resize_mapping( + PageRange::new(start, start + 2 * PAGE_SIZE).unwrap(), + NonZeroPageSize::new(usize::MAX & !(PAGE_SIZE - 1)).unwrap(), + ) + }, + Err(VmemResizeError::InvalidAddr { .. }) + )); + } + + fn contents_by_area(vmm: &Vmem) -> Vec<(Range, bool)> { + vmm.iter() + .map(|(range, vma)| { + let contents = vma.flags().contains(VmFlags::VM_HAS_CONTENTS); + (range.clone(), contents) + }) + .collect() + } + + #[test] + fn copying_remap_of_shared_mapping_passes_only_access_permissions() { + let backend: &'static DummyVmemBackend = Box::leak(Box::new(DummyVmemBackend { + rejected_address: None, + remap_unsupported: true, + calls: Mutex::new(Vec::new()), + releases: Mutex::new(Vec::new()), + permission_updates: Mutex::new(Vec::new()), + })); + let data = (0..2 * PAGE_SIZE) + .map(|i| u8::try_from(i % 251).unwrap() + 1) + .collect::>(); + // The copy goes through the mapped addresses, so back them with real memory. + let memory = vec![0u8; 9 * PAGE_SIZE].leak(); + let aligned = memory.as_ptr().align_offset(PAGE_SIZE); + let pages = &mut memory[aligned..aligned + 8 * PAGE_SIZE]; + pages[..data.len()].copy_from_slice(&data); + let start = pages.as_mut_ptr() as usize; + let flags = VmFlags::VM_READ + | VmFlags::VM_WRITE + | VmFlags::VM_SHARED + | VmFlags::VM_MAYREAD + | VmFlags::VM_MAYWRITE + | VmFlags::VM_MAYSHARE; + let mut vmm = Vmem::new(backend); + unsafe { + vmm.create_mapping( + NonZeroAddress::new(start), + NonZeroPageSize::new(2 * PAGE_SIZE).unwrap(), + VmArea::new(flags, false), + CreatePagesFlags::FIXED_ADDR, + ) + } + .unwrap(); + + let destination = unsafe { + vmm.move_mappings( + PageRange::new(start, start + 2 * PAGE_SIZE).unwrap(), + NonZeroAddress::new(start + 4 * PAGE_SIZE), + NonZeroPageSize::new(4 * PAGE_SIZE).unwrap(), + ) + } + .unwrap() + .as_usize(); + + assert_eq!(destination, start + 4 * PAGE_SIZE); + assert_eq!( + TransparentConstPtr::::from_usize(destination) + .to_owned_slice(data.len()) + .unwrap()[..], + data[..] + ); + assert_eq!( + *backend.permission_updates.lock(), + [( + destination..destination + 4 * PAGE_SIZE, + MemoryRegionPermissions::READ | MemoryRegionPermissions::WRITE, + )] + ); + let vmas = vmm + .iter() + .map(|(range, vma)| (range.clone(), vma.flags().contains(VmFlags::VM_SHARED))) + .collect::>(); + assert_eq!(vmas, [(destination..destination + 4 * PAGE_SIZE, true)]); + } } diff --git a/litebox_platform_linux_userland/src/page_mgmt.rs b/litebox_platform_linux_userland/src/page_mgmt.rs index 3046f59284..f07afaa7c5 100644 --- a/litebox_platform_linux_userland/src/page_mgmt.rs +++ b/litebox_platform_linux_userland/src/page_mgmt.rs @@ -54,6 +54,20 @@ impl litebox::platform::PageManagementProvider for Li /// the allocator to place into a region holding no existing mapping. #[cfg(target_arch = "aarch64")] const TASK_ADDR_MAX: usize = 0x0000_FFFF_FFFF_F000; // (1 << 48) - PAGE_SIZE; + /// With ASLR disabled the host's top-down mmap area begins just below + /// `0x7FFF_F800_0000`; with ASLR enabled it is randomized by at most 1 TiB + /// below that. Placing guest memory under this limit keeps it out of the + /// host's way, so `fork` can restore the guest at the parent's addresses + /// in a fresh runner without colliding with that runner's host mappings. + #[cfg(target_arch = "x86_64")] + const PLACEMENT_ADDR_MAX: usize = 0x7000_0000_0000; + /// The kernel may place a rejected hint anywhere, including inside the + /// host's mmap area, so vmem must pick exact addresses itself. + #[cfg(target_arch = "x86_64")] + const HINT_PLACEMENT_BEHAVIOR: HintPlacementBehavior = HintPlacementBehavior::Unspecified; + /// Exact `MAP_FIXED_NOREPLACE` placement would fail on hosts with fewer + /// than 48 VA bits; see the `TASK_ADDR_MAX` note above. + #[cfg(target_arch = "aarch64")] const HINT_PLACEMENT_BEHAVIOR: HintPlacementBehavior = HintPlacementBehavior::Directional(AllocationDirection::TopDown); @@ -126,19 +140,71 @@ impl litebox::platform::PageManagementProvider for Li &self, old_range: core::ops::Range, new_range: core::ops::Range, - _permissions: MemoryRegionPermissions, + #[cfg_attr(target_arch = "aarch64", expect(unused_variables))] + permissions: MemoryRegionPermissions, ) -> Result, litebox::platform::page_mgmt::RemapError> { + // Without `MREMAP_FIXED` the kernel ignores `new_range` and may move the pages into the + // host's mmap area. `MREMAP_FIXED` replaces whatever is mapped at the destination, and the + // host may hold mappings vmem does not know about, so claim the destination first. + // + // The claim also provides the grown tail, so only the old pages move: growing them would + // extend pages that a fork restore mapped from its process image into the image's + // following bytes instead of fresh zeroed pages. + #[cfg(target_arch = "x86_64")] + let (flags, moved_len) = { + >::allocate_pages( + self, + new_range.clone(), + // Shared anonymous memory is private on the host, as in `allocate_pages`. + permissions - MemoryRegionPermissions::SHARED, + false, + false, + FixedAddressBehavior::NoReplace, + ) + .map_err(|error| match error { + // A host mapping vmem does not know about holds the destination; the caller's + // copy places the pages elsewhere instead. + litebox::platform::page_mgmt::AllocationError::AddressInUse => { + litebox::platform::page_mgmt::RemapError::UnsupportedByPlatform + } + _ => litebox::platform::page_mgmt::RemapError::OutOfMemory, + })?; + ( + MRemapFlags::MREMAP_MAYMOVE | MRemapFlags::MREMAP_FIXED, + old_range.len(), + ) + }; + // Only a fork restore, which is x86_64-only, maps guest pages from a file. + #[cfg(target_arch = "aarch64")] + let (flags, moved_len) = (MRemapFlags::MREMAP_MAYMOVE, new_range.len()); let res = unsafe { syscalls::syscall5( syscalls::Sysno::mremap, old_range.start, old_range.len(), - new_range.len(), - MRemapFlags::MREMAP_MAYMOVE.bits() as usize, + moved_len, + flags.bits() as usize, new_range.start, ) - .expect("mremap failed") }; + // A moved range's pages and its claimed tail stay separate host mappings, and kernels + // before 6.17 cannot move more than one mapping at once. Release the claim, which the + // kernel may already have unmapped, and let the caller copy instead. Later kernels can + // move several mappings but may stop partway when the host runs out of memory or + // mappings; the moved pages are then released with the claim, and the caller's copy + // panics on the missing source. + #[cfg(target_arch = "x86_64")] + let res = res.map_err(|_| { + // SAFETY: vmem reserved `new_range` for this move, so only the claim made above and + // any source pages a partial move placed in it can be unmapped. + let _ = unsafe { + syscalls::syscall2(syscalls::Sysno::munmap, new_range.start, new_range.len()) + } + .expect("munmap failed"); + litebox::platform::page_mgmt::RemapError::UnsupportedByPlatform + })?; + #[cfg(target_arch = "aarch64")] + let res = res.expect("mremap failed"); Ok(UserMutPtr::from_usize(res)) } @@ -302,4 +368,70 @@ mod tests { prev = page.end; } } + + #[cfg(target_arch = "x86_64")] + #[test] + fn moving_growth_of_file_pages_adds_zeroed_pages() { + const PAGE: usize = 4096; + let platform = LinuxUserland::new(); + let read_write = libc::PROT_READ | libc::PROT_WRITE; + // SAFETY: The name is a valid C string. + let fd = unsafe { libc::memfd_create(c"remap".as_ptr(), 0) }; + assert!(fd >= 0); + let contents: Vec = [0xa5; PAGE].into_iter().chain([0x5a; PAGE]).collect(); + // SAFETY: `contents` is valid for its length. + let written = unsafe { libc::write(fd, contents.as_ptr().cast(), contents.len()) }; + assert_eq!(written, isize::try_from(contents.len()).unwrap()); + // SAFETY: A private mapping of the descriptor's first page replaces nothing. + let old = unsafe { + libc::mmap( + core::ptr::null_mut(), + PAGE, + read_write, + libc::MAP_PRIVATE, + fd, + 0, + ) + }; + assert_ne!(old, libc::MAP_FAILED); + // Find a free destination by mapping and releasing it. + // SAFETY: Anonymous mappings replace nothing, and nothing uses the released one. + let new = unsafe { + let new = libc::mmap( + core::ptr::null_mut(), + 2 * PAGE, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ); + assert_ne!(new, libc::MAP_FAILED); + assert_eq!(libc::munmap(new, 2 * PAGE), 0); + new as usize + }; + let old = old as usize; + + // SAFETY: Nothing else uses the old page, and the destination is free. + let moved = unsafe { + >::remap_pages( + platform, + old..old + PAGE, + new..new + 2 * PAGE, + MemoryRegionPermissions::READ | MemoryRegionPermissions::WRITE, + ) + } + .unwrap(); + + assert_eq!(moved.as_usize(), new); + // SAFETY: The remap left both pages mapped readable and writable. + let pages = unsafe { core::slice::from_raw_parts_mut(new as *mut u8, 2 * PAGE) }; + assert!(pages[..PAGE].iter().all(|&byte| byte == 0xa5)); + assert!(pages[PAGE..].iter().all(|&byte| byte == 0)); + pages[PAGE] = 1; + // SAFETY: Nothing uses the pages or the descriptor anymore. + unsafe { + assert_eq!(libc::munmap(new as *mut libc::c_void, 2 * PAGE), 0); + assert_eq!(libc::close(fd), 0); + } + } } diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 5379ae6d08..97abb5ae82 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -8,7 +8,9 @@ use litebox_platform_linux_userland::SeccompScope; use std::path::PathBuf; use litebox_broker_local_userland as broker; -use litebox_common_linux::program_startup::LinuxProgramStartup; +use litebox_common_linux::program_startup::{ + LinuxForkStartup, LinuxProcessStartup, LinuxProgramStartup, +}; use litebox_common_linux::signal::SigSet; // Use a stable non-root guest identity instead of mirroring the host user. This keeps shim @@ -122,6 +124,7 @@ fn run_with_seccomp(cli_args: CliArgs, seccomp_scope: SeccompScope) -> Result Result Some( + LinuxProcessStartup::decode(&startup.payload) + .context("invalid child Linux process startup")?, + ), + None => None, + }; + let (task_params, prog_path, argv, envp) = match startup { + Some(LinuxProcessStartup::Fork(startup)) => { + // The process image is loaded before seccomp forbids inspecting it. + let program = restore_fork(&shim, *startup, initial_thread, process_image)?; + litebox_platform_linux_userland::LinuxUserland::enable_seccomp_filter( + &broker_positional_io_fds, + &broker_shutdown_fds, + seccomp_scope, + ); + return Ok(run_program(&shim, program)); + } + Some(LinuxProcessStartup::Program(startup)) => { + let LinuxProgramStartup { + parent_process_id, uid, euid, gid, egid, blocked_signals, ignored_signals, - inherited_fds: Some(inherited_fds), - cwd: Some(cwd), - umask: Some(umask), - }, - path, - argv, - envp, - ) - } else { - let prog_path = cli_args - .program_and_arguments - .first() - .context("program path missing")? - .clone(); - if !prog_path.starts_with('/') { - anyhow::bail!("program path must be absolute (e.g., /usr/bin/ls), got: {prog_path}"); + umask, + path, + cwd, + argv, + envp, + inherited_fds, + } = startup; + ( + litebox_common_linux::TaskParams { + pid: process_id, + ppid: parent_process_id, + uid, + euid, + gid, + egid, + blocked_signals, + ignored_signals, + inherited_fds: Some(inherited_fds), + cwd: Some(cwd), + umask: Some(umask), + }, + path, + argv, + envp, + ) } - let argv = cli_args - .program_and_arguments - .iter() - .map(|value| std::ffi::CString::new(value.as_bytes())) - .collect::, _>>() - .context("invalid program argument")?; - let proxy_url = cli_args.broker_proxy_url; - let mut environment = cli_args.environment_variables; - if cli_args.forward_environment_variables { - environment.extend(std::env::vars().map(|(key, value)| format!("{key}={value}"))); + None => { + let prog_path = cli_args + .program_and_arguments + .first() + .context("program path missing")? + .clone(); + if !prog_path.starts_with('/') { + anyhow::bail!( + "program path must be absolute (e.g., /usr/bin/ls), got: {prog_path}" + ); + } + let argv = cli_args + .program_and_arguments + .iter() + .map(|value| std::ffi::CString::new(value.as_bytes())) + .collect::, _>>() + .context("invalid program argument")?; + let proxy_url = cli_args.broker_proxy_url; + let mut environment = cli_args.environment_variables; + if cli_args.forward_environment_variables { + environment.extend(std::env::vars().map(|(key, value)| format!("{key}={value}"))); + } + apply_broker_proxy_environment(&mut environment, proxy_url.as_deref()); + let envp = environment + .iter() + .map(|value| std::ffi::CString::new(value.as_bytes())) + .collect::, _>>() + .context("invalid environment variable")?; + ( + litebox_common_linux::TaskParams { + pid: process_id, + ppid: 0, + uid: u32::from(DEFAULT_GUEST_UID), + euid: u32::from(DEFAULT_GUEST_UID), + gid: u32::from(DEFAULT_GUEST_GID), + egid: u32::from(DEFAULT_GUEST_GID), + blocked_signals: SigSet::empty(), + ignored_signals: SigSet::empty(), + inherited_fds: None, + cwd: None, + umask: None, + }, + prog_path, + argv, + envp, + ) } - apply_broker_proxy_environment(&mut environment, proxy_url.as_deref()); - let envp = environment - .iter() - .map(|value| std::ffi::CString::new(value.as_bytes())) - .collect::, _>>() - .context("invalid environment variable")?; - ( - litebox_common_linux::TaskParams { - pid: process_id, - ppid: 0, - uid: u32::from(DEFAULT_GUEST_UID), - euid: u32::from(DEFAULT_GUEST_UID), - gid: u32::from(DEFAULT_GUEST_GID), - egid: u32::from(DEFAULT_GUEST_GID), - blocked_signals: SigSet::empty(), - ignored_signals: SigSet::empty(), - inherited_fds: None, - cwd: None, - umask: None, - }, - prog_path, - argv, - envp, - ) }; litebox_platform_linux_userland::LinuxUserland::enable_seccomp_filter( @@ -228,7 +252,14 @@ fn run_with_seccomp(cli_args: CliArgs, seccomp_scope: SeccompScope) -> Result, + program: litebox_shim_linux::LoadedProgram, +) -> i32 { #[cfg(feature = "lock_tracing")] litebox::sync::start_recording(); @@ -259,7 +290,87 @@ fn run_with_seccomp(cli_args: CliArgs, seccomp_scope: SeccompScope) -> Result, + startup: LinuxForkStartup, + initial_thread: litebox::thread::Thread, + process_image: Option, +) -> Result> { + let process_image = process_image + .map(|image| { + let image = std::fs::File::from(image); + let len = image.metadata()?.len(); + std::io::Result::Ok((image, len)) + }) + .transpose() + .context("failed to inspect the process image")?; + shim.restore_fork(startup, initial_thread, |offset, pages| { + let Some((image, image_len)) = &process_image else { + return Ok(()); + }; + map_process_image(image, *image_len, offset, pages).map_err(|error| { + error + .raw_os_error() + .and_then(|errno| litebox_common_linux::errno::Errno::try_from(errno).ok()) + .unwrap_or(litebox_common_linux::errno::Errno::EIO) + }) + }) + .context("failed to continue the forked process") +} + +#[cfg(not(target_arch = "x86_64"))] +fn restore_fork( + _shim: &litebox_shim_linux::LinuxShim, + _startup: LinuxForkStartup, + _initial_thread: litebox::thread::Thread, + _process_image: Option, +) -> Result> { + anyhow::bail!("fork is unsupported on this architecture") +} + +/// Maps `image`, which is `image_len` bytes long, privately over the page-aligned whole `pages` +/// from `offset`, so they share the image's memory until written instead of copying it. +/// +/// The pages start zero-filled, and those past the image's end stay so, as accessing a mapping +/// there would fault. +#[cfg(target_arch = "x86_64")] +fn map_process_image( + image: &std::fs::File, + image_len: u64, + offset: u64, + pages: &mut [u8], +) -> std::io::Result<()> { + use std::os::fd::AsRawFd as _; + + let Some(available) = image_len.checked_sub(offset).filter(|&len| len > 0) else { + return Ok(()); + }; + let len = usize::try_from(available).map_or(pages.len(), |len| len.min(pages.len())); + let offset = libc::off_t::try_from(offset) + .map_err(|_| std::io::Error::from_raw_os_error(libc::EOVERFLOW))?; + // SAFETY: `pages` is exclusively borrowed and page-aligned, and the readable and writable + // private mapping replacing its first `len` bytes keeps them valid, only changing their + // contents. + let mapped = unsafe { + libc::mmap( + pages.as_mut_ptr().cast(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_FIXED, + image.as_raw_fd(), + offset, + ) + }; + if mapped == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + Ok(()) } fn apply_broker_proxy_environment(environment: &mut Vec, proxy_url: Option<&str>) { diff --git a/litebox_runner_linux_userland/tests/fork_parent.c b/litebox_runner_linux_userland/tests/fork_parent.c new file mode 100644 index 0000000000..f32bedd39a --- /dev/null +++ b/litebox_runner_linux_userland/tests/fork_parent.c @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUFFER_SIZE (4 * 1024 * 1024) + +static int global_value = 1; +static volatile sig_atomic_t usr1_count; +static volatile sig_atomic_t sigchld_count; +// A page written and then made inaccessible, and an inaccessible reservation never written. +static unsigned char *hidden; +static unsigned char *reserved; + +static void on_usr1(int signal) { + (void)signal; + usr1_count++; +} + +static void on_sigchld(int signal) { + (void)signal; + sigchld_count++; +} + +// Returns whether the inaccessible page still holds what was written before it became so. +static int hidden_is_intact(void) { + if (mprotect(hidden, 4096, PROT_READ) != 0) { + return 0; + } + int intact = memcmp(hidden, "hidden", sizeof "hidden") == 0; + return mprotect(hidden, 4096, PROT_NONE) == 0 && intact; +} + +static unsigned long checksum(const unsigned char *buffer, size_t size) { + unsigned long sum = 0; + for (size_t i = 0; i < size; i++) { + sum = sum * 31 + buffer[i]; + } + return sum; +} + +static int wait_for(pid_t child, const char *name) { + int status = 0; + pid_t waited; + do { + waited = waitpid(child, &status, 0); + } while (waited == -1 && errno == EINTR); + printf("%s child=%d waited=%d exited=%d code=%d\n", name, child, waited, WIFEXITED(status), + WEXITSTATUS(status)); + return waited == child ? 0 : 1; +} + +// Runs in the child returned by glibc's `fork`. +static int run_child(pid_t parent, unsigned char *buffer, unsigned long expected_sum, int pipe_fd, + int (*exec_only)(void)) { + int failures = 0; + // The child sees the parent's memory as of the fork. + failures += global_value != 2; + failures += checksum(buffer, BUFFER_SIZE) != expected_sum; + // Execute-only memory keeps its code. + failures += exec_only() != 42; + // An inaccessible reservation stays reserved and zero-filled. + failures += mprotect(reserved, 4096, PROT_READ) != 0 || reserved[0] != 0; + // Writes stay in the child. + global_value = 3; + memset(buffer, 0x5a, BUFFER_SIZE); + // `raise` signals the thread ID glibc stored through `CLONE_CHILD_SETTID`. + failures += raise(SIGUSR1) != 0; + failures += usr1_count != 1; + // The close-on-exec descriptor is inherited with its flag. + failures += fcntl(pipe_fd, F_GETFD) != FD_CLOEXEC; + pid_t pid = getpid(); + failures += pid == parent || getppid() != parent || gettid() != pid; + + pid_t grandchild = fork(); + if (grandchild == 0) { + // The inaccessible page's contents survive a second fork before the child touches it. + _exit(global_value == 3 && hidden_is_intact() ? 5 : 6); + } + int status = 0; + failures += grandchild < 0 || waitpid(grandchild, &status, 0) != grandchild || + !WIFEXITED(status) || WEXITSTATUS(status) != 5; + failures += !hidden_is_intact(); + + char message[64]; + int length = snprintf(message, sizeof message, "child pid=%d failures=%d\n", pid, failures); + failures += write(pipe_fd, message, length) != length; + printf("child-stdout pid=%d\n", pid); + fflush(stdout); + return failures == 0 ? 7 : 8; +} + +int main(void) { + pid_t parent = getpid(); + signal(SIGUSR1, on_usr1); + signal(SIGCHLD, on_sigchld); + unsigned char *buffer = malloc(BUFFER_SIZE); + if (buffer == NULL) { + return 2; + } + for (size_t i = 0; i < BUFFER_SIZE; i++) { + buffer[i] = (unsigned char)(i * 7 + i / 4096); + } + unsigned long expected_sum = checksum(buffer, BUFFER_SIZE); + int pipe_fds[2]; + if (pipe2(pipe_fds, O_CLOEXEC) != 0) { + return 3; + } + // `mov eax, 42; ret` + static const unsigned char return_42[] = {0xb8, 0x2a, 0x00, 0x00, 0x00, 0xc3}; + unsigned char *code = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (code == MAP_FAILED) { + return 3; + } + memcpy(code, return_42, sizeof return_42); + if (mprotect(code, 4096, PROT_EXEC) != 0) { + return 3; + } + int (*exec_only)(void) = (int (*)(void))(void *)code; + hidden = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + reserved = mmap(NULL, 1024 * 1024, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (hidden == MAP_FAILED || reserved == MAP_FAILED) { + return 3; + } + memcpy(hidden, "hidden", sizeof "hidden"); + if (mprotect(hidden, 4096, PROT_NONE) != 0) { + return 3; + } + global_value = 2; + fflush(stdout); + + pid_t child = fork(); + if (child == 0) { + close(pipe_fds[0]); + exit(run_child(parent, buffer, expected_sum, pipe_fds[1], exec_only)); + } + if (child < 0) { + printf("fork-failed errno=%d\n", errno); + return 4; + } + close(pipe_fds[1]); + char message[128] = {0}; + ssize_t received = 0; + for (;;) { + ssize_t n = read(pipe_fds[0], message + received, sizeof message - 1 - received); + if (n > 0) { + received += n; + } else if (n == 0 || errno != EINTR) { + break; + } + } + printf("pipe %s", message); + int failures = wait_for(child, "fork"); + + pid_t raw_child = syscall(SYS_fork); + if (raw_child == 0) { + syscall(SYS_exit_group, global_value == 2 ? 9 : 10); + } + if (raw_child < 0) { + perror("SYS_fork"); + return 5; + } + failures += wait_for(raw_child, "raw-fork"); + + pid_t again = waitpid(-1, NULL, WNOHANG); + int again_errno = errno; + printf("parent pid=%d global=%d intact=%d usr1=%d sigchld=%d echild=%d\n", getpid(), + global_value, + checksum(buffer, BUFFER_SIZE) == expected_sum && exec_only() == 42 && + hidden_is_intact(), + usr1_count, sigchld_count > 0, again == -1 && again_errno == ECHILD); + return failures; +} diff --git a/litebox_runner_linux_userland/tests/fork_threads_parent.c b/litebox_runner_linux_userland/tests/fork_threads_parent.c new file mode 100644 index 0000000000..801cbfe7db --- /dev/null +++ b/litebox_runner_linux_userland/tests/fork_threads_parent.c @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +// Forks while sibling threads run guest code, block reading a pipe, and sleep, and forks from +// several threads at once, checking that each child copies memory the counting thread could have +// left and that the siblings go on unaffected. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#define FORKERS 2 +#define FORKS_PER_FORKER 3 + +// Two counters far enough apart that copying memory reaches one long after the other. +static struct { + volatile unsigned long head; + char gap[4 * 1024 * 1024]; + volatile unsigned long tail; +} counters; +static atomic_int stop; +static int reader_pipe[2]; +static atomic_int forkers_ready; + +static void *count_up(void *arg) { + (void)arg; + for (unsigned long value = 1; !atomic_load_explicit(&stop, memory_order_relaxed); value++) { + counters.head = value; + counters.tail = value; + } + return NULL; +} + +// Returns whether the counters hold values `count_up` could have left between two instructions. +static int counters_consistent(void) { + unsigned long head = counters.head; + unsigned long tail = counters.tail; + return head == tail || head == tail + 1; +} + +static void *read_pipe(void *arg) { + (void)arg; + char byte = 0; + ssize_t count; + do { + count = read(reader_pipe[0], &byte, 1); + } while (count == -1 && errno == EINTR); + return (void *)(long)(count == 1 && byte == 'x'); +} + +static void *sleep_through_forks(void *arg) { + (void)arg; + struct timespec duration = {.tv_sec = 0, .tv_nsec = 300 * 1000 * 1000}; + return (void *)(long)(nanosleep(&duration, NULL) == 0); +} + +static int wait_exit_code(pid_t child) { + int status = 0; + pid_t waited; + do { + waited = waitpid(child, &status, 0); + } while (waited == -1 && errno == EINTR); + return waited == child && WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +// Forks by system call, as glibc's fork() holds process-wide locks that would keep the forkers +// from forking at once. Each child only reads memory and exits. +static void *fork_repeatedly(void *arg) { + (void)arg; + long failures = 0; + for (int i = 0; i < FORKS_PER_FORKER; i++) { + // Spin, rather than block, so the forkers make the system call at the same time. + atomic_fetch_add(&forkers_ready, 1); + while (atomic_load(&forkers_ready) < (i + 1) * FORKERS) { + } + pid_t child = (pid_t)syscall(SYS_fork); + if (child == 0) { + _exit(counters_consistent() ? 0 : 1); + } + failures += child < 0 || wait_exit_code(child) != 0; + } + return (void *)failures; +} + +int main(void) { + pthread_t counter, reader, sleeper, forkers[FORKERS]; + if (pipe(reader_pipe) != 0 || pthread_create(&counter, NULL, count_up, NULL) != 0 || + pthread_create(&reader, NULL, read_pipe, NULL) != 0 || + pthread_create(&sleeper, NULL, sleep_through_forks, NULL) != 0) { + printf("setup-error\n"); + return 2; + } + // Let the counter get going and the reader and the sleeper block. + while (counters.tail < 1000) { + } + struct timespec settle = {.tv_sec = 0, .tv_nsec = 20 * 1000 * 1000}; + nanosleep(&settle, NULL); + + pid_t child = fork(); + if (child == 0) { + _exit(counters_consistent() ? 7 : 1); + } + int code = child < 0 ? -1 : wait_exit_code(child); + + // Forkers pause each other, and the other threads, as they fork at once. + long failures = 0; + for (int i = 0; i < FORKERS; i++) { + if (pthread_create(&forkers[i], NULL, fork_repeatedly, NULL) != 0) { + printf("setup-error\n"); + return 3; + } + } + for (int i = 0; i < FORKERS; i++) { + void *forker_failures; + pthread_join(forkers[i], &forker_failures); + failures += (long)forker_failures; + } + + // The siblings go on after the forks. + unsigned long last = counters.tail; + while (counters.tail == last) { + } + void *read_ok, *slept; + if (write(reader_pipe[1], "x", 1) != 1) { + printf("write-error\n"); + return 4; + } + pthread_join(reader, &read_ok); + pthread_join(sleeper, &slept); + atomic_store(&stop, 1); + pthread_join(counter, NULL); + printf("threads-fork code=%d failures=%ld read=%ld slept=%ld\n", code, failures, (long)read_ok, + (long)slept); + return 0; +} diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index 491da6d6f1..c7d2524e7e 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -61,6 +61,8 @@ impl litebox_broker_core::stdio::StdioProvider for CapturingStdioProvider { // colliding with this sweep's dynamic `_rewriter` outputs. const DEDICATED_C_TESTS: &[&str] = &[ "async_x16.c", + "fork_parent.c", + "fork_threads_parent.c", "gate_signals.c", "sigreturn.c", "sigreturn_simd.c", @@ -947,6 +949,125 @@ fn vfork_children_run_shell_commands() { } } +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[test] +fn fork_child_resumes_from_parent_snapshot() { + let parent = common::compile("./tests/fork_parent.c", "fork_parent", true, false); + let mut runner = Runner::new(&parent, "fork_parent"); + runner.allow_process_duplication(); + + let output = String::from_utf8(runner.output()).unwrap(); + let line = |prefix: &str| { + output + .lines() + .find(|line| line.starts_with(prefix)) + .unwrap_or_else(|| panic!("missing {prefix:?} output in {output:?}")) + }; + let parent_line = line("parent "); + let parent_pid = numeric_field(parent_line, "pid="); + assert_eq!(numeric_field(parent_line, "global="), 2); + assert_eq!(numeric_field(parent_line, "intact="), 1); + assert_eq!(numeric_field(parent_line, "usr1="), 0); + assert_eq!(numeric_field(parent_line, "sigchld="), 1); + assert_eq!(numeric_field(parent_line, "echild="), 1); + + let fork_line = line("fork "); + let child = numeric_field(fork_line, "child="); + assert_ne!(child, parent_pid); + assert_eq!(numeric_field(fork_line, "waited="), child); + assert_eq!(numeric_field(fork_line, "exited="), 1); + assert_eq!(numeric_field(fork_line, "code="), 7); + let pipe_line = line("pipe "); + assert_eq!(numeric_field(pipe_line, "pid="), child); + assert_eq!(numeric_field(pipe_line, "failures="), 0); + assert_eq!(numeric_field(line("child-stdout "), "pid="), child); + + let raw_line = line("raw-fork "); + let raw_child = numeric_field(raw_line, "child="); + assert!(raw_child != parent_pid && raw_child != child); + assert_eq!(numeric_field(raw_line, "waited="), raw_child); + assert_eq!(numeric_field(raw_line, "exited="), 1); + assert_eq!(numeric_field(raw_line, "code="), 9); +} + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[test] +fn fork_pauses_sibling_threads() { + let parent = common::compile( + "./tests/fork_threads_parent.c", + "fork_threads_parent", + true, + false, + ); + let mut runner = Runner::new(&parent, "fork_threads_parent"); + runner.allow_process_duplication(); + + let output = String::from_utf8(runner.output()).unwrap(); + let line = output + .lines() + .find(|line| line.starts_with("threads-fork ")) + .unwrap_or_else(|| panic!("missing threads-fork output in {output:?}")); + assert_eq!(numeric_field(line, "code="), 7); + assert_eq!(numeric_field(line, "failures="), 0); + assert_eq!(numeric_field(line, "read="), 1); + assert_eq!(numeric_field(line, "slept="), 1); +} + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[test] +fn fork_requires_process_duplication() { + let parent = common::compile( + "./tests/fork_parent.c", + "fork_without_duplication", + true, + false, + ); + let output = Runner::new(&parent, "fork_without_duplication").output_expect_failure(); + let output = String::from_utf8(output).unwrap(); + let line = output + .lines() + .find(|line| line.starts_with("fork-failed ")) + .unwrap_or_else(|| panic!("missing fork failure in {output:?}")); + assert_eq!(numeric_field(line, "errno="), libc::EPERM); +} + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[test] +fn forked_shell_subshells_and_pipelines() { + const SCRIPT: &str = r#" +value=$(echo substituted) +echo "substitution=$value" +echo piped | { read line; echo "pipeline=$line"; } +(exit 3) +echo "subshell=$?" +cd /tmp && umask 027 && (read marker < cwd-marker; echo "nested cwd=$marker umask=$(umask)") +"#; + let mut runner = Runner::new(Path::new("/usr/bin/dash"), "fork_dash"); + runner + .allow_process_duplication() + .args(["-c", SCRIPT]) + .with_fs_path(|root| { + // dash's `pwd` prints its own copy of the cwd, so the forked subshell opens a + // relative path to check the cwd the child actually restored. + std::fs::create_dir_all(root.join("tmp")).unwrap(); + std::fs::write(root.join("tmp/cwd-marker"), "tmp\n").unwrap(); + }); + + let output = String::from_utf8(runner.output()).unwrap(); + let lines: Vec<&str> = output.lines().collect(); + for expected in [ + "substitution=substituted", + "pipeline=piped", + "subshell=3", + "nested cwd=tmp umask=0027", + ] { + assert!( + lines.contains(&expected), + "missing {expected:?} in {output:?}" + ); + } +} + /// Get the path of a program using `which` fn run_which(prog: &str) -> std::path::PathBuf { let prog_path_str = std::process::Command::new("which") @@ -1306,7 +1427,8 @@ fn run_test_broker_connection( |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), Some(setup_deadline))?; - channel.send_memfd(control_ring.memory(), Some(setup_deadline)) + channel.send_memfd(control_ring.memory(), Some(setup_deadline))?; + channel.send_process_image(None, Some(setup_deadline)) }, ) .expect("broker host setup failed") @@ -2138,6 +2260,40 @@ finally: } } +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[test] +fn python_os_fork() { + const SCRIPT: &str = r#" +import os +data = {"value": 1} +read_fd, write_fd = os.pipe() +pid = os.fork() +if pid == 0: + os.close(read_fd) + data["value"] = 2 + os.write(write_fd, f"child {os.getpid()} {os.getppid()} {data['value']}".encode()) + os._exit(11) +os.close(write_fd) +with os.fdopen(read_fd) as pipe: + message = pipe.read() +waited, status = os.waitpid(pid, 0) +print("pipe", message == f"child {pid} {os.getpid()} 2") +print("wait", waited == pid, os.waitstatus_to_exitcode(status)) +print("parent", data["value"]) +"#; + let mut runner = python_runner("python_os_fork"); + runner.allow_process_duplication().args(["-c", SCRIPT]); + + let output = String::from_utf8(runner.output()).unwrap(); + let lines: Vec<&str> = output.lines().collect(); + for expected in ["pipe True", "wait True 11", "parent 1"] { + assert!( + lines.contains(&expected), + "missing {expected:?} in {output:?}" + ); + } +} + #[cfg(target_os = "linux")] #[test] fn test_runner_with_python_repl_pty() { diff --git a/litebox_runner_windows_on_linux_userland/src/lib.rs b/litebox_runner_windows_on_linux_userland/src/lib.rs index 613c852e61..9a91a4fa8d 100644 --- a/litebox_runner_windows_on_linux_userland/src/lib.rs +++ b/litebox_runner_windows_on_linux_userland/src/lib.rs @@ -93,6 +93,7 @@ pub fn run(cli_args: CliArgs) -> Result<()> { coordinator, positional_io_fds: _broker_positional_io_fds, shutdown_fd: _broker_shutdown_fd, + process_image: _broker_process_image, } = connection; let (litebox, process_id, initial_thread) = litebox::LiteBox::new_process_with_broker_local(platform, local); diff --git a/litebox_shim_linux/src/lib.rs b/litebox_shim_linux/src/lib.rs index d70726c474..21442b8f98 100644 --- a/litebox_shim_linux/src/lib.rs +++ b/litebox_shim_linux/src/lib.rs @@ -162,6 +162,27 @@ impl MemoryManager { state.current = brk; } + /// Returns the initial and current program break. + #[cfg(target_arch = "x86_64")] + fn program_break(&self) -> (usize, usize) { + let state = self.brk.lock(); + (state.initial, state.current) + } + + /// Restores the program break of a process duplicated by `fork`, whose memory already + /// covers it. + /// + /// # Panics + /// + /// Panics if the initial program break has already been set. + #[cfg(target_arch = "x86_64")] + fn restore_program_break(&self, initial: usize, current: usize) { + let mut state = self.brk.lock(); + assert_eq!(state.initial, 0, "initial brk is already set"); + state.initial = initial; + state.current = current; + } + /// Sets or queries the Linux program break. /// /// # Panics @@ -499,6 +520,106 @@ impl LinuxShim { }) } + /// Continues a process duplicated by `fork` using the initial thread allocated during broker + /// negotiation. + /// + /// `load_image` fills page-aligned whole pages with the parent's process image from an + /// offset, possibly by mapping the image over them privately. The pages start zero-filled, + /// so ranges the image does not hold may be left untouched. + /// + /// Each memory region is restored at the parent's address and fails if the address is in + /// use. Regions formerly backed by files are restored as anonymous memory. + #[cfg(target_arch = "x86_64")] + pub fn restore_fork( + &self, + startup: litebox_common_linux::program_startup::LinuxForkStartup, + initial_thread: litebox::thread::Thread, + load_image: impl FnMut(u64, &mut [u8]) -> Result<(), Errno>, + ) -> Result, ForkRestoreError> { + let litebox_common_linux::program_startup::LinuxForkStartup { + parent_process_id, + uid, + euid, + gid, + egid, + umask, + cwd, + comm, + blocked_signals, + signal_actions, + alternate_signal_stack, + registers, + thread_pointer, + syscall_entry_point, + set_child_tid, + clear_child_tid, + initial_program_break, + program_break, + regions, + fds, + } = startup; + let pid = self.0.process_id; + if syscall_entry_point != self.0.platform.get_syscall_entry_point() { + return Err(ForkRestoreError::IncompatibleRunner); + } + + Task::restore_fork_image(&self.0.mm, ®ions, load_image)?; + self.0 + .mm + .restore_program_break(initial_program_break, program_break); + + let files = syscalls::file::FilesState::new(); + files.set_max_fd(syscalls::process::RLIMIT_NOFILE_CUR); + files + .install_forked_fds(&self.0, &fds) + .map_err(ForkRestoreError::Descriptors)?; + let credentials = Arc::new(syscalls::process::Credentials { + uid, + euid, + gid, + egid, + }); + let fs_state = syscalls::file::FsState::new(&credentials); + fs_state + .inherit_cwd(&cwd) + .map_err(ForkRestoreError::WorkingDirectory)?; + fs_state.set_umask(umask); + + let thread = syscalls::process::ThreadState::new_process(pid); + thread.set_forked_init_state(registers, thread_pointer, set_child_tid, clear_child_tid); + let entrypoints = crate::LinuxShimEntrypoints { + _not_send: core::marker::PhantomData, + task: Task { + global: self.0.clone(), + litebox_thread: Cell::new(Some(initial_thread)), + thread, + wait_state: wait::WaitState::new(self.0.platform), + vfork: RefCell::new(None), + pid, + ppid: parent_process_id, + credentials, + comm: comm.into(), + fs: Arc::new(fs_state).into(), + files: Arc::new(files).into(), + signals: syscalls::signal::SignalState::forked( + blocked_signals, + &signal_actions, + alternate_signal_stack, + ), + }, + }; + // The broker starts each process without child reaping. + if entrypoints.task.signals.reaps_children() { + entrypoints.task.set_child_reaping(true); + } + entrypoints.task.open_signals(); + let process = LinuxShimProcess(entrypoints.task.process().clone()); + Ok(LoadedProgram { + entrypoints, + process, + }) + } + /// Returns the global memory manager. pub fn memory_manager(&self) -> &MemoryManager { &self.0.mm @@ -519,6 +640,27 @@ pub struct LoadedProgram { pub process: LinuxShimProcess, } +/// A failure to continue a process duplicated by `fork`. +#[cfg(target_arch = "x86_64")] +#[derive(Debug, thiserror::Error)] +pub enum ForkRestoreError { + /// The runner's code addresses differ from the parent's runner. + #[error("the runner's address layout differs from the parent's")] + IncompatibleRunner, + /// A memory region could not be restored at its address. + #[error("failed to restore the memory region at {0:#x}")] + Memory(usize, #[source] MappingError), + /// The process image could not be read. + #[error("failed to read the process image")] + Image(#[source] Errno), + /// The inherited descriptors could not be installed. + #[error("failed to install the inherited descriptors")] + Descriptors(#[source] loader::elf::ElfLoaderError), + /// The working directory could not be restored. + #[error("failed to restore the working directory")] + WorkingDirectory(#[source] Errno), +} + /// A handle to a process loaded via [`LinuxShim::load_program`]. /// /// This can be used to wait for the process to exit. @@ -1252,6 +1394,16 @@ impl Task { } SyscallRequest::Clone { args } => self.sys_clone(ctx, &args), SyscallRequest::Clone3 { args } => self.sys_clone3(ctx, args), + SyscallRequest::Fork => { + #[cfg(target_arch = "x86_64")] + { + self.sys_fork(ctx) + } + #[cfg(not(target_arch = "x86_64"))] + { + Err(Errno::ENOSYS) + } + } SyscallRequest::Vfork => { #[cfg(target_arch = "x86_64")] { diff --git a/litebox_shim_linux/src/loader/elf.rs b/litebox_shim_linux/src/loader/elf.rs index 2793d606b8..40ea97b598 100644 --- a/litebox_shim_linux/src/loader/elf.rs +++ b/litebox_shim_linux/src/loader/elf.rs @@ -588,6 +588,8 @@ mod tests { fn initial_stack_is_placed_top_down() { let task = crate::syscalls::tests::init_platform(); let addr_max = >::TASK_ADDR_MAX; + let placement_max = + >::PLACEMENT_ADDR_MAX; crate::syscalls::tests::create_file(&task, "/stack-pie", &minimal_elf(ET_DYN, None)); let mut elf_loader = @@ -606,6 +608,11 @@ mod tests { load_info.user_stack_top, addr_max / 2, ); + assert!( + load_info.user_stack_top <= placement_max, + "initial stack ended at {:#x}, above the placement limit {placement_max:#x}", + load_info.user_stack_top, + ); } #[test] diff --git a/litebox_shim_linux/src/syscalls/file.rs b/litebox_shim_linux/src/syscalls/file.rs index 873b7ea2e4..26024d0999 100644 --- a/litebox_shim_linux/src/syscalls/file.rs +++ b/litebox_shim_linux/src/syscalls/file.rs @@ -26,7 +26,7 @@ use litebox_common_linux::{ FileDescriptorFlags, FileStat, InodeType, IoReadVec, IoWriteVec, IoctlArg, OFlags, Statx, StatxMask, TimeParam, errno::Errno, - program_startup::{InheritedFd, InheritedFdKind}, + program_startup::{ForkedFd, InheritedFd, InheritedFdKind}, signal::Signal, vmem::PAGE_SIZE, }; @@ -328,6 +328,25 @@ impl FilesState { Ok(()) } + /// Installs the descriptors a process duplicated by `fork` inherited, with their + /// close-on-exec flags. + #[cfg(target_arch = "x86_64")] + pub(crate) fn install_forked_fds( + &self, + global: &GlobalState, + forked_fds: &[ForkedFd], + ) -> Result<(), crate::loader::elf::ElfLoaderError> { + let inherited_fds: alloc::vec::Vec<_> = forked_fds.iter().map(|fd| fd.inherited).collect(); + self.install_inherited_fds(global, &inherited_fds)?; + for forked in forked_fds.iter().filter(|fd| fd.close_on_exec) { + let fd = self + .typed_fd_from_raw(forked.inherited.fd as usize) + .expect("an installed inherited descriptor stays open"); + set_file_descriptor_flags(&fd, global, FileDescriptorFlags::FD_CLOEXEC); + } + Ok(()) + } + /// Installs an inherited descriptor at `raw_fd`, duplicating the descriptor at `first_fd` if /// it shares an already installed open file description and calling `adopt` otherwise. fn install_inherited_fd( @@ -550,9 +569,33 @@ impl Task { /// close-on-exec, in ascending order. /// /// Each is returned as its startup record, whose handle the caller replaces with the child's, - /// and the object the child inherits. + /// and the object the child inherits. This fails like [`Self::inheritable_fds`]. + pub(crate) fn fds_inherited_across_exec( + &self, + ) -> Result)>, Errno> { + Ok(self + .inheritable_fds(false)? + .into_iter() + .map(|(fd, object)| (fd.inherited, object)) + .collect()) + } + + /// Returns every descriptor, with its close-on-exec flag, in ascending order, as a fresh + /// runner continuing a `fork` child inherits them. + /// + /// Each is returned as its startup record, whose handle the caller replaces with the child's, + /// and the object the child inherits. This fails like [`Self::inheritable_fds`]. + #[cfg(target_arch = "x86_64")] + pub(crate) fn fds_inherited_across_fork( + &self, + ) -> Result)>, Errno> { + self.inheritable_fds(true) + } + + /// Returns the descriptors a fresh runner inherits, skipping those marked close-on-exec + /// unless `include_close_on_exec` is set. /// - /// Linux keeps every such descriptor, whatever it refers to. A fresh runner can share only + /// Linux keeps every inherited descriptor, whatever it refers to. A fresh runner can share only /// objects the broker holds, and not all of those yet, so this fails with `EAGAIN` if one of /// them is: /// @@ -566,9 +609,10 @@ impl Task { /// /// Transferred descriptors also differ from Linux in that the parent does not observe the /// child's reads of an inherited directory. - pub(crate) fn fds_inherited_across_exec( + fn inheritable_fds( &self, - ) -> Result)>, Errno> { + include_close_on_exec: bool, + ) -> Result)>, Errno> { let files = self.files.borrow(); let alive_fds: alloc::vec::Vec = files.raw_descriptor_store.read().iter_alive().collect(); @@ -576,9 +620,9 @@ impl Task { let mut next_fd = 0; for raw_fd in alive_fds { let fd = files.typed_fd_from_raw(raw_fd)?; - if get_file_descriptor_flags(&fd, &self.global) - .contains(FileDescriptorFlags::FD_CLOEXEC) - { + let close_on_exec = get_file_descriptor_flags(&fd, &self.global) + .contains(FileDescriptorFlags::FD_CLOEXEC); + if close_on_exec && !include_close_on_exec { continue; } if raw_fd - next_fd > MAX_INHERITED_FD_GAP { @@ -599,10 +643,13 @@ impl Task { _ => return Err(Errno::EAGAIN), }; next_fd = raw_fd + 1; - let inherited_fd = InheritedFd { - fd: u32::try_from(raw_fd).map_err(|_| Errno::EAGAIN)?, - handle: ObjectHandle::default(), - kind, + let inherited_fd = ForkedFd { + inherited: InheritedFd { + fd: u32::try_from(raw_fd).map_err(|_| Errno::EAGAIN)?, + handle: ObjectHandle::default(), + kind, + }, + close_on_exec, }; inherited.push((inherited_fd, object)); } diff --git a/litebox_shim_linux/src/syscalls/mm.rs b/litebox_shim_linux/src/syscalls/mm.rs index 3c9764e3d1..cdc9bf8142 100644 --- a/litebox_shim_linux/src/syscalls/mm.rs +++ b/litebox_shim_linux/src/syscalls/mm.rs @@ -78,7 +78,7 @@ fn finalize_trampoline_gates( } } -fn prot_flags_from_permissions(permissions: MemoryRegionPermissions) -> ProtFlags { +pub(super) fn prot_flags_from_permissions(permissions: MemoryRegionPermissions) -> ProtFlags { let mut prot = ProtFlags::PROT_NONE; prot.set( ProtFlags::PROT_READ, @@ -383,19 +383,23 @@ impl Task { // temporary buffer to read the data from fs (without worrying page // faults) and write it to the user buffer with page fault handling. let mut file_offset = offset; - let mut buffer = [0; PAGE_SIZE]; + // Read in the largest chunks the broker transfers, as each read is a round trip. + let max_read = usize::try_from(litebox_broker_protocol::fs::MAX_FILE_TRANSFER_SIZE) + .expect("the broker transfer size fits usize"); + let mut buffer = alloc::vec![0; len.min(max_read)]; let mut copied = 0; while copied < len { - let size = - self.do_read(fd, &mut buffer, Some(file_offset)) - .map_err(|e| match e { - // The raw fd was resolved once at syscall entry and is intentionally - // not retained; this payload is discarded when converted to EBADF. - Errno::EBADF => MappingError::BadFD(-1), - Errno::EISDIR => MappingError::NotAFile, - Errno::EACCES => MappingError::NotForReading, - _ => unimplemented!(), - })?; + let want = (len - copied).min(buffer.len()); + let size = self + .do_read(fd, &mut buffer[..want], Some(file_offset)) + .map_err(|e| match e { + // The raw fd was resolved once at syscall entry and is intentionally + // not retained; this payload is discarded when converted to EBADF. + Errno::EBADF => MappingError::BadFD(-1), + Errno::EISDIR => MappingError::NotAFile, + Errno::EACCES => MappingError::NotForReading, + _ => unimplemented!(), + })?; if size == 0 { break; } @@ -2316,6 +2320,134 @@ mod tests { task.sys_munmap(new_addr, 0x2000).unwrap(); } + #[test] + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + fn self_placed_mappings_stay_below_placement_limit() { + let task = init_platform(); + let limit = >::PLACEMENT_ADDR_MAX; + assert!(limit < >::TASK_ADDR_MAX); + let below_limit = |address: UserPtrMut, len: usize| { + assert!( + address.as_usize() + len <= limit, + "mapping at {:#x} crosses the placement limit {limit:#x}", + address.as_usize(), + ); + }; + let read_write = ProtFlags::PROT_READ | ProtFlags::PROT_WRITE; + let anonymous = MapFlags::MAP_ANON | MapFlags::MAP_PRIVATE; + + let unhinted = task + .sys_mmap(0, 2 * PAGE_SIZE, read_write, anonymous, -1, 0) + .unwrap(); + below_limit(unhinted, 2 * PAGE_SIZE); + + let hinted = task + .sys_mmap(limit, PAGE_SIZE, read_write, anonymous, -1, 0) + .unwrap(); + below_limit(hinted, PAGE_SIZE); + + // Native remapping must honor the destination vmem chose instead of letting the host + // kernel pick one, and must still carry the contents along. + unhinted + .write_slice_at_offset::(0, &[0xa5; 0x10]) + .unwrap(); + let moved = task + .sys_mremap( + unhinted, + PAGE_SIZE, + 4 * PAGE_SIZE, + MRemapFlags::MREMAP_MAYMOVE, + 0, + ) + .unwrap(); + below_limit(moved, 4 * PAGE_SIZE); + assert_eq!(moved.read_at_offset::(0).unwrap(), 0xa5_u8); + + let fixed = task + .sys_mmap( + limit, + PAGE_SIZE, + read_write, + anonymous | MapFlags::MAP_FIXED_NOREPLACE, + -1, + 0, + ) + .unwrap(); + assert_eq!(fixed.as_usize(), limit); + + for (address, len) in [ + (unhinted, 2 * PAGE_SIZE), + (hinted, PAGE_SIZE), + (moved, 4 * PAGE_SIZE), + (fixed, PAGE_SIZE), + ] { + task.sys_munmap(address, len).unwrap(); + } + } + + #[test] + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + fn repeated_moving_growth_keeps_contents() { + const PAGE: isize = PAGE_SIZE.cast_signed(); + let task = init_platform(); + let read_write = ProtFlags::PROT_READ | ProtFlags::PROT_WRITE; + let anonymous = MapFlags::MAP_ANON | MapFlags::MAP_PRIVATE; + // Occupies the page after a mapping so that growing it has to move it. + let block_after = |address: UserPtrMut, len: usize| { + task.sys_mmap( + address.as_usize() + len, + PAGE_SIZE, + ProtFlags::PROT_NONE, + anonymous | MapFlags::MAP_FIXED_NOREPLACE, + -1, + 0, + ) + .ok() + }; + + let first = task + .sys_mmap(0, 2 * PAGE_SIZE, read_write, anonymous, -1, 0) + .unwrap(); + first + .write_slice_at_offset::(0, &[0xa5; 0x10]) + .unwrap(); + let first_block = block_after(first, 2 * PAGE_SIZE); + let second = task + .sys_mremap( + first, + 2 * PAGE_SIZE, + 4 * PAGE_SIZE, + MRemapFlags::MREMAP_MAYMOVE, + 0, + ) + .unwrap(); + assert_ne!(second.as_usize(), first.as_usize()); + second + .write_slice_at_offset::(3 * PAGE, &[0x5a; 0x10]) + .unwrap(); + + // The moved pages and the grown tail are separate host mappings that must move together. + let second_block = block_after(second, 4 * PAGE_SIZE); + let third = task + .sys_mremap( + second, + 4 * PAGE_SIZE, + 8 * PAGE_SIZE, + MRemapFlags::MREMAP_MAYMOVE, + 0, + ) + .unwrap(); + assert_ne!(third.as_usize(), second.as_usize()); + assert_eq!(third.read_at_offset::(0).unwrap(), 0xa5_u8); + assert_eq!(third.read_at_offset::(3 * PAGE).unwrap(), 0x5a_u8); + assert_eq!(third.read_at_offset::(5 * PAGE).unwrap(), 0_u8); + + task.sys_munmap(third, 8 * PAGE_SIZE).unwrap(); + for block in [first_block, second_block].into_iter().flatten() { + task.sys_munmap(block, PAGE_SIZE).unwrap(); + } + } + #[test] #[cfg_attr( target_os = "macos", diff --git a/litebox_shim_linux/src/syscalls/process.rs b/litebox_shim_linux/src/syscalls/process.rs index 36980e45db..4921bc33b9 100644 --- a/litebox_shim_linux/src/syscalls/process.rs +++ b/litebox_shim_linux/src/syscalls/process.rs @@ -20,13 +20,25 @@ use litebox::event::polling::{Pollee, TryOpError}; use litebox::event::wait::WaitError; use litebox::event::{Events, IOPollable as _}; use litebox::platform::ArchSpecificRegister; +#[cfg(target_arch = "x86_64")] +use litebox::platform::RawConstPointer as _; use litebox::platform::TimerHandle; use litebox::process::{ChildStatus, ProcessError}; use litebox::sync::{Mutex, RwLock}; use litebox::utils::TruncateExt as _; +#[cfg(target_arch = "x86_64")] +use litebox_broker_protocol::process::MAX_CHILD_MEMORY_WRITE_SIZE; use litebox_broker_protocol::process::ProcessExitStatus; +#[cfg(target_arch = "x86_64")] +use litebox_common_linux::ProtFlags; +#[cfg(target_arch = "x86_64")] +use litebox_common_linux::program_startup::{ForkMemoryRegion, LinuxForkStartup}; use litebox_common_linux::signal::{CLD_EXITED, Signal}; use litebox_common_linux::vmem::VmFlags; +#[cfg(target_arch = "x86_64")] +use litebox_common_linux::vmem::{ + CreatePagesFlags, MappingError, NonZeroAddress, NonZeroPageSize, PAGE_SIZE, +}; use litebox_common_linux::{ ArchPrctlArg, CloneFlags, FutexArgs, IntervalTimer, ItimerVal, PrctlArg, Rusage, TimeParam, errno::Errno, program_startup::LinuxProgramStartup, @@ -84,6 +96,27 @@ impl ThreadState { }) } + /// Prepares the initial thread of a process duplicated by `fork` to return zero from the + /// `fork` with the rest of `registers` and `thread_pointer` as its FS base, storing its thread + /// ID at `set_child_tid` and clearing `clear_child_tid` when it exits, unless they are zero. + #[cfg(target_arch = "x86_64")] + pub(crate) fn set_forked_init_state( + &self, + mut registers: litebox_common_linux::PtRegs, + thread_pointer: usize, + set_child_tid: usize, + clear_child_tid: usize, + ) { + registers.rax = 0; + let user_ptr = |address: usize| (address != 0).then(|| UserPtrMut::from_usize(address)); + self.init_state.set(ThreadInitState::Forked { + registers: Box::new(registers), + thread_pointer, + set_child_tid: user_ptr(set_child_tid), + }); + self.clear_child_tid.set(user_ptr(clear_child_tid)); + } + pub(crate) fn begin_detach_from_process(&self) -> Option> { let tid = self.tid.take()?; self.process.remove_thread(tid); @@ -134,6 +167,26 @@ impl Drop for ThreadDetachGuard<'_, Platform> { } } +/// Keeps the other threads of a process paused for `fork` until dropped. +#[cfg(target_arch = "x86_64")] +struct ForkPause<'a, Platform: ShimPlatform> { + process: &'a ProcessState, +} + +#[cfg(target_arch = "x86_64")] +impl Drop for ForkPause<'_, Platform> { + fn drop(&mut self) { + let mut inner = self.process.inner.lock(); + inner.pausing_thread = None; + self.process + .fork_pause + .underlying_atomic() + .store(0, Ordering::Release); + drop(inner); + self.process.fork_pause.wake_all(); + } +} + /// Thread state that can be accessed from a remote thread. struct ThreadRemote { /// Always set under the process `inner` lock, but can be read without @@ -163,6 +216,9 @@ pub(crate) struct ProcessState { /// Number of threads in this process. Always updated under the `inner` /// mutex lock. nr_threads: ::RawMutex, + /// One while a thread pauses the others for `fork`, which block on it. Always updated under + /// the `inner` mutex lock. + fork_pause: ::RawMutex, inner: Arc>>, /// Started child processes that have not been reaped, mapped by process ID. children: Mutex>>, @@ -292,6 +348,11 @@ struct ProcessInner { exit_status: ExitStatus, /// The thread list for the process, mapped by thread ID. threads: BTreeMap>>, + /// The thread pausing the others for `fork`, if any. + pausing_thread: Option>>, + /// Number of threads paused for `fork`, which each count themselves until no thread pauses + /// them. + paused_threads: u32, } #[derive(Clone, Copy, Debug)] @@ -310,9 +371,12 @@ impl ProcessState { group_exit: false, is_killing_other_threads: false, threads: BTreeMap::from_iter([(pid, remote)]), + pausing_thread: None, + paused_threads: 0, })); Self { nr_threads, + fork_pause: ::RawMutex::INIT, child_events: Arc::new(ChildEvents { changed: AtomicBool::new(false), pollee: Pollee::new(), @@ -494,6 +558,10 @@ impl ProcessState { // The last thread exited. Prevent new threads. inner.group_exit = true; } + // A thread pausing the others for `fork` waits for this one to pause or detach. + if let Some(pausing_thread) = &inner.pausing_thread { + pausing_thread.interrupt(); + } // Notify waiters if this is the last thread of the process // (`wait_for_exit`) or if this is the last thread being killed @@ -689,6 +757,77 @@ impl Task { true } + /// Blocks while another thread pauses this process's other threads for `fork`. + /// + /// Threads pause as they start to wait or are about to run guest code, where they hold no + /// state the pausing thread snapshots. + pub(crate) fn wait_while_paused(&self) { + let process = &self.thread.process; + { + let mut inner = process.inner.lock(); + match &inner.pausing_thread { + Some(pausing_thread) if !Arc::ptr_eq(pausing_thread, &self.thread.remote) => { + pausing_thread.interrupt(); + } + _ => return, + } + inner.paused_threads += 1; + } + // Stay paused, and counted, until no thread pauses the others, even if another thread + // starts pausing them as soon as this one stops. + loop { + let _ = process.fork_pause.block(1); + let mut inner = process.inner.lock(); + if inner.pausing_thread.is_none() { + inner.paused_threads -= 1; + return; + } + } + } + + /// Pauses this process's other threads until the returned guard drops, so they neither + /// change the memory nor the state `fork` snapshots. + /// + /// Fails with `ERESTARTNOINTR` if a signal or an exit interrupts the wait for them to pause, + /// so `fork` restarts once the signal is handled, as Linux's does if a signal arrives while + /// it duplicates the process. + #[cfg(target_arch = "x86_64")] + fn pause_other_threads(&self) -> Result, Errno> { + let process = &self.thread.process; + loop { + let mut inner = process.inner.lock(); + if inner.pausing_thread.is_none() { + inner.pausing_thread = Some(self.thread.remote.clone()); + process + .fork_pause + .underlying_atomic() + .store(1, Ordering::Release); + for (&tid, thread) in &inner.threads { + if tid != self.tid() { + thread.interrupt(); + } + } + break; + } + drop(inner); + // Another thread pauses this one for its own `fork`. + self.wait_while_paused(); + } + let pause = ForkPause { process }; + // Each thread that pauses or detaches interrupts this wait to reevaluate it. + self.wait_cx() + .wait_until(|| { + let inner = process.inner.lock(); + inner.paused_threads + 1 == process.nr_threads() + }) + .map_err(|_| Errno::ERESTARTNOINTR)?; + // The other threads may have paused as they exit the whole process. + if self.is_exiting() { + return Err(Errno::ERESTARTNOINTR); + } + Ok(pause) + } + /// Transfers a surviving nonleader exec caller to the process leader /// identity while retaining its broker thread ownership. fn rebind_exec_identity(&self) { @@ -725,6 +864,12 @@ enum ThreadInitState { tls: Option, set_child_tid: Option>, }, + #[cfg(target_arch = "x86_64")] + Forked { + registers: Box, + thread_pointer: usize, + set_child_tid: Option>, + }, } /// Credentials of a process @@ -1041,6 +1186,253 @@ impl Task { self.begin_vfork(ctx, None, false) } + /// Handle syscall `fork`. + #[cfg(target_arch = "x86_64")] + pub(crate) fn sys_fork(&self, ctx: &litebox_common_linux::PtRegs) -> Result { + self.fork(ctx, 0, 0) + } + + /// Duplicates this process in a fresh runner, which continues the child at the parent's + /// addresses, as `fork`, or `clone` or `clone3` without `CLONE_VM`, do. + /// + /// The child stores its thread ID at `set_child_tid` and clears `clear_child_tid` when it + /// exits, unless they are zero. + /// + /// The child gets a copy of the parent's memory, program break, registers, and FS base, of + /// its credentials, command name, working directory, and umask, of its descriptors, sharing + /// their open file descriptions, and of its signal dispositions, blocked mask, and alternate + /// stack, but none of its pending signals, timers, or robust futex list. Memory regions + /// formerly backed by files are copied as anonymous memory, and `MADV_DONTFORK` regions are + /// copied too. Floating-point and vector state is not copied yet, because x86-64 platforms + /// cannot capture a guest thread's vector state until they implement + /// [`GuestVectorStateProvider`](litebox::platform::GuestVectorStateProvider) instead of + /// using its no-op default. Like Linux, the child has a single thread, a copy of the calling + /// one; the parent's other threads pause while the parent is copied. + /// + /// Only a process outside a `vfork` window, with default resource-limit and alarm state, no + /// shared memory mappings, no ELF file mid-load, and only descriptors a fresh runner can + /// inherit is duplicated; otherwise this fails with `EAGAIN`. + #[cfg(target_arch = "x86_64")] + fn fork( + &self, + ctx: &litebox_common_linux::PtRegs, + set_child_tid: usize, + clear_child_tid: usize, + ) -> Result { + if self.vfork.borrow().is_some() { + return Err(Errno::EAGAIN); + } + let pause = self.pause_other_threads()?; + if !self.thread.process.limits.has_default_state() + || !self.thread.process.has_default_alarm_state() + || !self.global.elf_patch_cache.lock().is_empty() + { + return Err(Errno::EAGAIN); + } + let mut regions = Vec::new(); + for (range, flags) in self.global.mm.mappings() { + // Mappings without flags are reserved by the runner, which reserves its own. + if flags.is_empty() { + continue; + } + if flags.contains(VmFlags::VM_SHARED) { + return Err(Errno::EAGAIN); + } + regions.push(ForkMemoryRegion { range, flags }); + } + let (fds, objects): (Vec<_>, Vec<_>) = + self.fds_inherited_across_fork()?.into_iter().unzip(); + let fs = self.fs.borrow(); + let cwd = alloc::string::ToString::to_string(fs.context.read().cwd()); + let umask = u32::from(fs.umask().bits()); + drop(fs); + let (blocked_signals, signal_actions, alternate_signal_stack) = self.signals.fork_state(); + let (initial_program_break, program_break) = self.global.mm.program_break(); + let mut startup = LinuxForkStartup { + parent_process_id: self.pid, + uid: self.credentials.uid, + euid: self.credentials.euid, + gid: self.credentials.gid, + egid: self.credentials.egid, + umask, + cwd, + comm: self.comm.get(), + blocked_signals, + signal_actions, + alternate_signal_stack, + registers: ctx.clone(), + thread_pointer: self + .global + .platform + .get_arch_specific_register(&ArchSpecificRegister::FsBase)?, + syscall_entry_point: self.global.platform.get_syscall_entry_point(), + set_child_tid, + clear_child_tid, + initial_program_break, + program_break, + regions, + fds, + }; + // The child keeps the references it inherits until it exits, so check the size first. + startup.encode().map_err(|_| Errno::ENOMEM)?; + + // Release the children the broker reaped, which hold process capacity until observed. + let _ = self.observe_child_terminations(); + let child = self + .global + .litebox + .allocate_child_process() + .map_err(Errno::from)?; + let child_pid = i32::try_from(child.identity().process_id.0) + .expect("broker process IDs must fit Linux pid_t"); + let prepared = self + .write_fork_image(&child, &startup.regions) + .and_then(|()| { + child + .inherit(&self.global.litebox, &objects) + .map_err(Errno::from) + }); + let handles = match prepared { + Ok(handles) => handles, + Err(errno) => { + // The pending child never runs, and no one observes its status. + let _ = child.exit(ProcessExitStatus::Unknown); + return Err(errno); + } + }; + for (forked, handle) in startup.fds.iter_mut().zip(handles) { + forked.inherited.handle = handle; + } + // The child's memory and objects are copied, so the other threads may go on. + drop(pause); + let payload = startup + .encode() + .expect("the startup fit with placeholder handles"); + // The broker no longer holds a pending child if starting it fails. + child.start(&payload).map_err(Errno::from)?; + self.thread.process.add_child(child_pid, child); + // Failure means the process service failed, so no termination can be observed. + let _ = self.observe_child_terminations(); + Ok(child_pid.cast_unsigned() as usize) + } + + /// Writes the contents of each of `regions` that has contents to the pending `child`'s + /// process image, back to back, skipping zero-filled pages at the ends of each chunk. + #[cfg(target_arch = "x86_64")] + fn write_fork_image( + &self, + child: &litebox::process::Process, + regions: &[ForkMemoryRegion], + ) -> Result<(), Errno> { + const CHUNK_SIZE: usize = MAX_CHILD_MEMORY_WRITE_SIZE as usize; + // Folding a page vectorizes, unlike stopping at its first nonzero byte. + let has_data = |page: &[u8]| page.iter().fold(0, |acc, &byte| acc | byte) != 0; + let mut image_offset = 0u64; + for region in regions.iter().filter(|region| region.has_contents()) { + let write_region = || -> Result<(), Errno> { + for start in region.range.clone().step_by(CHUNK_SIZE) { + let len = CHUNK_SIZE.min(region.range.end - start); + let chunk = UserPtr::::from_usize(start) + .to_owned_slice::(len) + .ok_or(Errno::ENOMEM)?; + let Some(first) = chunk.chunks(PAGE_SIZE).position(has_data) else { + continue; + }; + let last = chunk.chunks(PAGE_SIZE).rposition(has_data).unwrap(); + let data = &chunk[first * PAGE_SIZE..(last + 1) * PAGE_SIZE]; + let offset = + image_offset + (start - region.range.start + first * PAGE_SIZE) as u64; + child.write_memory(offset, data).map_err(Errno::from)?; + } + Ok(()) + }; + // Regions without read access, such as inaccessible ones that may hold data and + // execute-only ones that host protection keys can make unreadable, are readable only + // while copied. + if region.flags.contains(VmFlags::VM_READ) { + write_region()?; + } else { + let start = UserPtrMut::from_usize(region.range.start); + let len = region.range.len(); + let prot = super::mm::prot_flags_from_permissions(region.flags.into()); + self.sys_mprotect_raw(start, len, prot | ProtFlags::PROT_READ)?; + let written = write_region(); + self.sys_mprotect_raw(start, len, prot)?; + written?; + } + image_offset += region.range.len() as u64; + } + Ok(()) + } + + /// Maps each of `regions` at its address in `mm`, filling those that have contents from the + /// process image [`Self::write_fork_image`] wrote, which `load_image` loads from an offset + /// into a region's pages. + /// + /// Restoring fails if any region's address is in use. + #[cfg(target_arch = "x86_64")] + pub(crate) fn restore_fork_image( + mm: &crate::MemoryManager, + regions: &[ForkMemoryRegion], + mut load_image: impl FnMut(u64, &mut [u8]) -> Result<(), Errno>, + ) -> Result<(), crate::ForkRestoreError> { + let mut image_offset = 0u64; + for region in regions { + let (Some(address), Some(length)) = ( + NonZeroAddress::::new(region.range.start), + NonZeroPageSize::::new(region.range.len()), + ) else { + return Err(crate::ForkRestoreError::Memory( + region.range.start, + MappingError::UnAligned, + )); + }; + let mut flags = CreatePagesFlags::FIXED_ADDR | CreatePagesFlags::NOREPLACE; + if region.flags.contains(VmFlags::VM_GROWSDOWN) { + flags |= CreatePagesFlags::IS_STACK; + } + let has_contents = region.has_contents(); + let mut image_error = None; + let created = if has_contents { + // SAFETY: `NOREPLACE` fails rather than replacing any existing mapping. + unsafe { + mm.create_pages_with_permissions( + Some(address), + length, + flags, + region.flags.into(), + |pages| { + // SAFETY: The pages were just mapped readable and writable, and + // nothing else uses them until this function returns. + let pages = core::slice::from_raw_parts_mut( + pages.as_usize() as *mut u8, + length.as_usize(), + ); + if let Err(error) = load_image(image_offset, pages) { + image_error = Some(error); + return Err(MappingError::OutOfMemory); + } + Ok(0) + }, + ) + } + } else { + // A region without contents is never accessible, so creating it inaccessible + // keeps it marked as holding no data. + // SAFETY: `NOREPLACE` fails rather than replacing any existing mapping. + unsafe { mm.create_inaccessible_pages(Some(address), length, flags, |_| Ok(0)) } + }; + created.map_err(|error| match image_error { + Some(error) => crate::ForkRestoreError::Image(error), + None => crate::ForkRestoreError::Memory(region.range.start, error), + })?; + if has_contents { + image_offset += length.as_usize() as u64; + } + } + Ok(()) + } + /// Begins a constrained `vfork` child in the current runner, as `vfork`, or `clone` or /// `clone3` with `CLONE_VM | CLONE_VFORK`, do. /// @@ -1162,6 +1554,31 @@ impl Task { None }; + #[cfg(target_arch = "x86_64")] + if !flags.intersects(CloneFlags::VM | CloneFlags::THREAD | CloneFlags::VFORK) { + let supported_fork_flags = CloneFlags::CHILD_SETTID | CloneFlags::CHILD_CLEARTID; + if flags.intersects(!supported_fork_flags) { + log_unsupported!("fork clone with flags: {:?}", flags); + return Err(Errno::EINVAL); + } + let sigchld = litebox_common_linux::signal::Signal::SIGCHLD.as_i32(); + if exit_signal != u64::from(sigchld.cast_unsigned()) { + log_unsupported!("fork clone with exit signal {exit_signal}"); + return Err(Errno::EINVAL); + } + if sp.is_some() { + log_unsupported!("fork clone with a stack"); + return Err(Errno::EINVAL); + } + let child_tid: usize = child_tid.trunc(); + let tid_if = |flag| if flags.contains(flag) { child_tid } else { 0 }; + return self.fork( + ctx, + tid_if(CloneFlags::CHILD_SETTID), + tid_if(CloneFlags::CHILD_CLEARTID), + ); + } + #[cfg(target_arch = "x86_64")] if flags.contains(CloneFlags::VFORK) { let supported_vfork_flags = @@ -2457,6 +2874,19 @@ impl Task { let _ = child_tid_ptr.write_at_offset::(0, self.tid()); } } + #[cfg(target_arch = "x86_64")] + ThreadInitState::Forked { + registers, + thread_pointer, + set_child_tid, + } => { + *ctx = *registers; + self.sys_arch_prctl(ArchPrctlArg::SetFs(thread_pointer)) + .expect("failed to restore the forked thread's FS base"); + if let Some(child_tid_ptr) = set_child_tid { + let _ = child_tid_ptr.write_at_offset::(0, self.tid()); + } + } } } } @@ -3172,7 +3602,12 @@ mod tests { ) .expect("rt_sigprocmask failed"); }; - for restart in [None, Some(SyscallRestart::Sys), Some(SyscallRestart::NoHandler)] { + for restart in [ + None, + Some(SyscallRestart::Sys), + Some(SyscallRestart::NoIntr), + Some(SyscallRestart::NoHandler), + ] { // SIGCHLD is ignored by default, but is queued while blocked. Once unblocked, // it interrupts waits, yet runs no handler. sigprocmask(SigmaskHow::SIG_BLOCK); diff --git a/litebox_shim_linux/src/syscalls/signal/mod.rs b/litebox_shim_linux/src/syscalls/signal/mod.rs index da5c6ccff1..af151229df 100644 --- a/litebox_shim_linux/src/syscalls/signal/mod.rs +++ b/litebox_shim_linux/src/syscalls/signal/mod.rs @@ -129,6 +129,40 @@ impl SignalState { state } + /// Returns the initial signal state of a process duplicated by `fork`, which inherits the + /// `blocked` mask, the dispositions in `actions`, indexed by signal number minus one, and + /// the alternate signal stack, but none of the pending signals. + #[cfg(target_arch = "x86_64")] + pub fn forked(blocked: SigSet, actions: &[SigAction; NSIG], altstack: SigAltStack) -> Self { + let mut state = Self::new_process(); + state.set_signal_mask(blocked); + let handlers = Arc::get_mut(state.handlers.get_mut()) + .expect("new signal handlers must not be shared") + .inner + .get_mut(); + for (handler, action) in handlers.handlers.iter_mut().zip(actions) { + // The actions of SIGKILL and SIGSTOP cannot change. + if !handler.immutable { + handler.action = *action; + } + } + state.altstack.set(altstack); + state + } + + /// Returns the blocked mask, the dispositions indexed by signal number minus one, and the + /// alternate signal stack, which a child duplicated by `fork` inherits. + #[cfg(target_arch = "x86_64")] + pub(crate) fn fork_state(&self) -> (SigSet, [SigAction; NSIG], SigAltStack) { + let handlers = self.handlers.borrow(); + let handlers = handlers.inner.lock(); + ( + self.blocked.get(), + core::array::from_fn(|index| handlers.handlers[index].action), + self.altstack.get(), + ) + } + /// Returns the currently blocked signals. pub(crate) fn blocked(&self) -> SigSet { self.blocked.get() @@ -559,6 +593,8 @@ struct DeliverFault; pub(crate) enum SyscallRestart { /// From [`Errno::ERESTARTSYS`]: only a handler without `SA_RESTART` interrupts the syscall. Sys, + /// From [`Errno::ERESTARTNOINTR`]: no handler interrupts the syscall. + NoIntr, /// From [`Errno::ERESTARTNOHAND`]: any handler interrupts the syscall. NoHandler, } @@ -568,6 +604,7 @@ impl SyscallRestart { pub(crate) fn from_errno(errno: Errno) -> Option { match errno { Errno::ERESTARTSYS => Some(Self::Sys), + Errno::ERESTARTNOINTR => Some(Self::NoIntr), Errno::ERESTARTNOHAND => Some(Self::NoHandler), _ => None, } @@ -577,6 +614,7 @@ impl SyscallRestart { fn is_interrupted_by(self, action: &SigAction) -> bool { match self { Self::Sys => !action.flags.contains(SaFlags::RESTART), + Self::NoIntr => false, Self::NoHandler => true, } } diff --git a/litebox_shim_linux/src/wait.rs b/litebox_shim_linux/src/wait.rs index 2f49f71f24..38d2817d3a 100644 --- a/litebox_shim_linux/src/wait.rs +++ b/litebox_shim_linux/src/wait.rs @@ -43,6 +43,7 @@ impl Task { restart: Option, ) -> bool { self.wait_state.0.prepare_to_run_guest(|| { + self.wait_while_paused(); self.queue_async_signals(); self.process_signals(ctx, restart); !self.is_exiting() @@ -77,6 +78,7 @@ pub(crate) fn wait_errno(timeout: Option, error: impl Into impl litebox::event::wait::CheckForInterrupt for Task { fn check_for_interrupt(&self) -> bool { + self.wait_while_paused(); self.queue_async_signals(); self.is_exiting() || self.has_pending_signals() }