diff --git a/dev_tests/src/ratchet.rs b/dev_tests/src/ratchet.rs index 94aa8c11b0..11ff1f42d3 100644 --- a/dev_tests/src/ratchet.rs +++ b/dev_tests/src/ratchet.rs @@ -41,7 +41,7 @@ fn ratchet_pub_uses() -> Result<()> { ("litebox_broker_local_userland/", 2), ("litebox_broker_platform_linux_userland/", 3), ("litebox_broker_platform_windows_userland/", 1), - ("litebox_broker_transport_linux_userland/", 3), + ("litebox_broker_transport_linux_userland/", 2), ("litebox_broker_transport_windows_userland/", 4), ("litebox_common_linux/", 1), ("litebox_common_macos/", 2), diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index 8e526586b1..6a54d944c7 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -17,7 +17,7 @@ use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, UnixStreamLocalSetupChannel, @@ -84,13 +84,14 @@ fn negotiate( BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; - let control_memory = setup.receive_control_ring(Some(setup_deadline))?; + let layout = ControlRingLayout::MULTI_LANE; + let control_memory = setup.receive_control_ring(layout, Some(setup_deadline))?; let process_image = setup.receive_process_image(Some(setup_deadline))?; let positional_io_fds = [ shared_memory.as_fd().as_raw_fd(), control_memory.as_fd().as_raw_fd(), ]; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_ring = ControlRing::new(control_memory, layout).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid broker control ring: {error:?}"), @@ -268,8 +269,8 @@ mod tests { }; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixStreamHostSetupChannel, }; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -318,18 +319,19 @@ mod tests { UnixControlRingLocalCallChannel, UnixControlRingLocalShutdown, UnixControlRingLocalNotificationChannel, - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, ) { - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let layout = ControlRingLayout::MULTI_LANE; + let local_memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); let host_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.as_fd().try_clone_to_owned().unwrap(), + layout, ) .unwrap(); - let local_ring = ControlRing::new(local_memory).unwrap(); - let host_ring = ControlRing::new(host_memory).unwrap(); + let local_ring = ControlRing::new(local_memory, layout).unwrap(); + let host_ring = ControlRing::new(host_memory, layout).unwrap(); let host_activation = std::thread::spawn(move || host_channel.into_active(host_ring).unwrap()); let weak_association_coordinator = Arc::downgrade(association_coordinator); @@ -340,14 +342,12 @@ mod tests { } }) .unwrap(); - let (request_source, response_sink, host_notifications, host_shutdown) = - host_activation.join().unwrap(); + let (host_lanes, host_notifications, host_shutdown) = host_activation.join().unwrap(); ( local_call, local_shutdown, local_notifications, - request_source, - response_sink, + host_lanes, host_notifications, host_shutdown, ) @@ -362,8 +362,7 @@ mod tests { active_channel, association_shutdown, notification_channel, - host_request_source, - host_response_sink, + host_lanes, _host_notifications, host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); @@ -380,8 +379,7 @@ mod tests { .unwrap(); host_shutdown.shutdown().unwrap(); - drop(host_request_source); - drop(host_response_sink); + drop(host_lanes); failure_receiver .recv_timeout(Duration::from_secs(1)) @@ -401,8 +399,7 @@ mod tests { active_channel, association_shutdown, _notification_channel, - mut host_request_source, - _host_response_sink, + mut host_lanes, _host_notifications, _host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); @@ -419,10 +416,9 @@ mod tests { let (failure_sender, failure_receiver) = mpsc::sync_channel(1); association_coordinator.install_dispatch(move || failure_sender.send(()).unwrap()); failure_receiver.try_recv().unwrap(); - assert_eq!( - host_request_source.recv_request().unwrap(), - HostReceive::PeerClosed - ); + // Lane 0 is always demanded, so the broker may hand it out before it + // sees the close. + while host_lanes.next_lane().unwrap().is_some() {} drop(active_channel); } @@ -435,8 +431,7 @@ mod tests { active_channel, association_shutdown, notification_channel, - _host_request_source, - _host_response_sink, + _host_lanes, mut host_notifications, host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index 1f33f25a37..bcdf1423f7 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -10,7 +10,7 @@ use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport_windows_userland::control_ring::{ WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, }; @@ -43,9 +43,10 @@ pub fn connect(control_pipe: &OsStr) -> Result<(BrokerConnection, Option Option { let start: usize = region.start.trunc(); - (region.len >= CONTROL_RING_MEMORY_SIZE as u64 && start.is_multiple_of(size_of::())) + (region.len >= LAYOUT.memory_size() as u64 && start.is_multiple_of(size_of::())) .then_some(Self { start }) } @@ -257,18 +258,18 @@ impl KernelControlRing { } fn u32_at(&self, offset: usize) -> Result<*mut u32, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u32(offset)) + self.word(offset, LAYOUT.permits_u32(offset)) } fn u64_at(&self, offset: usize) -> Result<*mut u64, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u64(offset)) + self.word(offset, LAYOUT.permits_u64(offset)) } fn bytes_at(&self, offset: usize, len: usize) -> Result { - if !MemoryAccessPolicy::ControlRing.permits_byte_range(offset, len) { + if !LAYOUT.permits_byte_range(offset, len) { return Err(SharedMemoryError::InvalidRange); } - checked(self.start, CONTROL_RING_MEMORY_SIZE, offset, len) + checked(self.start, LAYOUT.memory_size(), offset, len) } } @@ -276,7 +277,7 @@ impl KernelControlRing { // process's lifetime and disjoint from private memory. impl SharedMemory for KernelControlRing { fn len(&self) -> usize { - CONTROL_RING_MEMORY_SIZE + LAYOUT.memory_size() } fn read(&self, offset: usize, destination: &mut [u8]) -> Result<(), SharedMemoryError> { @@ -395,12 +396,16 @@ pub fn connect( .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange)) }; let LocalControlRingEndpoints { - request_producer, - response_consumer, + lanes, + lane_demand: _, notification_consumer, - } = ControlRing::new(ring()?) + } = ControlRing::new(ring()?, LAYOUT) .map_err(ChannelError::Ring)? .into_local(); + let [lane] = <[_; 1]>::try_from(lanes) + .unwrap_or_else(|_| unreachable!("a single-lane layout has one lane")); + let (request_producer, response_consumer) = + (lane.request_producer, lane.response_consumer); let notifications = Notifications { consumer: notification_consumer, words: ring()?, @@ -442,7 +447,7 @@ impl Notifications { if received != 0 { return Ok(received); } - let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + let epoch = LAYOUT.notifications().producer_epoch_offset(); match broker_wait(self.words.start, epoch, wait_epoch, deadline) { Ok(()) => {} Err(Status::TimedOut) => return Ok(0), @@ -661,7 +666,7 @@ fn futex( /// Has the kernel serve published broker requests: a wake on the requests' /// producer epoch of the control ring at `start`. fn broker_wake(start: usize) -> Result<(), Status> { - let offset = ControlRingDirection::Requests.producer_epoch_offset(); + let offset = LAYOUT.requests(0).producer_epoch_offset(); futex(start, |addr| FutexRequest::wake(addr, 1), offset) } @@ -691,8 +696,8 @@ fn broker_wait( /// /// On a failed check. pub fn self_check(notifications: &Notifications, platform: &VmUserland) { - let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); - let served = ControlRingDirection::Requests.consumer_epoch_offset(); + let epoch = LAYOUT.notifications().producer_epoch_offset(); + let served = LAYOUT.requests(0).consumer_epoch_offset(); let now = || platform.now(); let current = |offset| { notifications @@ -745,19 +750,19 @@ mod tests { fn ring(memory: &mut [u64]) -> (Notifications, ControlRingProducer) { let region = UserRange { start: memory.as_mut_ptr() as u64, - len: CONTROL_RING_MEMORY_SIZE as u64, + len: LAYOUT.memory_size() as u64, }; let ring = || KernelControlRing::new(region).expect("room for the ring"); let notifications = Notifications { - consumer: ControlRing::new(ring()) + consumer: ControlRing::new(ring(), LAYOUT) .expect("ring") .into_local() .notification_consumer, words: ring(), }; - let producer = ControlRing::new(ring()) + let producer = ControlRing::new(ring(), LAYOUT) .expect("ring") - .into_broker() + .into_host() .notification_producer; (notifications, producer) } @@ -770,7 +775,7 @@ mod tests { } fn ring_memory() -> Vec { - alloc::vec![0u64; CONTROL_RING_MEMORY_SIZE / size_of::()] + alloc::vec![0u64; LAYOUT.memory_size() / size_of::()] } #[test] diff --git a/litebox_broker_transport/Cargo.toml b/litebox_broker_transport/Cargo.toml index 94efa2df8a..63739c585e 100644 --- a/litebox_broker_transport/Cargo.toml +++ b/litebox_broker_transport/Cargo.toml @@ -3,6 +3,10 @@ name = "litebox_broker_transport" version = "0.1.0" edition = "2024" +[features] +# Local call-lane pool. +std = [] + [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs new file mode 100644 index 0000000000..77554c1ffd --- /dev/null +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -0,0 +1,330 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Local-side admission of concurrent calls to call lanes. + +use alloc::string::ToString; +use alloc::vec::Vec; +use core::ops::{Deref, DerefMut}; +use std::io::Error; +use std::sync::{Condvar, Mutex, MutexGuard}; + +/// Admission of concurrent calls to a fixed set of call lanes. +/// +/// A call holds a lane from writing its request until reading its response, +/// so calls on different lanes never wait for each other. Calls claim the +/// lowest free lane, so an association uses only as many lanes as it has +/// concurrent calls; callers beyond the lane count wait for a lane to free up. +/// Lane 0 is always demanded; before a call first claims any higher lane, the +/// claim raises the demand so the broker starts serving that lane. +/// +/// The pool also records the failure that ended the association, since claims +/// must check it and wait under the same lock. Other users of the association, +/// such as its notification receiver, check that record through +/// [`current_failure`](Self::current_failure) and +/// [`run_if_live`](Self::run_if_live). +pub struct LocalCallLanePool { + state: Mutex>, + lane_freed: Condvar, +} + +struct LocalCallLanePoolInner { + /// Each lane, or `None` while a call holds it. + lanes: Vec>, + /// Number of lanes demanded so far. + demanded_lane_count: usize, + /// Number of claims waiting for a free lane. Freeing a lane notifies only + /// if one waits, since a notification can cost a system call even with no + /// waiters. + waiting_claim_count: usize, + failure: Option, +} + +impl LocalCallLanePoolInner { + /// Fails with a copy of the association's failure, if one was recorded. + fn check_live(&self) -> Result<(), Error> { + match &self.failure { + Some(failure) => Err(copy_error(failure)), + None => Ok(()), + } + } +} + +/// A lane held by one call; releasing or dropping the claim frees the lane. +pub struct LaneClaim<'lanes, Lane> { + lanes: &'lanes LocalCallLanePool, + index: usize, + /// The lane, or `None` once released. + lane: Option, +} + +impl Deref for LaneClaim<'_, Lane> { + type Target = Lane; + + fn deref(&self) -> &Lane { + self.lane + .as_ref() + .expect("claimed lane is present until released") + } +} + +impl DerefMut for LaneClaim<'_, Lane> { + fn deref_mut(&mut self) -> &mut Lane { + self.lane + .as_mut() + .expect("claimed lane is present until released") + } +} + +impl Drop for LaneClaim<'_, Lane> { + fn drop(&mut self) { + if self.lane.is_some() { + let lanes = self.lanes; + self.free(&mut lanes.lock()); + } + } +} + +impl LaneClaim<'_, Lane> { + /// Runs `operation` on the lane and frees the lane, both while excluding + /// failure recording, so a call completes only while the association is + /// live. + /// + /// If the association has failed or `operation` fails, the claim keeps + /// the lane until dropped, so the caller can fail the association first. + pub fn release_if_live( + &mut self, + operation: impl FnOnce(&mut Lane) -> Result, + ) -> Result { + let lanes = self.lanes; + let mut state = lanes.lock(); + state.check_live()?; + let value = operation(self)?; + self.free(&mut state); + Ok(value) + } + + /// Returns the lane to its slot, under the admission lock. + fn free(&mut self, state: &mut LocalCallLanePoolInner) { + state.lanes[self.index] = self.lane.take(); + if state.waiting_claim_count > 0 { + self.lanes.lane_freed.notify_one(); + } + } +} + +impl LocalCallLanePool { + /// Creates live admission over `lanes`, of which only lane 0 is demanded. + /// + /// # Panics + /// + /// Panics if `lanes` is empty. + pub fn new(lanes: Vec) -> Self { + assert!(!lanes.is_empty(), "call lanes need at least one lane"); + Self { + state: Mutex::new(LocalCallLanePoolInner { + lanes: lanes.into_iter().map(Some).collect(), + demanded_lane_count: 1, + waiting_claim_count: 0, + failure: None, + }), + lane_freed: Condvar::new(), + } + } + + /// Claims the lowest free lane, blocking while every lane is held. + /// + /// When the claimed lane is the first beyond those demanded so far, + /// `demand` is called with the new lane count, under the admission lock so + /// demand only rises. If it fails, the lane stays free and undemanded. + pub fn claim( + &self, + demand: impl FnOnce(usize) -> Result<(), Error>, + ) -> Result, Error> { + let mut state = self.lock(); + let index = loop { + state.check_live()?; + if let Some(index) = state.lanes.iter().position(Option::is_some) { + break index; + } + state.waiting_claim_count += 1; + state = self.wait_for_free_lane(state); + state.waiting_claim_count -= 1; + }; + if index >= state.demanded_lane_count { + demand(index + 1)?; + state.demanded_lane_count = index + 1; + } + Ok(LaneClaim { + lanes: self, + index, + lane: state.lanes[index].take(), + }) + } + + /// Records the first failure, which ends the association, and wakes every + /// waiting claim. + /// + /// Returns whether this call recorded the first failure. + pub fn record_failure(&self, error: Error) -> bool { + let mut state = self.lock(); + if state.failure.is_some() { + return false; + } + state.failure = Some(error); + self.lane_freed.notify_all(); + true + } + + /// Returns a copy of the failure that ended the association, if one was + /// recorded. + pub fn current_failure(&self) -> Option { + self.lock().failure.as_ref().map(copy_error) + } + + /// Runs an operation while excluding failure recording. + pub fn run_if_live(&self, operation: impl FnOnce() -> Result) -> Result { + let state = self.lock(); + state.check_live()?; + operation() + } + + fn lock(&self) -> MutexGuard<'_, LocalCallLanePoolInner> { + self.state.lock().expect("broker call-lanes mutex poisoned") + } + + fn wait_for_free_lane<'a>( + &self, + state: MutexGuard<'a, LocalCallLanePoolInner>, + ) -> MutexGuard<'a, LocalCallLanePoolInner> { + self.lane_freed + .wait(state) + .expect("broker call-lanes mutex poisoned") + } +} + +/// Copies `error`, since [`Error`] is not `Clone`. +fn copy_error(error: &Error) -> Error { + match error.raw_os_error() { + Some(code) => Error::from_raw_os_error(code), + None => Error::new(error.kind(), error.to_string()), + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + use std::io::{Error, ErrorKind}; + use std::sync::mpsc; + use std::time::Duration; + + use super::{LocalCallLanePool, Vec}; + + fn no_demand(_: usize) -> Result<(), Error> { + unreachable!("lane 0 needs no demand") + } + + #[test] + fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { + let lanes = LocalCallLanePool::new(vec![0, 1, 2]); + let mut demands = Vec::new(); + let first = lanes.claim(no_demand).unwrap(); + let second = lanes + .claim(|count| { + demands.push(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*first, *second), (0, 1)); + drop(first); + let reused = lanes.claim(no_demand).unwrap(); + assert_eq!(*reused, 0); + drop((reused, second)); + let again = lanes.claim(no_demand).unwrap(); + let next = lanes + .claim(|_| panic!("lane 1 is already demanded")) + .unwrap(); + assert_eq!((*again, *next), (0, 1)); + assert_eq!(demands, [2]); + } + + #[test] + fn a_failed_demand_leaves_the_lane_free_and_undemanded() { + let lanes = LocalCallLanePool::new(vec![0, 1]); + let _first = lanes.claim(no_demand).unwrap(); + assert!(matches!( + lanes.claim(|_| Err(Error::from(ErrorKind::Other))), + Err(error) if error.kind() == ErrorKind::Other + )); + let mut demanded = None; + let second = lanes + .claim(|count| { + demanded = Some(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*second, demanded), (1, Some(2))); + } + + #[test] + fn claims_wait_while_every_lane_is_held() { + let lanes = LocalCallLanePool::new(vec![0]); + let held = lanes.claim(no_demand).unwrap(); + let (claimed, claim) = mpsc::channel(); + std::thread::scope(|scope| { + scope.spawn(|| { + let lane = lanes.claim(no_demand).map(|lane| *lane); + claimed.send(lane.is_ok()).unwrap(); + }); + assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); + drop(held); + assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); + }); + } + + #[test] + fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { + let lanes = LocalCallLanePool::new(vec![0, 1]); + let mut released = lanes.claim(no_demand).unwrap(); + assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); + // Dropping a released claim leaves its lane free. + drop(released); + let mut held = lanes.claim(no_demand).unwrap(); + assert_eq!(*held, 0); + assert!(matches!( + held.release_if_live(|_| Err::<(), _>(Error::from(ErrorKind::Other))), + Err(error) if error.kind() == ErrorKind::Other + )); + assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); + assert!(lanes.record_failure(Error::from(ErrorKind::BrokenPipe))); + assert!(matches!( + held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); + assert_eq!(*held, 0); + } + + #[test] + fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { + let lanes = LocalCallLanePool::new(vec![0]); + let _held = lanes.claim(no_demand).unwrap(); + std::thread::scope(|scope| { + let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); + std::thread::sleep(Duration::from_millis(10)); + assert!(lanes.record_failure(Error::from(ErrorKind::BrokenPipe))); + assert!(matches!( + waiter.join().unwrap(), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); + }); + assert!(!lanes.record_failure(Error::from(ErrorKind::Other))); + assert_eq!( + lanes.current_failure().unwrap().kind(), + ErrorKind::BrokenPipe + ); + assert!(matches!( + lanes.run_if_live(|| Ok(())), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); + } +} diff --git a/litebox_broker_transport/src/channel.rs b/litebox_broker_transport/src/channel.rs index b344898404..01fc79882d 100644 --- a/litebox_broker_transport/src/channel.rs +++ b/litebox_broker_transport/src/channel.rs @@ -129,40 +129,52 @@ pub trait HostNotificationChannel { /// /// An error is terminal when this channel is part of an active /// association. Before returning it, the transport must terminate that - /// association and unblock its request, response, notification, and - /// shutdown endpoints. This lets the association runtime preserve a clean - /// peer close while relying on the request endpoint to report transport - /// failures. + /// association and unblock its call-lane, notification, and shutdown + /// endpoints. This lets the association runtime preserve a clean peer + /// close while relying on the call lanes to report transport failures. fn send_notification(&mut self, notification: &BrokerNotification) -> Result<(), Self::Error>; } -/// Host-side endpoint that reads active broker requests from an association's -/// peer. +/// Host-side serving end of one call lane of an active association. /// -/// This is the request-reading half of an active association's control -/// transport. It is transport-neutral so a generic association runtime, -/// such as the one in `litebox_broker_userland`, can dispatch requests without -/// depending on a concrete transport crate. -pub trait HostRequestSource { +/// A lane carries one call at a time: the broker reads a request from it and +/// answers that request on the same lane before reading the next. Calls on +/// different lanes are independent, so a broker can serve each lane on its +/// own thread. The trait is transport-neutral so a generic association +/// runtime, such as the one in `litebox_broker_userland`, can serve calls +/// without depending on a concrete transport crate. +pub trait HostCallLane { /// Channel-specific error type. type Error; - /// Receives one active broker request. + /// Receives this lane's next broker request. fn recv_request(&mut self) -> Result, Self::Error>; + + /// Serializes and sends the response to this lane's last request. + fn send_response(&mut self, response: &BrokerResponse) -> Result<(), Self::Error>; } -/// Host-side endpoint that publishes active broker responses to an -/// association's peer. +/// Host-side source of the call lanes of an active association. /// -/// This is the response-writing half of an active association's control -/// transport. Implementations are typically cheaply [`Clone`]able so -/// concurrent request workers can each hold one. -pub trait HostResponseSink { +/// The local peer starts using lanes as its concurrent calls grow, and this +/// source hands out each lane once the local peer starts using it, so a broker +/// serves only lanes in use. The local peer always uses the first lane, so the +/// first [`next_lane`](Self::next_lane) returns it without blocking unless the +/// association has already ended. +pub trait HostCallLaneSource { /// Channel-specific error type. type Error; - /// Serializes and sends one complete active broker response. - fn send_response(&self, response: &BrokerResponse) -> Result<(), Self::Error>; + /// Lane type handed out by this source. + type Lane: HostCallLane; + + /// Blocks until the local peer starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` when the local peer closed the association cleanly. + /// An error is terminal: before returning it, the transport must terminate + /// the association and unblock every endpoint. + fn next_lane(&mut self) -> Result, Self::Error>; } /// RAII-independent shutdown handle for an active broker association. diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index ed001f447b..610d96fc23 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -9,6 +9,7 @@ //! rather than interpreting the same shared memory differently. use alloc::sync::Arc; +use alloc::vec::Vec; use core::mem::size_of; use core::ops::Range; use core::sync::atomic::{Ordering, fence}; @@ -18,9 +19,6 @@ use crate::shared_memory::{ControlRingMemory, SharedMemoryError}; /// Size of one shared control-ring slot. pub const CONTROL_RING_SLOT_SIZE: usize = 128; -/// Versioned token exchanged before endpoints activate this control-ring ABI. -pub const CONTROL_RING_READY: &[u8] = b"litebox-control-ring-ready-v1"; - /// Size of the fixed metadata at the start of a control-ring slot. pub const CONTROL_RING_SLOT_HEADER_SIZE: usize = 16; @@ -36,9 +34,6 @@ const _: () = assert!( ); const _: () = assert!(CONTROL_RING_SLOT_SIZE.is_multiple_of(size_of::())); -/// Number of slots in each request or response direction. -pub const CONTROL_RING_SLOT_COUNT: u64 = 64; - /// Number of slots in the broker-to-local notification direction. pub const CONTROL_RING_NOTIFICATION_SLOT_COUNT: u64 = 64; @@ -47,21 +42,10 @@ pub const CONTROL_RING_NOTIFICATION_SLOT_COUNT: u64 = 64; /// yet is far shorter than a scheduler time slice. pub const CONTROL_RING_PARALLEL_SPIN_LIMIT: u32 = 4000; -/// Exact shared-memory size required for all association control directions. -pub const CONTROL_RING_MEMORY_SIZE: usize = - CONTROL_RING_DATA_SIZE + CONTROL_RING_SYNC_DIRECTION_SIZE * 3; - -// The fixed count is representable by `usize` on every supported target. -#[allow(clippy::cast_possible_truncation)] -const CONTROL_RING_DIRECTION_SIZE: usize = - CONTROL_RING_SLOT_SIZE * CONTROL_RING_SLOT_COUNT as usize; // The fixed count is representable by `usize` on every supported target. #[allow(clippy::cast_possible_truncation)] const CONTROL_RING_NOTIFICATION_DIRECTION_SIZE: usize = CONTROL_RING_SLOT_SIZE * CONTROL_RING_NOTIFICATION_SLOT_COUNT as usize; -const CONTROL_RING_DATA_SIZE: usize = - CONTROL_RING_DIRECTION_SIZE * 2 + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE; -const CONTROL_RING_SYNC_DIRECTION_SIZE: usize = 24; const PRODUCER_EPOCH_OFFSET: usize = 0; const CONSUMER_EPOCH_OFFSET: usize = 4; const CONSUMER_HEAD_OFFSET: usize = 8; @@ -69,7 +53,234 @@ const CONSUMER_HEAD_OFFSET: usize = 8; const CONSUMER_WAITING_OFFSET: usize = 16; /// Nonzero while the producer may block on the consumer epoch. const PRODUCER_WAITING_OFFSET: usize = 20; -const _: () = assert!(CONTROL_RING_SYNC_DIRECTION_SIZE.is_multiple_of(size_of::())); +/// Size of the synchronization words of one direction. +const SYNC_WORDS_SIZE: usize = 24; +/// Cache-line size assumed when separating words that different threads poll. +const CACHE_LINE_SIZE: usize = 64; +/// Number of lanes the local peer asks the broker to serve. +const DEMANDED_LANE_COUNT_OFFSET: usize = 0; +/// Incremented whenever the local peer raises its demanded lane count. +const DEMAND_EPOCH_OFFSET: usize = 4; +const DEMAND_WORDS_SIZE: usize = 8; + +/// Shape of a control-ring mapping: its call lanes and their capacity. +/// +/// A mapping holds one or more *lanes*, each a request direction and a +/// response direction of the same slot count, and one broker-to-local +/// notification direction. Calls on different lanes never queue behind each +/// other, so a deployment that serves each lane separately can run as many +/// calls in parallel as it has lanes. +/// +/// The mapping starts with the slots of every direction, in the order lane 0 +/// requests, lane 0 responses, lane 1 requests, and so on, then notifications. +/// The synchronization words of each direction follow in the same order, one +/// block of `sync_stride` bytes per direction. With more than one lane, the +/// lane-demand words come last: the local peer raises the number of lanes it +/// uses there, so the broker can start serving a lane only once it is needed. +/// Lane 0 is always in use. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ControlRingLayout { + lane_count: usize, + call_slots: u64, + sync_stride: usize, + ready: &'static [u8], +} + +const _: () = assert!(ControlRingLayout::SINGLE_LANE.memory_size() == 24_648); +const _: () = assert!(ControlRingLayout::MULTI_LANE.memory_size() == 11_336); + +impl ControlRingLayout { + /// One lane of 64 slots per direction, served by one broker thread. + pub const SINGLE_LANE: Self = Self { + lane_count: 1, + call_slots: 64, + sync_stride: SYNC_WORDS_SIZE, + ready: b"litebox-control-ring-ready-v1", + }; + + /// Eight single-slot lanes, each carrying one call at a time. + /// + /// The synchronization words of each direction have a cache line to + /// themselves, so neither the two directions of a call nor lanes in use at + /// the same time contend for one. + pub const MULTI_LANE: Self = Self { + lane_count: 8, + call_slots: 1, + sync_stride: CACHE_LINE_SIZE, + ready: b"litebox-control-ring-lanes-ready-v1", + }; + + /// Returns the number of lanes. + pub const fn lane_count(self) -> usize { + self.lane_count + } + + /// Returns the versioned token endpoints exchange before activating a + /// mapping with this layout. + pub const fn ready_token(self) -> &'static [u8] { + self.ready + } + + /// Returns the exact size of a mapping with this layout. + pub const fn memory_size(self) -> usize { + if self.lane_count > 1 { + self.demand_offset() + DEMAND_WORDS_SIZE + } else { + self.demand_offset() + } + } + + /// Returns the local-to-broker request direction of `lane`. + /// + /// # Panics + /// + /// Panics if `lane` is not less than [`Self::lane_count`]. + pub const fn requests(self, lane: usize) -> ControlRingDirection { + assert!(lane < self.lane_count, "control-ring lane out of range"); + self.direction(2 * lane) + } + + /// Returns the broker-to-local response direction of `lane`. + /// + /// # Panics + /// + /// Panics if `lane` is not less than [`Self::lane_count`]. + pub const fn responses(self, lane: usize) -> ControlRingDirection { + assert!(lane < self.lane_count, "control-ring lane out of range"); + self.direction(2 * lane + 1) + } + + /// Returns the broker-to-local notification direction. + pub const fn notifications(self) -> ControlRingDirection { + self.direction(2 * self.lane_count) + } + + /// Returns whether this layout permits a byte-copy range. + /// + /// Concrete shared-memory implementations use this together with + /// [`Self::permits_u32`] and [`Self::permits_u64`] to keep their own byte + /// and typed-word accesses disjoint. A peer can bypass these checks through + /// its backing-resource alias, so implementations must remain memory-safe + /// under arbitrary peer writes. + pub const fn permits_byte_range(self, offset: usize, length: usize) -> bool { + let Some(end) = offset.checked_add(length) else { + return false; + }; + if end > self.memory_size() { + return false; + } + if length == 0 { + return true; + } + let slot_offset = offset % CONTROL_RING_SLOT_SIZE; + end <= self.data_size() + && slot_offset >= size_of::() + && slot_offset + length <= CONTROL_RING_SLOT_SIZE + } + + /// Returns whether this layout permits a `u32` access at `offset`. + pub const fn permits_u32(self, offset: usize) -> bool { + let Some(relative) = offset.checked_sub(self.data_size()) else { + return false; + }; + if relative < self.sync_size() { + return matches!( + relative % self.sync_stride, + PRODUCER_EPOCH_OFFSET + | CONSUMER_EPOCH_OFFSET + | CONSUMER_WAITING_OFFSET + | PRODUCER_WAITING_OFFSET + ); + } + self.lane_count > 1 + && matches!( + relative - self.sync_size(), + DEMANDED_LANE_COUNT_OFFSET | DEMAND_EPOCH_OFFSET + ) + } + + /// Returns whether this layout permits a `u64` access at `offset`. + pub const fn permits_u64(self, offset: usize) -> bool { + if offset < self.data_size() { + return offset.is_multiple_of(CONTROL_RING_SLOT_SIZE); + } + let relative = offset - self.data_size(); + relative < self.sync_size() && relative % self.sync_stride == CONSUMER_HEAD_OFFSET + } + + /// Returns the number of `u32` words an endpoint may block on: the + /// producer and consumer epochs of every direction and, with more than one + /// lane, the lane-demand epoch. + pub const fn waitable_word_count(self) -> usize { + 2 * self.direction_count() + if self.lane_count > 1 { 1 } else { 0 } + } + + /// Returns the position of the waitable `u32` at `offset` among the + /// [`Self::waitable_word_count`] words, or `None` if no endpoint blocks + /// on it. + pub const fn waitable_word_index(self, offset: usize) -> Option { + let Some(relative) = offset.checked_sub(self.data_size()) else { + return None; + }; + if relative < self.sync_size() { + let direction = relative / self.sync_stride; + return match relative % self.sync_stride { + PRODUCER_EPOCH_OFFSET => Some(2 * direction), + CONSUMER_EPOCH_OFFSET => Some(2 * direction + 1), + _ => None, + }; + } + if self.lane_count > 1 && relative - self.sync_size() == DEMAND_EPOCH_OFFSET { + Some(2 * self.direction_count()) + } else { + None + } + } + + const fn direction_count(self) -> usize { + 2 * self.lane_count + 1 + } + + // Layout slot counts are small constants representable by `usize`. + #[allow(clippy::cast_possible_truncation)] + const fn call_direction_size(self) -> usize { + CONTROL_RING_SLOT_SIZE * self.call_slots as usize + } + + const fn data_size(self) -> usize { + 2 * self.lane_count * self.call_direction_size() + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE + } + + const fn sync_size(self) -> usize { + self.direction_count() * self.sync_stride + } + + const fn demand_offset(self) -> usize { + self.data_size() + self.sync_size() + } + + /// Returns the direction at `index` in mapping order, the notification + /// direction coming after every lane's. + const fn direction(self, index: usize) -> ControlRingDirection { + ControlRingDirection { + data_offset: index * self.call_direction_size(), + slot_count: if index == 2 * self.lane_count { + CONTROL_RING_NOTIFICATION_SLOT_COUNT + } else { + self.call_slots + }, + sync_offset: self.data_size() + index * self.sync_stride, + } + } +} + +const _: () = assert!(SYNC_WORDS_SIZE.is_multiple_of(size_of::())); +const _: () = assert!(CACHE_LINE_SIZE >= SYNC_WORDS_SIZE); +const _: () = assert!( + ControlRingLayout::MULTI_LANE + .data_size() + .is_multiple_of(CACHE_LINE_SIZE) +); /// Access model enforced by a concrete shared-memory mapping. /// @@ -79,8 +290,9 @@ const _: () = assert!(CONTROL_RING_SYNC_DIRECTION_SIZE.is_multiple_of(size_of::< pub enum MemoryAccessPolicy { /// General shared memory supporting only byte copies. Bytes, - /// Control-ring memory supporting its ABI-defined byte and word accesses. - ControlRing, + /// Control-ring memory supporting the byte and word accesses its layout + /// defines. + ControlRing(ControlRingLayout), } impl MemoryAccessPolicy { @@ -88,163 +300,72 @@ impl MemoryAccessPolicy { pub const fn permits_byte_range(self, offset: usize, length: usize) -> bool { match self { Self::Bytes => true, - Self::ControlRing => memory_permits_byte_range(offset, length), + Self::ControlRing(layout) => layout.permits_byte_range(offset, length), } } /// Returns whether this policy permits a `u32` access at `offset`. pub const fn permits_u32(self, offset: usize) -> bool { - matches!(self, Self::ControlRing) && memory_permits_u32(offset) + match self { + Self::Bytes => false, + Self::ControlRing(layout) => layout.permits_u32(offset), + } } /// Returns whether this policy permits a `u64` access at `offset`. pub const fn permits_u64(self, offset: usize) -> bool { - matches!(self, Self::ControlRing) && memory_permits_u64(offset) - } -} - -/// Returns whether the control-ring ABI permits a byte-copy range. -/// -/// Concrete shared-memory implementations use this together with -/// [`memory_permits_u32`] and [`memory_permits_u64`] to keep their own byte and -/// typed-word accesses disjoint. A peer can bypass these checks through its -/// backing-resource alias, so implementations must remain memory-safe under -/// arbitrary peer writes. -pub const fn memory_permits_byte_range(offset: usize, length: usize) -> bool { - let Some(end) = offset.checked_add(length) else { - return false; - }; - if end > CONTROL_RING_MEMORY_SIZE { - return false; - } - if length == 0 { - return true; - } - direction_permits_byte_range(offset, length, 0, CONTROL_RING_SLOT_COUNT) - || direction_permits_byte_range( - offset, - length, - CONTROL_RING_DIRECTION_SIZE, - CONTROL_RING_SLOT_COUNT, - ) - || direction_permits_byte_range( - offset, - length, - CONTROL_RING_DIRECTION_SIZE * 2, - CONTROL_RING_NOTIFICATION_SLOT_COUNT, - ) -} - -/// Returns whether the control-ring ABI permits a `u32` access at `offset`. -pub const fn memory_permits_u32(offset: usize) -> bool { - let Some(relative) = offset.checked_sub(CONTROL_RING_DATA_SIZE) else { - return false; - }; - relative < CONTROL_RING_SYNC_DIRECTION_SIZE * 3 - && matches!( - relative % CONTROL_RING_SYNC_DIRECTION_SIZE, - PRODUCER_EPOCH_OFFSET - | CONSUMER_EPOCH_OFFSET - | CONSUMER_WAITING_OFFSET - | PRODUCER_WAITING_OFFSET - ) -} - -/// Returns whether the control-ring ABI permits a `u64` access at `offset`. -pub const fn memory_permits_u64(offset: usize) -> bool { - if offset < CONTROL_RING_DATA_SIZE { - return offset.is_multiple_of(CONTROL_RING_SLOT_SIZE); - } - let relative = offset - CONTROL_RING_DATA_SIZE; - relative < CONTROL_RING_SYNC_DIRECTION_SIZE * 3 - && relative % CONTROL_RING_SYNC_DIRECTION_SIZE == CONSUMER_HEAD_OFFSET -} - -const fn direction_permits_byte_range( - offset: usize, - length: usize, - direction_start: usize, - slot_count: u64, -) -> bool { - let Some(relative) = offset.checked_sub(direction_start) else { - return false; - }; - #[allow(clippy::cast_possible_truncation)] - let direction_size = CONTROL_RING_SLOT_SIZE * slot_count as usize; - if relative >= direction_size { - return false; + match self { + Self::Bytes => false, + Self::ControlRing(layout) => layout.permits_u64(offset), + } } - let slot_offset = relative % CONTROL_RING_SLOT_SIZE; - let Some(slot_end) = slot_offset.checked_add(length) else { - return false; - }; - slot_offset >= size_of::() && slot_end <= CONTROL_RING_SLOT_SIZE } -/// One direction in the shared control-ring mapping. +/// One direction in a control-ring mapping, as placed by its +/// [`ControlRingLayout`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ControlRingDirection { - /// Local-to-broker request ring. - Requests, - /// Broker-to-local response ring. - Responses, - /// Broker-to-local asynchronous notification ring. - Notifications, +pub struct ControlRingDirection { + data_offset: usize, + slot_count: u64, + sync_offset: usize, } impl ControlRingDirection { fn slot_range(self, slot: u64) -> Range { - debug_assert!(slot < self.slot_count()); + debug_assert!(slot < self.slot_count); let slot = usize::try_from(slot).expect("control-ring slot index is bounded"); - let data_offset = match self { - Self::Requests => 0, - Self::Responses => CONTROL_RING_DIRECTION_SIZE, - Self::Notifications => CONTROL_RING_DIRECTION_SIZE * 2, - }; - let start = data_offset + slot * CONTROL_RING_SLOT_SIZE; + let start = self.data_offset + slot * CONTROL_RING_SLOT_SIZE; start..start + CONTROL_RING_SLOT_SIZE } /// Returns the number of slots in this direction. pub const fn slot_count(self) -> u64 { - match self { - Self::Requests | Self::Responses => CONTROL_RING_SLOT_COUNT, - Self::Notifications => CONTROL_RING_NOTIFICATION_SLOT_COUNT, - } + self.slot_count } /// Returns the `u32` epoch incremented when the producer publishes work for /// this direction. pub const fn producer_epoch_offset(self) -> usize { - self.sync_offset() + PRODUCER_EPOCH_OFFSET + self.sync_offset + PRODUCER_EPOCH_OFFSET } /// Returns the `u32` epoch incremented when the consumer publishes progress /// for this direction. pub const fn consumer_epoch_offset(self) -> usize { - self.sync_offset() + CONSUMER_EPOCH_OFFSET + self.sync_offset + CONSUMER_EPOCH_OFFSET } /// Returns the `u64` consumer-head offset for this direction. pub const fn consumer_head_offset(self) -> usize { - self.sync_offset() + CONSUMER_HEAD_OFFSET + self.sync_offset + CONSUMER_HEAD_OFFSET } const fn consumer_waiting_offset(self) -> usize { - self.sync_offset() + CONSUMER_WAITING_OFFSET + self.sync_offset + CONSUMER_WAITING_OFFSET } const fn producer_waiting_offset(self) -> usize { - self.sync_offset() + PRODUCER_WAITING_OFFSET - } - - const fn sync_offset(self) -> usize { - CONTROL_RING_DATA_SIZE - + match self { - Self::Requests => 0, - Self::Responses => CONTROL_RING_SYNC_DIRECTION_SIZE, - Self::Notifications => CONTROL_RING_SYNC_DIRECTION_SIZE * 2, - } + self.sync_offset + PRODUCER_WAITING_OFFSET } } @@ -342,42 +463,95 @@ pub enum ControlRingReadError { Decode(DecodeError), } -/// Exact-size shared memory containing request, response, and notification rings. +/// Exact-size shared memory holding the directions of a [`ControlRingLayout`]. pub struct ControlRing { memory: Memory, + layout: ControlRingLayout, } -/// Role-bound association ring endpoints owned by the local peer. -pub struct LocalControlRingEndpoints { +/// The local peer's ends of one call lane. +pub struct LocalControlRingLane { /// Local-to-broker request producer. pub request_producer: ControlRingProducer, /// Broker-to-local response consumer. pub response_consumer: ControlRingConsumer, - /// Broker-to-local notification consumer. - pub notification_consumer: ControlRingConsumer, } -/// Role-bound association ring endpoints owned by the broker peer. -pub struct BrokerControlRingEndpoints { +/// The broker peer's ends of one call lane. +pub struct HostControlRingLane { /// Local-to-broker request consumer. pub request_consumer: ControlRingConsumer, /// Broker-to-local response producer. pub response_producer: ControlRingProducer, +} + +/// Role-bound association ring endpoints owned by the local peer. +pub struct LocalControlRingEndpoints { + /// Call lanes, in lane order. + pub lanes: Vec>, + /// Lane-demand publisher, present when the layout has more than one lane. + pub lane_demand: Option>, + /// Broker-to-local notification consumer. + pub notification_consumer: ControlRingConsumer, +} + +/// Role-bound association ring endpoints owned by the broker peer. +pub struct HostControlRingEndpoints { + /// Call lanes, in lane order. + pub lanes: Vec>, + /// Lane-demand observer, present when the layout has more than one lane. + pub lane_demand: Option>, /// Broker-to-local notification producer. pub notification_producer: ControlRingProducer, } +impl LocalControlRingEndpoints { + /// Returns handles that interrupt every wait these endpoints offer. + pub fn wake_handles(&self) -> Vec> { + self.lanes + .iter() + .flat_map(|lane| { + [ + lane.request_producer.wake_handle(), + lane.response_consumer.wake_handle(), + ] + }) + .chain([self.notification_consumer.wake_handle()]) + .collect() + } +} + +impl HostControlRingEndpoints { + /// Returns handles that interrupt every wait these endpoints offer. + pub fn wake_handles(&self) -> Vec> { + self.lanes + .iter() + .flat_map(|lane| { + [ + lane.request_consumer.wake_handle(), + lane.response_producer.wake_handle(), + ] + }) + .chain([self.notification_producer.wake_handle()]) + .chain(self.lane_demand.as_ref().map(HostLaneDemand::wake_handle)) + .collect() + } +} + impl ControlRing { - /// Attaches to an exact-size shared control-ring mapping. - pub fn new(memory: Memory) -> Result { + /// Attaches to an exact-size shared control-ring mapping with `layout`. + pub fn new(memory: Memory, layout: ControlRingLayout) -> Result { let actual = memory.len(); - if actual != CONTROL_RING_MEMORY_SIZE { - return Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual, - }); + let expected = layout.memory_size(); + if actual != expected { + return Err(ControlRingError::MemoryLengthMismatch { expected, actual }); } - Ok(Self { memory }) + Ok(Self { memory, layout }) + } + + /// Returns the layout of this mapping. + pub const fn layout(&self) -> ControlRingLayout { + self.layout } /// Returns the copy-only shared-memory resource. @@ -387,39 +561,50 @@ impl ControlRing { /// Consumes the mapping into endpoints owned by the local peer. pub fn into_local(self) -> LocalControlRingEndpoints { + let layout = self.layout; let ring = Arc::new(self); LocalControlRingEndpoints { - request_producer: ControlRingProducer::new( - Arc::clone(&ring), - ControlRingDirection::Requests, - ), - response_consumer: ControlRingConsumer::new( - Arc::clone(&ring), - ControlRingDirection::Responses, - ), - notification_consumer: ControlRingConsumer::new( - ring, - ControlRingDirection::Notifications, - ), + lanes: (0..layout.lane_count()) + .map(|lane| LocalControlRingLane { + request_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }) + .collect(), + lane_demand: (layout.lane_count() > 1).then(|| LocalLaneDemand { + ring: Arc::clone(&ring), + }), + notification_consumer: ControlRingConsumer::new(ring, layout.notifications()), } } /// Consumes the mapping into endpoints owned by the broker peer. - pub fn into_broker(self) -> BrokerControlRingEndpoints { + pub fn into_host(self) -> HostControlRingEndpoints { + let layout = self.layout; let ring = Arc::new(self); - BrokerControlRingEndpoints { - request_consumer: ControlRingConsumer::new( - Arc::clone(&ring), - ControlRingDirection::Requests, - ), - response_producer: ControlRingProducer::new( - Arc::clone(&ring), - ControlRingDirection::Responses, - ), - notification_producer: ControlRingProducer::new( - ring, - ControlRingDirection::Notifications, - ), + HostControlRingEndpoints { + lanes: (0..layout.lane_count()) + .map(|lane| HostControlRingLane { + request_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }) + .collect(), + lane_demand: (layout.lane_count() > 1).then(|| HostLaneDemand { + ring: Arc::clone(&ring), + started_lane_count: 0, + }), + notification_producer: ControlRingProducer::new(ring, layout.notifications()), } } @@ -506,6 +691,118 @@ impl ControlRing { } } +/// Local-peer end of the lane-demand words of a multi-lane control ring. +/// +/// The local peer raises the number of lanes it uses before it first writes a +/// request to a lane, and the broker starts serving a lane once it is +/// demanded. Lane 0 is always demanded. +pub struct LocalLaneDemand { + ring: Arc>, +} + +impl LocalLaneDemand { + /// Publishes that the local peer uses the first `lanes` lanes. + /// + /// The count must not decrease within an association. + /// + /// # Panics + /// + /// Panics if `lanes` exceeds the layout's lane count. + pub fn request(&self, lanes: usize) -> Result<(), ControlRingError> { + let layout = self.ring.layout; + assert!(lanes <= layout.lane_count(), "lane demand out of range"); + let lanes = u32::try_from(lanes).expect("lane count is bounded by the layout"); + let demand = layout.demand_offset(); + self.ring + .memory + .store_u32_release(demand + DEMANDED_LANE_COUNT_OFFSET, lanes.to_le())?; + self.ring + .memory + .increment_u32_release(demand + DEMAND_EPOCH_OFFSET)?; + Ok(()) + } +} + +impl LocalLaneDemand { + /// Wakes the broker after [`Self::request`]. + /// + /// The wake is unconditional: demand rises at most once per lane, so + /// there is nothing to gain from an announcement word. + pub fn wake_host(&self) -> Result<(), Memory::Error> { + self.ring + .memory + .wake_one(self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET) + } +} + +/// Result of polling a [`HostLaneDemand`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ControlRingDemandStatus { + /// The local peer demands this lane, which the broker has yet to serve. + Lane(usize), + /// The broker serves every demanded lane. + Satisfied { + /// Demand epoch sampled before the demand was read. + wait_epoch: u32, + }, +} + +/// Broker-peer end of the lane-demand words of a multi-lane control ring. +/// +/// It hands out each lane, in lane order, once the local peer demands it. +/// The local peer can publish any count; the broker clamps it to the layout's +/// lanes and never takes a lane back. +pub struct HostLaneDemand { + ring: Arc>, + started_lane_count: usize, +} + +impl HostLaneDemand { + /// Returns the next demanded lane the broker has yet to serve, if any. + pub fn try_next_lane(&mut self) -> Result { + let layout = self.ring.layout; + let demand = layout.demand_offset(); + let wait_epoch = self + .ring + .memory + .load_u32_acquire(demand + DEMAND_EPOCH_OFFSET)?; + let demanded_lane_count = u32::from_le( + self.ring + .memory + .load_u32_acquire(demand + DEMANDED_LANE_COUNT_OFFSET)?, + ); + let demanded_lane_count = usize::try_from(demanded_lane_count) + .unwrap_or(usize::MAX) + .clamp(1, layout.lane_count()); + if self.started_lane_count < demanded_lane_count { + let lane = self.started_lane_count; + self.started_lane_count += 1; + return Ok(ControlRingDemandStatus::Lane(lane)); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) + } + + /// Returns a handle that can interrupt [`Self::wait_for_demand`]. + pub fn wake_handle(&self) -> ControlRingWakeHandle { + ControlRingWakeHandle::new( + &self.ring, + self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET, + ) + } +} + +impl HostLaneDemand { + /// Waits for new demand after [`ControlRingDemandStatus::Satisfied`]. + /// + /// The caller must poll again after this possibly spurious wakeup. + pub fn wait_for_demand(&mut self, wait_epoch: u32) -> Result<(), Memory::Error> { + self.ring.memory.wait_while_equal( + self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET, + wait_epoch, + ) + } +} + /// Trusted endpoint-local state for one control-ring producer. pub struct ControlRingProducer { ring: Arc>, @@ -524,7 +821,7 @@ pub struct ControlRingProducer { /// without gaining access to ring memory. pub struct ControlRingWakeHandle { ring: Arc>, - wait_epoch: ControlRingWaitEpoch, + epoch_offset: usize, } /// Shared-memory wait and wake support for control-ring epoch words. @@ -570,33 +867,25 @@ pub trait WaitableSharedMemory: ControlRingMemory { } } -#[derive(Clone, Copy)] -enum ControlRingWaitEpoch { - Producer(ControlRingDirection), - Consumer(ControlRingDirection), -} - -impl ControlRingWaitEpoch { - const fn offset(self) -> usize { - match self { - Self::Producer(direction) => direction.producer_epoch_offset(), - Self::Consumer(direction) => direction.consumer_epoch_offset(), - } - } -} - impl Clone for ControlRingWakeHandle { fn clone(&self) -> Self { Self { ring: Arc::clone(&self.ring), - wait_epoch: self.wait_epoch, + epoch_offset: self.epoch_offset, } } } impl ControlRingWakeHandle { + fn new(ring: &Arc>, epoch_offset: usize) -> Self { + Self { + ring: Arc::clone(ring), + epoch_offset, + } + } + pub(crate) const fn wait_epoch_offset(&self) -> usize { - self.wait_epoch.offset() + self.epoch_offset } pub(crate) fn memory(&self) -> &Memory { @@ -694,10 +983,7 @@ impl ControlRingProducer { /// Returns a handle that can interrupt a wait on this producer's ring /// direction. pub fn wake_handle(&self) -> ControlRingWakeHandle { - ControlRingWakeHandle { - ring: Arc::clone(&self.ring), - wait_epoch: ControlRingWaitEpoch::Consumer(self.direction), - } + ControlRingWakeHandle::new(&self.ring, self.direction.consumer_epoch_offset()) } #[cfg(test)] @@ -810,10 +1096,7 @@ impl ControlRingConsumer { /// Returns a handle that can interrupt a wait on this consumer's ring /// direction. pub fn wake_handle(&self) -> ControlRingWakeHandle { - ControlRingWakeHandle { - ring: Arc::clone(&self.ring), - wait_epoch: ControlRingWaitEpoch::Producer(self.direction), - } + ControlRingWakeHandle::new(&self.ring, self.direction.producer_epoch_offset()) } #[cfg(test)] @@ -951,170 +1234,358 @@ mod tests { const SEQUENCE_RANGE: core::ops::Range = 0..8; const LENGTH_RANGE: core::ops::Range = 8..12; const RESERVED_RANGE: core::ops::Range = 12..16; + const LAYOUT: ControlRingLayout = ControlRingLayout::SINGLE_LANE; + const LAYOUTS: [ControlRingLayout; 2] = [ + ControlRingLayout::SINGLE_LANE, + ControlRingLayout::MULTI_LANE, + ]; + const MEMORY_SIZE: usize = LAYOUT.memory_size(); + const REQUESTS: ControlRingDirection = LAYOUT.requests(0); + const RESPONSES: ControlRingDirection = LAYOUT.responses(0); + const NOTIFICATIONS: ControlRingDirection = LAYOUT.notifications(); + const SLOT_COUNT: u64 = REQUESTS.slot_count(); #[test] fn mapping_requires_the_exact_control_ring_size() { - assert_eq!(CONTROL_RING_MEMORY_SIZE, 24_648); - assert!(matches!( - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE - 1)), - Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual - }) if actual == CONTROL_RING_MEMORY_SIZE - 1 - )); - assert!(matches!( - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE + 1)), - Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual - }) if actual == CONTROL_RING_MEMORY_SIZE + 1 - )); - assert!(ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)).is_ok()); + assert_eq!(ControlRingLayout::SINGLE_LANE.memory_size(), 24_648); + for layout in LAYOUTS { + let size = layout.memory_size(); + assert!(matches!( + ControlRing::new(TestMemory::new(size - 1), layout), + Err(ControlRingError::MemoryLengthMismatch { expected, actual }) + if expected == size && actual == size - 1 + )); + assert!(matches!( + ControlRing::new(TestMemory::new(size + 1), layout), + Err(ControlRingError::MemoryLengthMismatch { expected, actual }) + if expected == size && actual == size + 1 + )); + assert!(ControlRing::new(TestMemory::new(size), layout).is_ok()); + } } #[test] fn memory_layout_separates_byte_and_word_regions() { - assert!(memory_permits_u64(0)); - assert!(!memory_permits_u32(0)); - assert!(!memory_permits_byte_range(0, 1)); - assert!(memory_permits_byte_range( - size_of::(), - CONTROL_RING_SLOT_SIZE - size_of::() - )); - assert!(!memory_permits_byte_range( - size_of::(), - CONTROL_RING_SLOT_SIZE - size_of::() + 1 - )); + for layout in LAYOUTS { + assert!(layout.permits_u64(0)); + assert!(!layout.permits_u32(0)); + assert!(!layout.permits_byte_range(0, 1)); + assert!( + layout.permits_byte_range( + size_of::(), + CONTROL_RING_SLOT_SIZE - size_of::() + ) + ); + assert!(!layout.permits_byte_range( + size_of::(), + CONTROL_RING_SLOT_SIZE - size_of::() + 1 + )); - let sync_start = CONTROL_RING_DATA_SIZE; - assert!(memory_permits_u32(sync_start + PRODUCER_EPOCH_OFFSET)); - assert!(memory_permits_u32(sync_start + CONSUMER_EPOCH_OFFSET)); - assert!(memory_permits_u64(sync_start + CONSUMER_HEAD_OFFSET)); - assert!(memory_permits_u32(sync_start + CONSUMER_WAITING_OFFSET)); - assert!(memory_permits_u32(sync_start + PRODUCER_WAITING_OFFSET)); - assert!(!memory_permits_u64(sync_start + CONSUMER_WAITING_OFFSET)); - assert!(!memory_permits_u32(sync_start + CONSUMER_HEAD_OFFSET)); - assert!(!memory_permits_byte_range(sync_start, 1)); - assert!(!memory_permits_u32(CONTROL_RING_MEMORY_SIZE)); - assert!(!memory_permits_u64(CONTROL_RING_MEMORY_SIZE)); - assert!(!memory_permits_byte_range(CONTROL_RING_MEMORY_SIZE, 1)); + let sync_start = layout.data_size(); + assert!(!layout.permits_byte_range(sync_start - 1, 2)); + assert!(layout.permits_u32(sync_start + PRODUCER_EPOCH_OFFSET)); + assert!(layout.permits_u32(sync_start + CONSUMER_EPOCH_OFFSET)); + assert!(layout.permits_u64(sync_start + CONSUMER_HEAD_OFFSET)); + assert!(layout.permits_u32(sync_start + CONSUMER_WAITING_OFFSET)); + assert!(layout.permits_u32(sync_start + PRODUCER_WAITING_OFFSET)); + assert!(!layout.permits_u64(sync_start + CONSUMER_WAITING_OFFSET)); + assert!(!layout.permits_u32(sync_start + CONSUMER_HEAD_OFFSET)); + assert!(!layout.permits_byte_range(sync_start, 1)); + let size = layout.memory_size(); + assert!(!layout.permits_u32(size)); + assert!(!layout.permits_u64(size)); + assert!(!layout.permits_byte_range(size, 1)); + assert!(layout.permits_byte_range(size, 0)); + } + + let multi_lane = ControlRingLayout::MULTI_LANE; + let padding = multi_lane.data_size() + SYNC_WORDS_SIZE; + assert!(!multi_lane.permits_u32(padding)); + assert!(!multi_lane.permits_u64(padding)); + let demand = multi_lane.demand_offset(); + assert!(multi_lane.permits_u32(demand + DEMANDED_LANE_COUNT_OFFSET)); + assert!(multi_lane.permits_u32(demand + DEMAND_EPOCH_OFFSET)); + assert!(!multi_lane.permits_u64(demand)); + assert!(!multi_lane.permits_byte_range(demand, 1)); } #[test] fn directions_occupy_disjoint_data_and_sync_ranges() { - let directions = [ - ControlRingDirection::Requests, - ControlRingDirection::Responses, - ControlRingDirection::Notifications, - ]; - let mut next_data_offset = 0; - for direction in directions { - let first = direction.slot_range(0); - let last = direction.slot_range(direction.slot_count() - 1); - assert_eq!(first.start, next_data_offset); + for layout in LAYOUTS { + let directions: Vec<_> = (0..layout.lane_count()) + .flat_map(|lane| [layout.requests(lane), layout.responses(lane)]) + .chain([layout.notifications()]) + .collect(); + let mut next_data_offset = 0; + for direction in &directions { + let first = direction.slot_range(0); + let last = direction.slot_range(direction.slot_count() - 1); + assert_eq!(first.start, next_data_offset); + assert_eq!( + last.end - first.start, + usize::try_from(direction.slot_count()).unwrap() * CONTROL_RING_SLOT_SIZE + ); + next_data_offset = last.end; + } + assert_eq!(next_data_offset, layout.data_size()); + + for (index, direction) in directions.iter().enumerate() { + assert_eq!( + direction.producer_epoch_offset(), + layout.data_size() + index * layout.sync_stride + ); + assert_eq!( + direction.consumer_epoch_offset(), + direction.producer_epoch_offset() + size_of::() + ); + assert_eq!( + direction.consumer_head_offset(), + direction.producer_epoch_offset() + size_of::() + ); + assert!( + direction + .consumer_head_offset() + .is_multiple_of(align_of::()) + ); + assert_eq!( + direction.consumer_waiting_offset(), + direction.consumer_head_offset() + size_of::() + ); + assert_eq!( + direction.producer_waiting_offset(), + direction.consumer_waiting_offset() + size_of::() + ); + } assert_eq!( - last.end - first.start, - usize::try_from(direction.slot_count()).unwrap() * CONTROL_RING_SLOT_SIZE + layout.notifications().sync_offset + layout.sync_stride, + layout.demand_offset() ); - next_data_offset = last.end; } - assert_eq!(next_data_offset, CONTROL_RING_DATA_SIZE); + assert_eq!( + ControlRingLayout::SINGLE_LANE.demand_offset(), + ControlRingLayout::SINGLE_LANE.memory_size() + ); + assert_eq!( + ControlRingLayout::MULTI_LANE.demand_offset() + DEMAND_WORDS_SIZE, + ControlRingLayout::MULTI_LANE.memory_size() + ); + } - for (index, direction) in directions.into_iter().enumerate() { - assert_eq!( - direction.producer_epoch_offset(), - CONTROL_RING_DATA_SIZE + index * CONTROL_RING_SYNC_DIRECTION_SIZE - ); - assert_eq!( - direction.consumer_epoch_offset(), - direction.producer_epoch_offset() + size_of::() + #[test] + fn waitable_words_are_indexed_densely() { + for layout in LAYOUTS { + let mut indices: Vec<_> = (0..layout.memory_size()) + .filter_map(|offset| layout.waitable_word_index(offset)) + .collect(); + assert_eq!(indices.len(), layout.waitable_word_count()); + indices.sort_unstable(); + indices.dedup(); + assert_eq!(indices.len(), layout.waitable_word_count()); + assert!( + indices + .iter() + .all(|&index| index < layout.waitable_word_count()) ); + for offset in 0..layout.memory_size() { + if layout.waitable_word_index(offset).is_some() { + assert!(layout.permits_u32(offset)); + } + } + let mut wait_offsets: Vec<_> = (0..layout.lane_count()) + .flat_map(|lane| [layout.requests(lane), layout.responses(lane)]) + .chain([layout.notifications()]) + .flat_map(|direction| { + [ + direction.producer_epoch_offset(), + direction.consumer_epoch_offset(), + ] + }) + .collect(); + if layout.lane_count() > 1 { + wait_offsets.push(layout.demand_offset() + DEMAND_EPOCH_OFFSET); + } + assert_eq!(wait_offsets.len(), layout.waitable_word_count()); + for offset in wait_offsets { + assert!(layout.waitable_word_index(offset).is_some()); + } + } + assert_eq!(ControlRingLayout::SINGLE_LANE.waitable_word_count(), 6); + } + + #[test] + fn endpoint_roles_bind_opposite_directions_to_one_mapping() { + for layout in LAYOUTS { + let local = test_ring_with(layout).into_local(); + assert_eq!(local.lanes.len(), layout.lane_count()); + assert_eq!(local.lane_demand.is_some(), layout.lane_count() > 1); + for (index, lane) in local.lanes.iter().enumerate() { + assert_eq!(lane.request_producer.direction(), layout.requests(index)); + assert_eq!(lane.response_consumer.direction(), layout.responses(index)); + assert!(Arc::ptr_eq( + &lane.request_producer.ring, + &local.notification_consumer.ring + )); + assert!(Arc::ptr_eq( + &lane.response_consumer.ring, + &local.notification_consumer.ring + )); + } assert_eq!( - direction.consumer_head_offset(), - direction.producer_epoch_offset() + size_of::() - ); - assert!( - direction - .consumer_head_offset() - .is_multiple_of(align_of::()) + local.notification_consumer.direction(), + layout.notifications() ); + assert_eq!(local.wake_handles().len(), 2 * layout.lane_count() + 1); + + let broker = test_ring_with(layout).into_host(); + assert_eq!(broker.lanes.len(), layout.lane_count()); + assert_eq!(broker.lane_demand.is_some(), layout.lane_count() > 1); + for (index, lane) in broker.lanes.iter().enumerate() { + assert_eq!(lane.request_consumer.direction(), layout.requests(index)); + assert_eq!(lane.response_producer.direction(), layout.responses(index)); + assert!(Arc::ptr_eq( + &lane.request_consumer.ring, + &broker.notification_producer.ring + )); + assert!(Arc::ptr_eq( + &lane.response_producer.ring, + &broker.notification_producer.ring + )); + } assert_eq!( - direction.consumer_waiting_offset(), - direction.consumer_head_offset() + size_of::() + broker.notification_producer.direction(), + layout.notifications() ); assert_eq!( - direction.producer_waiting_offset(), - direction.consumer_waiting_offset() + size_of::() + broker.wake_handles().len(), + 2 * layout.lane_count() + 1 + usize::from(layout.lane_count() > 1) ); } - assert_eq!( - ControlRingDirection::Notifications.producer_waiting_offset() + size_of::(), - CONTROL_RING_MEMORY_SIZE - ); } #[test] - fn endpoint_roles_bind_opposite_directions_to_one_mapping() { - let local = test_ring().into_local(); - assert_eq!( - local.request_producer.direction(), - ControlRingDirection::Requests - ); - assert_eq!( - local.response_consumer.direction(), - ControlRingDirection::Responses - ); - assert_eq!( - local.notification_consumer.direction(), - ControlRingDirection::Notifications + fn single_slot_lanes_carry_one_call_at_a_time() { + let layout = ControlRingLayout::MULTI_LANE; + let ring = Arc::new(test_ring_with(layout)); + let mut lanes: Vec<_> = (0..layout.lane_count()) + .map(|lane| { + let local = LocalControlRingLane { + request_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }; + let broker = HostControlRingLane { + request_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }; + (local, broker) + }) + .collect(); + for round in 0..3_u8 { + for (index, (local, broker)) in lanes.iter_mut().enumerate() { + let request = [round, u8::try_from(index).unwrap()]; + assert_eq!( + local.request_producer.try_write(&request), + Ok(ControlRingWriteStatus::Written) + ); + assert!(matches!( + local.request_producer.try_write(&request), + Ok(ControlRingWriteStatus::Full { .. }) + )); + assert_eq!( + broker.request_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Message(request.to_vec())) + ); + assert!(matches!( + broker.request_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Empty { .. }) + )); + broker.request_consumer.publish_head().unwrap(); + let response = [round, u8::try_from(index).unwrap(), 0xff]; + assert_eq!( + broker.response_producer.try_write(&response), + Ok(ControlRingWriteStatus::Written) + ); + assert_eq!( + local.response_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Message(response.to_vec())) + ); + local.response_consumer.publish_head().unwrap(); + } + } + } + + #[test] + fn broker_hands_out_lanes_in_order_as_demand_rises() { + let ring = Arc::new(test_ring_with(ControlRingLayout::MULTI_LANE)); + let local = LocalLaneDemand { + ring: Arc::clone(&ring), + }; + let mut broker = HostLaneDemand { + ring: Arc::clone(&ring), + started_lane_count: 0, + }; + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + let Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) = broker.try_next_lane() else { + panic!("only lane 0 is demanded initially"); + }; + local.request(3).unwrap(); + let epoch_offset = ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET; + assert_ne!( + ring.memory.load_u32_acquire(epoch_offset).unwrap(), + wait_epoch ); - assert!(Arc::ptr_eq( - &local.request_producer.ring, - &local.response_consumer.ring - )); - assert!(Arc::ptr_eq( - &local.request_producer.ring, - &local.notification_consumer.ring + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(1))); + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(2))); + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); - let broker = test_ring().into_broker(); - assert_eq!( - broker.request_consumer.direction(), - ControlRingDirection::Requests - ); - assert_eq!( - broker.response_producer.direction(), - ControlRingDirection::Responses - ); - assert_eq!( - broker.notification_producer.direction(), - ControlRingDirection::Notifications - ); - assert!(Arc::ptr_eq( - &broker.request_consumer.ring, - &broker.response_producer.ring + // A hostile peer can lower or overstate demand; the broker never + // takes back a lane and never exceeds the layout. + local.request(1).unwrap(); + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); - assert!(Arc::ptr_eq( - &broker.request_consumer.ring, - &broker.notification_producer.ring + ring.memory + .store_u32_release( + ring.layout.demand_offset() + DEMANDED_LANE_COUNT_OFFSET, + u32::MAX.to_le(), + ) + .unwrap(); + for lane in 3..ControlRingLayout::MULTI_LANE.lane_count() { + assert_eq!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Lane(lane)) + ); + } + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); } #[test] fn request_response_and_notification_rings_publish_independently() { let ring = Arc::new(test_ring()); - let mut request_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Requests); - let mut request_consumer = - ControlRingConsumer::new(Arc::clone(&ring), ControlRingDirection::Requests); - let mut response_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Responses); - let mut response_consumer = - ControlRingConsumer::new(Arc::clone(&ring), ControlRingDirection::Responses); - let mut notification_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Notifications); - let mut notification_consumer = - ControlRingConsumer::new(ring, ControlRingDirection::Notifications); + let mut request_producer = ControlRingProducer::new(Arc::clone(&ring), REQUESTS); + let mut request_consumer = ControlRingConsumer::new(Arc::clone(&ring), REQUESTS); + let mut response_producer = ControlRingProducer::new(Arc::clone(&ring), RESPONSES); + let mut response_consumer = ControlRingConsumer::new(Arc::clone(&ring), RESPONSES); + let mut notification_producer = ControlRingProducer::new(Arc::clone(&ring), NOTIFICATIONS); + let mut notification_consumer = ControlRingConsumer::new(ring, NOTIFICATIONS); assert_eq!( request_producer.try_write(&[1, 2, 3]), @@ -1181,11 +1652,7 @@ mod tests { #[test] fn each_direction_enforces_its_own_capacity() { - for direction in [ - ControlRingDirection::Requests, - ControlRingDirection::Responses, - ControlRingDirection::Notifications, - ] { + for direction in [REQUESTS, RESPONSES, NOTIFICATIONS] { let (mut producer, mut consumer) = test_ring().into_endpoints(direction, direction); for value in 0..direction.slot_count() { assert_eq!( @@ -1232,10 +1699,7 @@ mod tests { CONTROL_RING_SLOT_HEADER_SIZE - size_of::() ), (0, size_of::()), - ( - ControlRingDirection::Requests.producer_epoch_offset(), - size_of::() - ), + (REQUESTS.producer_epoch_offset(), size_of::()), ] ); } @@ -1243,11 +1707,8 @@ mod tests { #[test] fn failed_payload_metadata_or_word_publication_does_not_publish_progress() { for failed_write in [1, 2, 3] { - let ring = ControlRing::new(FailingWriteMemory::new()).unwrap(); - let (mut producer, mut consumer) = ring.into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(FailingWriteMemory::new(), LAYOUT).unwrap(); + let (mut producer, mut consumer) = ring.into_endpoints(REQUESTS, REQUESTS); producer.try_write(&[7]).unwrap(); producer.memory().fail_after(failed_write); assert_eq!( @@ -1282,14 +1743,14 @@ mod tests { let (mut producer, mut consumer) = test_endpoints(); producer.try_write(&[7; 100]).unwrap(); - for _ in 1..CONTROL_RING_SLOT_COUNT { + for _ in 1..SLOT_COUNT { producer.try_write(&[8]).unwrap(); } assert_eq!( consumer.try_read(|payload| Ok::<_, ()>(payload.len())), Ok(ControlRingReadStatus::Message(100)) ); - for _ in 1..CONTROL_RING_SLOT_COUNT { + for _ in 1..SLOT_COUNT { consumer.try_read(owned_bytes).unwrap(); } consumer.publish_head().unwrap(); @@ -1310,7 +1771,7 @@ mod tests { fn shared_epochs_publish_and_acknowledge_a_full_batch() { let (mut producer, mut consumer) = test_endpoints(); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..SLOT_COUNT { let value = u8::try_from(value).unwrap(); assert_eq!( producer.try_write(&[value]), @@ -1322,7 +1783,7 @@ mod tests { Ok(ControlRingWriteStatus::Full { wait_epoch: 0 }) ); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..SLOT_COUNT { let value = u8::try_from(value).unwrap(); assert_eq!( consumer.try_read(owned_bytes), @@ -1332,7 +1793,7 @@ mod tests { assert_eq!( consumer.try_read(owned_bytes), Ok(ControlRingReadStatus::Empty { - wait_epoch: u32::try_from(CONTROL_RING_SLOT_COUNT).unwrap() + wait_epoch: u32::try_from(SLOT_COUNT).unwrap() }) ); assert_eq!( @@ -1355,10 +1816,9 @@ mod tests { fn wakeup_epochs_wrap_without_controlling_ring_progress() { let (mut producer, mut consumer) = test_endpoints(); - producer.memory().store_u32_for_test( - ControlRingDirection::Requests.producer_epoch_offset(), - u32::MAX, - ); + producer + .memory() + .store_u32_for_test(REQUESTS.producer_epoch_offset(), u32::MAX); producer.try_write(&[7]).unwrap(); assert_eq!( consumer.try_read(owned_bytes), @@ -1369,15 +1829,14 @@ mod tests { Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) ); - consumer.memory().store_u32_for_test( - ControlRingDirection::Requests.consumer_epoch_offset(), - u32::MAX, - ); + consumer + .memory() + .store_u32_for_test(REQUESTS.consumer_epoch_offset(), u32::MAX); consumer.publish_head().unwrap(); assert_eq!( producer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(0) ); assert_eq!(producer.refresh_head(), Ok(())); @@ -1390,7 +1849,7 @@ mod tests { assert_eq!( consumer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(0) ); @@ -1401,7 +1860,7 @@ mod tests { assert_eq!( consumer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(1) ); } @@ -1438,10 +1897,7 @@ mod tests { fn assert_rejected(image: &[u8; CONTROL_RING_SLOT_SIZE], expected: ControlRingError) { let ring = test_ring(); install_slot(ring.memory(), image); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), Err(ControlRingReadError::Ring(expected)) @@ -1480,7 +1936,7 @@ mod tests { Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) ); install_slot(consumer.memory(), &raw_slot(1, 1, 0, &[7])); - consumer.head = CONTROL_RING_SLOT_COUNT; + consumer.head = SLOT_COUNT; assert_eq!( consumer.try_read(owned_bytes), Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) @@ -1557,11 +2013,8 @@ mod tests { #[test] fn torn_length_snapshot_is_rejected_before_payload_slicing() { let memory = TearingMemory::with_torn_length(&raw_slot(1, 1, 0, &[7])); - let ring = ControlRing::new(memory).unwrap(); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), @@ -1577,11 +2030,8 @@ mod tests { #[test] fn sequence_change_during_body_copy_is_rejected() { let memory = TearingMemory::with_changed_sequence(&raw_slot(1, 1, 0, &[7])); - let ring = ControlRing::new(memory).unwrap(); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), @@ -1630,17 +2080,18 @@ mod tests { } fn test_ring() -> ControlRing { - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)).unwrap() + test_ring_with(LAYOUT) + } + + fn test_ring_with(layout: ControlRingLayout) -> ControlRing { + ControlRing::new(TestMemory::new(layout.memory_size()), layout).unwrap() } fn test_endpoints() -> ( ControlRingProducer, ControlRingConsumer, ) { - test_ring().into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ) + test_ring().into_endpoints(REQUESTS, REQUESTS) } fn owned_bytes(payload: &[u8]) -> Result, ()> { @@ -1653,14 +2104,10 @@ mod tests { #[test] fn wakes_skip_peers_that_announce_no_wait() { - let (mut producer, mut consumer) = - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)) - .unwrap() - .into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ); - let direction = ControlRingDirection::Requests; + let (mut producer, mut consumer) = ControlRing::new(TestMemory::new(MEMORY_SIZE), LAYOUT) + .unwrap() + .into_endpoints(REQUESTS, REQUESTS); + let direction = REQUESTS; assert_eq!( producer.try_write(b"x"), Ok(ControlRingWriteStatus::Written) @@ -1696,15 +2143,11 @@ mod tests { #[test] fn waits_announce_themselves_only_while_blocking() { - let (mut producer, mut consumer) = - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)) - .unwrap() - .into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); - let requests = ControlRingDirection::Requests; - let responses = ControlRingDirection::Responses; + let (mut producer, mut consumer) = ControlRing::new(TestMemory::new(MEMORY_SIZE), LAYOUT) + .unwrap() + .into_endpoints(RESPONSES, REQUESTS); + let requests = REQUESTS; + let responses = RESPONSES; let Ok(ControlRingReadStatus::Empty { wait_epoch }) = consumer.try_read(|frame| Ok::<_, ()>(frame.to_vec())) else { @@ -1763,10 +2206,7 @@ mod tests { fn store_test_consumer_head(memory: &TestMemory, head: u64) { memory - .store_u64_release( - ControlRingDirection::Requests.consumer_head_offset(), - head.to_le(), - ) + .store_u64_release(REQUESTS.consumer_head_offset(), head.to_le()) .unwrap(); } @@ -1937,7 +2377,7 @@ mod tests { impl FailingWriteMemory { fn new() -> Self { Self { - bytes: Mutex::new(vec![0; CONTROL_RING_MEMORY_SIZE]), + bytes: Mutex::new(vec![0; MEMORY_SIZE]), write_count: AtomicUsize::new(0), fail_on_write: AtomicUsize::new(usize::MAX), } @@ -2054,7 +2494,7 @@ mod tests { } fn new(first_slot: &[u8; CONTROL_RING_SLOT_SIZE], tear: Tear) -> Self { - let mut bytes = vec![0; CONTROL_RING_MEMORY_SIZE]; + let mut bytes = vec![0; MEMORY_SIZE]; bytes[..CONTROL_RING_SLOT_SIZE].copy_from_slice(first_slot); Self { bytes: Mutex::new(bytes), diff --git a/litebox_broker_transport/src/lib.rs b/litebox_broker_transport/src/lib.rs index a103f59a2d..a99c052b62 100644 --- a/litebox_broker_transport/src/lib.rs +++ b/litebox_broker_transport/src/lib.rs @@ -9,9 +9,10 @@ //! shared-buffer pool built on them, and the portable shared control-ring state //! machines. //! -//! Nothing here is tied to an operating system or to hosted userland. The crate -//! is unconditionally `no_std`, so a kernel deployment can use the same -//! interfaces and control rings. Concrete association bindings, such as the +//! Nothing here is tied to an operating system or to hosted userland, except +//! the local call-lane pool behind the opt-in `std` feature. The crate is +//! `no_std` without it, so a kernel deployment can use the same interfaces and +//! control rings. Concrete association bindings, such as the //! Unix-domain-socket and memfd endpoints in //! `litebox_broker_transport_linux_userland`, live in separate crates, and a //! deployment may implement the channel traits directly instead of using the @@ -25,11 +26,12 @@ extern crate alloc; -#[cfg(test)] +#[cfg(any(test, feature = "std"))] extern crate std; +#[cfg(feature = "std")] +pub mod call_lane_pool; pub mod channel; pub mod control_ring; -pub mod pending_calls; pub mod setup_frame; pub mod shared_memory; diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs deleted file mode 100644 index e1ea50b1d6..0000000000 --- a/litebox_broker_transport/src/pending_calls.rs +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -//! Portable pending-call coordination over caller-supplied synchronization. - -use alloc::collections::BTreeMap; -use alloc::collections::btree_map::Entry; -use alloc::sync::Arc; -use core::ops::{Deref, DerefMut}; - -use litebox_broker_protocol::RequestId; -use litebox_broker_protocol::message::BrokerResponse; - -/// Maximum number of active calls waiting for broker responses. -pub const MAX_PENDING_CALLS: usize = 64; - -/// A mutex usable by [`PendingCalls`]. -pub trait PendingCallsMutex { - /// Guard granting access to the protected value. - type Guard<'a>: Deref + DerefMut - where - Self: 'a, - T: 'a; - - /// Locks the mutex. - fn lock(&self) -> Self::Guard<'_>; -} - -/// A condition variable paired with a [`PendingCallsMutex`]. -pub trait PendingCallsCondvar { - /// Mutex type whose guard this condition variable waits on. - type Mutex: PendingCallsMutex; - - /// Atomically releases the guard and waits for a notification. - fn wait<'a>( - &self, - guard: >::Guard<'a>, - ) -> >::Guard<'a> - where - Self::Mutex: 'a, - T: 'a; - - /// Wakes one waiter. - fn notify_one(&self); - - /// Wakes all waiters. - fn notify_all(&self); -} - -/// Supplies synchronization primitives for [`PendingCalls`]. -pub trait PendingCallsSync { - /// Mutex protecting a value of type `T`. - type Mutex: PendingCallsMutex; - - /// Condition variable paired with [`Self::Mutex`]. - type Condvar: PendingCallsCondvar>; - - /// Creates a mutex protecting `value`. - fn mutex(value: T) -> Self::Mutex; - - /// Creates a condition variable. - fn condvar() -> Self::Condvar; -} - -/// Failure from pending-call registry coordination. -#[derive(Debug)] -pub enum PendingCallsError { - /// The association had already failed. - AssociationFailed(Arc), - /// The guarded operation failed. - Operation(Error), - /// A request reused an active request identifier. - DuplicateRequestId, - /// A response did not identify an active request. - UnknownResponseId, -} - -/// Concurrent registry of requests awaiting broker responses. -/// -/// Callers waiting for responses also read them: while any call is pending, -/// one waiting caller holds the reader role and completes responses for every -/// caller until its own arrives, then hands the role to another pending call. -/// A response therefore reaches a lone caller without a thread handoff. -pub struct PendingCalls { - state: Sync::Mutex>, - capacity_available: Sync::Condvar>, -} - -struct PendingCallsInner { - calls: BTreeMap>>, - failure: Option>, - /// The call whose caller holds the reader role, if any call is pending. - reader: Option>>, -} - -/// Completion state shared between a pending call and its registry. -struct CallSlot { - state: Sync::Mutex>, - changed: Sync::Condvar>, -} - -struct CallState { - result: Option>>, - /// Whether this call's caller holds the reader role. - reads_responses: bool, -} - -impl CallSlot { - fn new(reads_responses: bool) -> Self { - Self { - state: Sync::mutex(CallState { - result: None, - reads_responses, - }), - changed: Sync::condvar(), - } - } - - fn resolve(&self, result: Result>, notify: bool) { - let mut state = self.state.lock(); - assert!( - state.result.is_none(), - "broker pending call already resolved" - ); - state.result = Some(result); - if notify { - self.changed.notify_one(); - } - } - - fn grant_reader_role(&self) { - self.state.lock().reads_responses = true; - self.changed.notify_one(); - } -} - -/// One registered request awaiting its broker response. -/// -/// Dropping the call unregisters it and hands off the reader role if it holds -/// it, so an abandoned call never strands other callers. -pub struct PendingCall<'calls, Sync: PendingCallsSync, Error> { - pending_calls: &'calls PendingCalls, - request_id: RequestId, - slot: Arc>, -} - -impl PendingCall<'_, Sync, Error> { - /// Blocks until the broker responds or the association fails. - /// - /// While this caller holds the reader role, it calls `read_responses` - /// repeatedly. Each call must block until it reads at least one response - /// and passes it to [`PendingCalls::complete`], or until the association - /// fails and the failure is recorded with [`PendingCalls::record_failure`]. - /// Only the caller holding the reader role calls `read_responses`. - pub fn wait(self, mut read_responses: impl FnMut()) -> Result> { - loop { - { - let mut state = self.slot.state.lock(); - loop { - if let Some(result) = state.result.take() { - return result; - } - if state.reads_responses { - break; - } - state = self.slot.changed.wait(state); - } - } - read_responses(); - } - } -} - -impl Drop for PendingCall<'_, Sync, Error> { - fn drop(&mut self) { - self.pending_calls.unregister(self.request_id, &self.slot); - } -} - -impl PendingCalls { - /// Creates an empty live pending-call registry. - pub fn new() -> Self { - Self { - state: Sync::mutex(PendingCallsInner { - calls: BTreeMap::new(), - failure: None, - reader: None, - }), - capacity_available: Sync::condvar(), - } - } - - /// Registers a request, blocking while the pending-call limit is full. - /// - /// The first call registered while no call is pending takes the reader - /// role. - pub fn register( - &self, - request_id: RequestId, - ) -> Result, PendingCallsError> { - let mut state = self.state.lock(); - while state.calls.len() >= MAX_PENDING_CALLS && state.failure.is_none() { - state = self.capacity_available.wait(state); - } - let state = &mut *state; - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - let Entry::Vacant(entry) = state.calls.entry(request_id) else { - return Err(PendingCallsError::DuplicateRequestId); - }; - let slot = Arc::new(CallSlot::new(state.reader.is_none())); - entry.insert(Arc::clone(&slot)); - if state.reader.is_none() { - state.reader = Some(Arc::clone(&slot)); - } - Ok(PendingCall { - pending_calls: self, - request_id, - slot, - }) - } - - /// Completes the pending call identified by `response`. - pub fn complete(&self, response: BrokerResponse) -> Result<(), PendingCallsError> { - let (slot, notify) = { - let mut state = self.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - let Some(slot) = state.calls.remove(&response.request_id) else { - return Err(PendingCallsError::UnknownResponseId); - }; - // Registrations wait only while the registry is full. - if state.calls.len() + 1 == MAX_PENDING_CALLS { - self.capacity_available.notify_all(); - } - // The reader completes its own call while reading, not waiting. - let notify = !state - .reader - .as_ref() - .is_some_and(|reader| Arc::ptr_eq(reader, &slot)); - (slot, notify) - }; - slot.resolve(Ok(response), notify); - Ok(()) - } - - /// Removes `slot` if it is still registered and passes on its reader role. - fn unregister(&self, request_id: RequestId, slot: &Arc>) { - let mut state = self.state.lock(); - if state - .calls - .get(&request_id) - .is_some_and(|registered| Arc::ptr_eq(registered, slot)) - { - state.calls.remove(&request_id); - if state.calls.len() + 1 == MAX_PENDING_CALLS { - self.capacity_available.notify_all(); - } - } - let holds_reader_role = state - .reader - .as_ref() - .is_some_and(|reader| Arc::ptr_eq(reader, slot)); - if holds_reader_role && state.failure.is_none() { - state.reader = state.calls.values().next().map(|next| { - next.grant_reader_role(); - Arc::clone(next) - }); - } - } - - /// Records the first terminal failure and resolves every pending call. - /// - /// Returns whether this call recorded the first failure. - pub fn record_failure(&self, error: Arc) -> bool { - let pending_calls = { - let mut state = self.state.lock(); - if state.failure.is_some() { - return false; - } - state.failure = Some(Arc::clone(&error)); - let pending_calls = core::mem::take(&mut state.calls); - self.capacity_available.notify_all(); - pending_calls - }; - for pending_call in pending_calls.into_values() { - pending_call.resolve(Err(Arc::clone(&error)), true); - } - true - } - - /// Returns the association's terminal failure, if one was recorded. - pub fn current_failure(&self) -> Option> { - self.state.lock().failure.as_ref().map(Arc::clone) - } - - /// Runs an operation while excluding failure recording. - pub fn run_if_live( - &self, - operation: impl FnOnce() -> Result, - ) -> Result> { - let state = self.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - operation().map_err(PendingCallsError::Operation) - } -} - -impl Default for PendingCalls { - fn default() -> Self { - Self::new() - } -} diff --git a/litebox_broker_transport_linux_userland/Cargo.toml b/litebox_broker_transport_linux_userland/Cargo.toml index 382b1ac501..eda1665ec4 100644 --- a/litebox_broker_transport_linux_userland/Cargo.toml +++ b/litebox_broker_transport_linux_userland/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } -litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0" } +litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0", features = ["std"] } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/litebox_broker_transport_linux_userland/src/lib.rs b/litebox_broker_transport_linux_userland/src/lib.rs index abe11c89e0..8d5de7b14f 100644 --- a/litebox_broker_transport_linux_userland/src/lib.rs +++ b/litebox_broker_transport_linux_userland/src/lib.rs @@ -19,7 +19,6 @@ #![cfg(target_os = "linux")] -mod pending_calls; mod setup; pub mod existing_runner; diff --git a/litebox_broker_transport_linux_userland/src/memfd.rs b/litebox_broker_transport_linux_userland/src/memfd.rs index 43816d9094..ee44b2e4a4 100644 --- a/litebox_broker_transport_linux_userland/src/memfd.rs +++ b/litebox_broker_transport_linux_userland/src/memfd.rs @@ -41,12 +41,7 @@ use rustix::net::{ }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_PARALLEL_SPIN_LIMIT, MemoryAccessPolicy, - WaitableSharedMemory, -}; -#[cfg(test)] -use litebox_broker_transport::control_ring::{ - memory_permits_byte_range, memory_permits_u32, memory_permits_u64, + CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingLayout, MemoryAccessPolicy, WaitableSharedMemory, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_platform::buffer::RawBytes; @@ -207,9 +202,12 @@ impl MemfdSharedMemory { Self::create_with_policy(length, MemoryAccessPolicy::Bytes) } - /// Creates and maps a sealed memfd for one control ring. - pub fn create_control_ring() -> IoResult { - Self::create_with_policy(CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy::ControlRing) + /// Creates and maps a sealed memfd for one control ring with `layout`. + pub fn create_control_ring(layout: ControlRingLayout) -> IoResult { + Self::create_with_policy( + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), + ) } fn create_with_policy(length: usize, policy: MemoryAccessPolicy) -> IoResult { @@ -238,12 +236,12 @@ impl MemfdSharedMemory { Self::from_received_fd_with_policy(fd, expected_length, MemoryAccessPolicy::Bytes) } - /// Maps a control-ring memfd received from the broker. - pub fn control_ring_from_received_fd(fd: OwnedFd) -> IoResult { + /// Maps a control-ring memfd with `layout` received from the broker. + pub fn control_ring_from_received_fd(fd: OwnedFd, layout: ControlRingLayout) -> IoResult { Self::from_received_fd_with_policy( fd, - CONTROL_RING_MEMORY_SIZE, - MemoryAccessPolicy::ControlRing, + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), ) } @@ -288,7 +286,7 @@ impl MemfdSharedMemory { unsafe { madvise(address.as_ptr().cast(), length, Advice::LinuxDontFork) }?; let spin_limit = match policy { MemoryAccessPolicy::Bytes => 0, - MemoryAccessPolicy::ControlRing => parallel_spin_limit(), + MemoryAccessPolicy::ControlRing(_) => parallel_spin_limit(), }; Ok(Self { fd, @@ -470,13 +468,14 @@ pub fn receive_memfd( MemfdSharedMemory::from_received_fd(fd, expected_length) } -/// Receives and maps one control-ring memfd from the broker. +/// Receives and maps one control-ring memfd with `layout` from the broker. pub fn receive_control_ring_memfd( stream: &mut UnixStream, + layout: ControlRingLayout, deadline: Option, ) -> IoResult { let fd = with_read_deadline(stream, deadline, receive_fd)?; - MemfdSharedMemory::control_ring_from_received_fd(fd) + MemfdSharedMemory::control_ring_from_received_fd(fd, layout) } /// Sparse memfd holding a process memory image for a runner being started. @@ -734,8 +733,7 @@ mod tests { SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE, SharedBufferSlotIndex, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_SLOT_COUNT, ControlRing, ControlRingReadStatus, - ControlRingWriteStatus, + ControlRing, ControlRingReadStatus, ControlRingWriteStatus, }; use litebox_broker_transport::shared_memory::SharedBufferPool; use rustix::io::FdFlags; @@ -744,6 +742,9 @@ mod tests { use std::thread; use std::time::Duration; + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + const MEMORY_SIZE: usize = LAYOUT.memory_size(); + #[test] fn mappings_share_bytes_and_validate_ranges() { let first = MemfdSharedMemory::create(64).unwrap(); @@ -796,19 +797,20 @@ mod tests { #[test] fn mappings_enforce_control_ring_typed_access() { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let sequence_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u64(*offset)) + let sequence_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u64(*offset)) .unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); - let body_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_byte_range(*offset, 1)) + let body_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_byte_range(*offset, 1)) .unwrap(); first @@ -866,7 +868,7 @@ mod tests { Err(SharedMemoryError::InvalidRange) ); assert_eq!( - second.load_u64_acquire(CONTROL_RING_MEMORY_SIZE), + second.load_u64_acquire(MEMORY_SIZE), Err(SharedMemoryError::InvalidRange) ); assert_eq!( @@ -884,9 +886,9 @@ mod tests { #[test] fn futex_wait_returns_without_peer_cooperation() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); let start = Instant::now(); @@ -897,13 +899,13 @@ mod tests { #[test] fn peer_descriptor_writes_are_read_as_untrusted_snapshots() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let peer_fd = memory.as_fd().try_clone_to_owned().unwrap(); - let sequence_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u64(*offset)) + let sequence_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u64(*offset)) .unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); let mut expected_sequence = [0; size_of::()]; @@ -983,9 +985,10 @@ mod tests { let received_pool = MemfdSharedMemory::from_received_fd(pool.fd.as_fd().try_clone_to_owned().unwrap(), 64) .unwrap(); - let ring = MemfdSharedMemory::create_control_ring().unwrap(); + let ring = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let received_ring = MemfdSharedMemory::control_ring_from_received_fd( ring.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); for memory in [&pool, &received_pool, &ring, &received_ring] { @@ -1052,13 +1055,13 @@ mod tests { #[test] fn transfers_exact_sealed_control_ring_mapping() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); - let ring = ControlRing::new(memory).unwrap(); + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); let (mut receiver, mut sender) = UnixStream::pair().unwrap(); send_memfd(&mut sender, ring.memory(), None).unwrap(); - let mapped = receive_control_ring_memfd(&mut receiver, None).unwrap(); - let mapped_ring = ControlRing::new(mapped).unwrap(); + let mapped = receive_control_ring_memfd(&mut receiver, LAYOUT, None).unwrap(); + let mapped_ring = ControlRing::new(mapped, LAYOUT).unwrap(); ring.memory().write(13, &[1, 2, 3]).unwrap(); let mut bytes = [0; 3]; mapped_ring.memory().read(13, &mut bytes).unwrap(); @@ -1073,18 +1076,23 @@ mod tests { #[test] fn shared_futex_wakeup_prevents_missed_cross_mapping_work() { - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let local_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let mut producer = ControlRing::new(local_memory) + let mut producer = ControlRing::new(local_memory, LAYOUT) .unwrap() .into_local() + .lanes + .swap_remove(0) .request_producer; - let mut consumer = ControlRing::new(broker_memory) + let mut consumer = ControlRing::new(broker_memory, LAYOUT) .unwrap() - .into_broker() + .into_host() + .lanes + .swap_remove(0) .request_consumer; let empty_checked = Arc::new(Barrier::new(2)); let broker_empty_checked = Arc::clone(&empty_checked); @@ -1100,7 +1108,7 @@ mod tests { }; broker_empty_checked.wait(); consumer.wait_for_message(producer_epoch).unwrap(); - for expected in 0..CONTROL_RING_SLOT_COUNT { + for expected in 0..LAYOUT.requests(0).slot_count() { let expected = u8::try_from(expected).unwrap(); loop { match consumer @@ -1136,7 +1144,7 @@ mod tests { }); empty_checked.wait(); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..LAYOUT.requests(0).slot_count() { let value = u8::try_from(value).unwrap(); assert_eq!( producer.try_write(&[value]), @@ -1250,7 +1258,7 @@ mod tests { .kind(), ErrorKind::InvalidInput ); - let control = MemfdSharedMemory::create_control_ring().unwrap(); + let control = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); assert_eq!( image .write_from_shared(0, &control, 0..1) diff --git a/litebox_broker_transport_linux_userland/src/pending_calls.rs b/litebox_broker_transport_linux_userland/src/pending_calls.rs deleted file mode 100644 index 4d092f09c1..0000000000 --- a/litebox_broker_transport_linux_userland/src/pending_calls.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::pending_calls::{ - PendingCalls as GenericPendingCalls, PendingCallsCondvar, PendingCallsError, PendingCallsMutex, - PendingCallsSync, -}; - -use crate::setup::{copy_io_error, invalid_data}; - -pub(crate) type PendingCalls = GenericPendingCalls; - -pub(crate) struct StdPendingCallsSync; - -pub(crate) struct StdPendingCallsMutex(Mutex); - -pub(crate) struct StdPendingCallsCondvar(Condvar, PhantomData); - -impl PendingCallsMutex for StdPendingCallsMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker pending mutex poisoned") - } -} - -impl PendingCallsCondvar for StdPendingCallsCondvar { - type Mutex = StdPendingCallsMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0.wait(guard).expect("broker pending mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl PendingCallsSync for StdPendingCallsSync { - type Mutex = StdPendingCallsMutex; - type Condvar = StdPendingCallsCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdPendingCallsMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdPendingCallsCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn pending_calls_error(error: PendingCallsError) -> Error { - match error { - PendingCallsError::AssociationFailed(error) => copy_io_error(&error), - PendingCallsError::Operation(error) => error, - PendingCallsError::DuplicateRequestId => invalid_data("duplicate broker request ID"), - PendingCallsError::UnknownResponseId => { - invalid_data("broker returned an unknown response ID") - } - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc; - use std::time::Duration; - - use litebox_broker_protocol::RequestId; - use litebox_broker_protocol::message::{BrokerResponse, BrokerResult}; - use litebox_broker_transport::pending_calls::{MAX_PENDING_CALLS, PendingCallsError}; - - use super::PendingCalls; - - fn response(id: u64) -> BrokerResponse { - BrokerResponse { - request_id: RequestId(id), - result: BrokerResult::ObjectClosed, - } - } - - #[test] - fn a_full_registry_wakes_registration_when_a_call_completes() { - let pending_calls = PendingCalls::new(); - let mut calls: Vec<_> = (0..MAX_PENDING_CALLS as u64) - .map(|id| pending_calls.register(RequestId(id)).unwrap()) - .collect(); - let (registered, registration) = mpsc::channel(); - std::thread::scope(|scope| { - scope.spawn(|| { - let call = pending_calls.register(RequestId(MAX_PENDING_CALLS as u64)); - registered.send(call.is_ok()).unwrap(); - }); - assert!( - registration - .recv_timeout(Duration::from_millis(50)) - .is_err() - ); - pending_calls.complete(response(0)).unwrap(); - assert!(registration.recv_timeout(Duration::from_secs(10)).unwrap()); - }); - assert!(calls.remove(0).wait(|| unreachable!()).is_ok()); - } - - #[test] - fn a_finished_reader_hands_its_role_to_a_pending_call() { - let pending_calls = PendingCalls::new(); - let reader = pending_calls.register(RequestId(1)).unwrap(); - let follower = pending_calls.register(RequestId(2)).unwrap(); - let follower_read = AtomicBool::new(false); - std::thread::scope(|scope| { - let follower = scope.spawn(|| { - follower.wait(|| { - follower_read.store(true, Ordering::Relaxed); - pending_calls.complete(response(2)).unwrap(); - }) - }); - let result = reader.wait(|| pending_calls.complete(response(1)).unwrap()); - assert_eq!(result.unwrap().request_id, RequestId(1)); - assert_eq!(follower.join().unwrap().unwrap().request_id, RequestId(2)); - }); - assert!(follower_read.load(Ordering::Relaxed)); - } - - #[test] - fn an_abandoned_reader_hands_off_its_role() { - let pending_calls = PendingCalls::new(); - let reader = pending_calls.register(RequestId(1)).unwrap(); - let follower = pending_calls.register(RequestId(2)).unwrap(); - drop(reader); - let result = follower.wait(|| pending_calls.complete(response(2)).unwrap()); - assert_eq!(result.unwrap().request_id, RequestId(2)); - assert!(matches!( - pending_calls.complete(response(1)), - Err(PendingCallsError::UnknownResponseId) - )); - } -} diff --git a/litebox_broker_transport_linux_userland/src/setup.rs b/litebox_broker_transport_linux_userland/src/setup.rs index b08588185e..93739788d4 100644 --- a/litebox_broker_transport_linux_userland/src/setup.rs +++ b/litebox_broker_transport_linux_userland/src/setup.rs @@ -101,8 +101,8 @@ pub(crate) fn wire_error(error: WireError) -> Error { ) } -/// Clones an error so one recorded terminal failure can be reported to every -/// waiter of an association. +/// Clones an error so the recorded failure that ended an association can be +/// reported to every waiter of the association. pub(crate) fn copy_io_error(error: &Error) -> Error { match error.raw_os_error() { Some(code) => Error::from_raw_os_error(code), diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 20c6764883..b083764300 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -24,12 +24,13 @@ use litebox_broker_protocol::wire::{ encode_notification, encode_response, }; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, PeerCredential, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, + ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + HostControlRingEndpoints, HostControlRingLane, HostLaneDemand, }; use crate::memfd::{MemfdProcessImage, MemfdSharedMemory}; @@ -99,16 +100,20 @@ pub struct UnixStreamHostSetupChannel { negotiated: bool, } -/// Request-reading endpoint of an active host control-ring association. -pub struct UnixControlRingHostRequestSource { - consumer: ControlRingConsumer, +type BrokerLane = HostControlRingLane; + +/// Source of the call lanes of an active host control-ring association, which +/// hands out each lane once the runner starts using it. +pub struct UnixControlRingHostCallLaneSource { + /// Each lane, or `None` once handed out. + lanes: Vec>, + demand: HostLaneDemand, association: Arc, } -/// Shared response-writing endpoint of an active host control-ring association. -#[derive(Clone)] -pub struct UnixControlRingHostResponseSink { - producer: Arc>>, +/// One call lane of an active host control-ring association. +pub struct UnixControlRingHostCallLane { + lane: BrokerLane, association: Arc, } @@ -119,13 +124,11 @@ pub struct UnixControlRingHostShutdown { /// State shared by every activated host endpoint of one association: the setup /// socket used for liveness and teardown, terminal status, and the wake handles -/// of all three ring directions. +/// of every ring wait. struct HostRingAssociation { control_stream: UnixStream, status: Mutex, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } enum HostAssociationStatus { @@ -182,13 +185,14 @@ impl UnixStreamHostSetupChannel { } /// Consumes a negotiated setup channel into independently usable active - /// request, response, notification, and shutdown handles. + /// call-lane, notification, and shutdown handles. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( mut self, ring: ControlRing, ) -> IoResult<( - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, )> { @@ -197,43 +201,47 @@ impl UnixStreamHostSetupChannel { "broker host setup channel activated before negotiation completed", )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_host(); + let wakes = endpoints.wake_handles(); + let HostControlRingEndpoints { + lanes, + lane_demand: Some(demand), + notification_producer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let Some(ready) = read_setup_frame(&mut self.stream, self.setup_deadline)? else { return Err(Error::new( ErrorKind::UnexpectedEof, "runner closed before control-ring setup acknowledgement", )); }; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "runner sent an invalid control-ring setup acknowledgement", )); } - write_setup_frame(&mut self.stream, CONTROL_RING_READY, self.setup_deadline)?; + write_setup_frame(&mut self.stream, ready_token, self.setup_deadline)?; let shutdown_stream = self.stream.try_clone()?; - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer, - } = ring.into_broker(); let association = Arc::new(HostRingAssociation { control_stream: shutdown_stream, status: Mutex::new(HostAssociationStatus::Live), - request_wake: request_consumer.wake_handle(), - response_wake: response_producer.wake_handle(), - notification_wake: notification_producer.wake_handle(), + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() .name("litebox-runner-liveness".to_owned()) .spawn(move || monitor_host_socket(&mut self.stream, &monitor_association))?; Ok(( - UnixControlRingHostRequestSource { - consumer: request_consumer, - association: Arc::clone(&association), - }, - UnixControlRingHostResponseSink { - producer: Arc::new(Mutex::new(response_producer)), + UnixControlRingHostCallLaneSource { + lanes: lanes.into_iter().map(Some).collect(), + demand, association: Arc::clone(&association), }, UnixControlRingHostNotificationChannel { @@ -301,23 +309,78 @@ impl HostSetupChannel for UnixStreamHostSetupChannel { } } -impl UnixControlRingHostRequestSource { - /// Receives one active broker request. +impl UnixControlRingHostCallLaneSource { + /// Blocks until the runner starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` once the runner closed the association. + /// + /// # Panics + /// + /// Panics if the lane demand hands out a lane more than once. + pub fn next_lane(&mut self) -> IoResult> { + loop { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + match self.demand.try_next_lane() { + Ok(ControlRingDemandStatus::Lane(index)) => { + let lane = self.lanes[index] + .take() + .expect("lane demand hands out each lane once"); + return Ok(Some(UnixControlRingHostCallLane { + lane, + association: Arc::clone(&self.association), + })); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) => { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + if let Err(error) = self.demand.wait_for_demand(wait_epoch) { + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + Err(error) => { + let error = ring_error(error); + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + } + } +} + +impl HostCallLaneSource for UnixControlRingHostCallLaneSource { + type Error = Error; + type Lane = UnixControlRingHostCallLane; + + fn next_lane(&mut self) -> IoResult> { + Self::next_lane(self) + } +} + +impl UnixControlRingHostCallLane { + /// Receives this lane's next broker request. pub fn recv_request(&mut self) -> IoResult> { + let consumer = &mut self.lane.request_consumer; loop { if let Some(error) = self.association.current_failure() { return Err(error); } - match self.consumer.try_read(decode_request) { + match consumer.try_read(decode_request) { Ok(ControlRingReadStatus::Message(request)) => { - self.association.acknowledge_request(&mut self.consumer)?; + self.association.acknowledge_request(consumer)?; return Ok(HostReceive::Message(request)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(terminal) = self.association.request_terminal_result() { - return terminal; + if let Some(ended) = self.association.ended() { + return ended.map(|()| HostReceive::PeerClosed); } - if let Err(error) = self.consumer.wait_for_message(wait_epoch) { + if let Err(error) = consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); let _ = self.association.fail(error); return result; @@ -341,26 +404,13 @@ impl UnixControlRingHostRequestSource { } } } -} - -impl HostRequestSource for UnixControlRingHostRequestSource { - type Error = Error; - fn recv_request(&mut self) -> IoResult> { - Self::recv_request(self) - } -} - -impl UnixControlRingHostResponseSink { - /// Serializes and sends one complete active broker response. - pub fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + /// Serializes and sends the response to this lane's last request. + pub fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { let frame = encode_response(response.clone()); - let mut producer = self - .producer - .lock() - .map_err(|_| Error::other("broker response writer mutex poisoned"))?; + let producer = &mut self.lane.response_producer; loop { - match self.association.try_publish(&mut producer, &frame)? { + match self.association.try_publish(producer, &frame)? { ControlRingWriteStatus::Written => return Ok(()), ControlRingWriteStatus::Full { wait_epoch } => { if let Err(error) = producer.wait_for_capacity(wait_epoch) { @@ -374,10 +424,14 @@ impl UnixControlRingHostResponseSink { } } -impl HostResponseSink for UnixControlRingHostResponseSink { +impl HostCallLane for UnixControlRingHostCallLane { type Error = Error; - fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + fn recv_request(&mut self) -> IoResult> { + Self::recv_request(self) + } + + fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { Self::send_response(self, response) } } @@ -438,12 +492,10 @@ impl HostRingAssociation { *status = HostAssociationStatus::Failed(Arc::new(error)); } } - let request_wake = self.request_wake.interrupt_wait(); - let response_wake = self.response_wake.interrupt_wait(); - let notification_wake = self.notification_wake.interrupt_wait(); - request_wake - .and(response_wake) - .and(notification_wake) + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) .and(shutdown_socket(&self.control_stream)) } @@ -457,19 +509,21 @@ impl HostRingAssociation { *status = HostAssociationStatus::PeerClosed; } } - let _ = self.request_wake.interrupt_wait(); - let _ = self.response_wake.interrupt_wait(); - let _ = self.notification_wake.interrupt_wait(); + for wake in &self.wakes { + let _ = wake.interrupt_wait(); + } } - fn request_terminal_result(&self) -> Option>> { + /// Returns how the association ended, if it has: `Ok` if the runner + /// closed it. + fn ended(&self) -> Option> { match &*self .status .lock() .expect("broker host association mutex poisoned") { HostAssociationStatus::Live => None, - HostAssociationStatus::PeerClosed => Some(Ok(HostReceive::PeerClosed)), + HostAssociationStatus::PeerClosed => Some(Ok(())), HostAssociationStatus::Failed(error) => Some(Err(copy_io_error(error))), } } @@ -562,7 +616,7 @@ mod tests { LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_NOTIFICATION_SLOT_COUNT, CONTROL_RING_SLOT_COUNT, LocalControlRingEndpoints, + CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRingLayout, LocalControlRingEndpoints, }; use crate::unix_socket::local::{ @@ -575,19 +629,23 @@ mod tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; + type Local = LocalControlRingEndpoints; + + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; fn ring_pair() -> ( ControlRing, ControlRing, ) { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); ( - ControlRing::new(first).unwrap(), - ControlRing::new(second).unwrap(), + ControlRing::new(first, LAYOUT).unwrap(), + ControlRing::new(second, LAYOUT).unwrap(), ) } @@ -635,39 +693,43 @@ mod tests { } fn activate_host() -> ( - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostShutdown, - Producer, - Consumer, + Local, UnixStream, ) { let (peer_stream, host_stream) = UnixStream::pair().unwrap(); let mut ack_stream = peer_stream.try_clone().unwrap(); let acknowledgement = thread::spawn(move || { - write_setup_frame(&mut ack_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut ack_stream, LAYOUT.ready_token(), None).unwrap(); assert_eq!( read_setup_frame(&mut ack_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); }); let (local_ring, host_ring) = ring_pair(); let channel = negotiated_host(host_stream); - let (source, sink, _notifications, shutdown) = channel.into_active(host_ring).unwrap(); + let (lanes, _notifications, shutdown) = channel.into_active(host_ring).unwrap(); acknowledgement.join().unwrap(); - let LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer: _, - } = local_ring.into_local(); - ( - source, - sink, - shutdown, - request_producer, - response_consumer, - peer_stream, - ) + (lanes, shutdown, local_ring.into_local(), peer_stream) + } + + /// Activates a host association and takes its always-demanded first lane. + fn activate_host_lane() -> ( + UnixControlRingHostCallLane, + UnixControlRingHostShutdown, + Local, + UnixStream, + ) { + let (mut lanes, shutdown, local, peer) = activate_host(); + let lane = lanes.next_lane().unwrap().unwrap(); + (lane, shutdown, local, peer) + } + + fn demand_lanes(local: &Local, lanes: usize) { + let demand = local.lane_demand.as_ref().unwrap(); + demand.request(lanes).unwrap(); + demand.wake_host().unwrap(); } fn notification_channel_pair() -> ( @@ -681,7 +743,7 @@ mod tests { let host_active = thread::spawn(move || host_control.into_active(host_ring).unwrap()); let (local_call, local_notifications, _local_shutdown) = local_setup.into_active(local_ring, || {}).unwrap(); - let (_source, _sink, host_notifications, shutdown) = host_active.join().unwrap(); + let (_lanes, host_notifications, shutdown) = host_active.join().unwrap(); ( local_call, local_notifications, @@ -765,119 +827,163 @@ mod tests { let host_active = thread::spawn(move || host.into_active(host_ring).unwrap()); let (local, _local_notifications, _local_shutdown) = local_setup.into_active(local_ring, || {}).unwrap(); - let (mut source, sink, _host_notifications, _shutdown) = host_active.join().unwrap(); + let (mut lanes, _host_notifications, _shutdown) = host_active.join().unwrap(); let caller = thread::spawn(move || local.call(request(7))); - let HostReceive::Message(received) = source.recv_request().unwrap() else { + let mut lane = lanes.next_lane().unwrap().unwrap(); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { panic!("expected ring request"); }; - sink.send_response(&response(received.request_id)).unwrap(); + lane.send_response(&response(received.request_id)).unwrap(); assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(7)); } #[test] - fn host_activation_decodes_requests_and_cloned_sinks_publish_complete_responses() { - let (mut source, sink, _shutdown, mut requests, mut responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(1))); - assert!(matches!( - source.recv_request().unwrap(), - HostReceive::Message(BrokerRequest { - request_id: RequestId(1), - .. - }) - )); + fn host_rejects_single_lane_layouts() { + let layout = ControlRingLayout::SINGLE_LANE; + let memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); + let (_peer, host_stream) = UnixStream::pair().unwrap(); + let Err(error) = + negotiated_host(host_stream).into_active(ControlRing::new(memory, layout).unwrap()) + else { + panic!("a single-lane layout was activated"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn host_hands_out_lanes_on_demand_and_each_lane_serves_its_own_calls() { + let (mut lanes, _shutdown, mut local, peer) = activate_host(); + let mut first = lanes.next_lane().unwrap().unwrap(); + let next = thread::spawn(move || lanes.next_lane().map(|lane| (lane, lanes))); + thread::sleep(Duration::from_millis(20)); + assert!(!next.is_finished()); + demand_lanes(&local, 2); + let (second, mut lanes) = next.join().unwrap().unwrap(); + let mut second = second.unwrap(); + + for (index, lane) in [&mut first, &mut second].into_iter().enumerate() { + let id = u64::try_from(index).unwrap(); + write_payload( + &mut local.lanes[index].request_producer, + &encode_request(request(id)), + ); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { + panic!("expected ring request"); + }; + assert_eq!(received.request_id, RequestId(id)); + lane.send_response(&response(received.request_id)).unwrap(); + } + for (index, lane) in local.lanes.iter_mut().take(2).enumerate() { + assert_eq!( + read_response(&mut lane.response_consumer).request_id, + RequestId(u64::try_from(index).unwrap()) + ); + } - let first = sink.clone(); - let writer = thread::spawn(move || first.send_response(&response(RequestId(3)))); - sink.send_response(&response(RequestId(7))).unwrap(); - writer.join().unwrap().unwrap(); - let mut ids = [ - read_response(&mut responses).request_id, - read_response(&mut responses).request_id, - ]; - ids.sort(); - assert_eq!(ids, [RequestId(3), RequestId(7)]); + // Each remaining lane is handed out once; then the source waits until + // the peer closes. + demand_lanes(&local, LAYOUT.lane_count()); + for _ in 2..LAYOUT.lane_count() { + assert!(lanes.next_lane().unwrap().is_some()); + } + let last = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + assert!(!last.is_finished()); + drop(peer); + assert!(last.join().unwrap().unwrap().is_none()); } #[test] - fn host_clean_close_wakes_request_wait_as_peer_closed() { - let (mut source, _sink, _shutdown, _requests, _responses, peer) = activate_host(); - let receiver = thread::spawn(move || source.recv_request()); + fn host_clean_close_wakes_request_and_lane_waits_as_peer_closed() { + let (mut lanes, _shutdown, _local, peer) = activate_host(); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); drop(peer); assert_eq!(receiver.join().unwrap().unwrap(), HostReceive::PeerClosed); + assert!(next.join().unwrap().unwrap().is_none()); } #[test] fn host_failure_preempts_queued_and_decoded_requests_but_peer_close_drains() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(1))); - source - .association + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(1)), + ); + lane.association .fail(Error::new(ErrorKind::TimedOut, "test failure")) .unwrap(); - assert_eq!( - source.recv_request().unwrap_err().kind(), - ErrorKind::TimedOut - ); + assert_eq!(lane.recv_request().unwrap_err().kind(), ErrorKind::TimedOut); - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(2))); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(2)), + ); assert!(matches!( - source.consumer.try_read(decode_request).unwrap(), + lane.lane.request_consumer.try_read(decode_request).unwrap(), ControlRingReadStatus::Message(_) )); - source - .association + lane.association .fail(Error::new(ErrorKind::TimedOut, "test failure")) .unwrap(); assert_eq!( - source - .association - .acknowledge_request(&mut source.consumer) + lane.association + .acknowledge_request(&mut lane.lane.request_consumer) .unwrap_err() .kind(), ErrorKind::TimedOut ); - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(3))); - source.association.peer_closed(); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(3)), + ); + lane.association.peer_closed(); assert!(matches!( - source.recv_request().unwrap(), + lane.recv_request().unwrap(), HostReceive::Message(BrokerRequest { request_id: RequestId(3), .. }) )); - assert_eq!(source.recv_request().unwrap(), HostReceive::PeerClosed); + assert_eq!(lane.recv_request().unwrap(), HostReceive::PeerClosed); } #[test] - fn dropping_host_shutdown_guard_wakes_request_wait_and_closes_socket() { - let (mut source, sink, shutdown, _requests, _responses, mut peer) = activate_host(); + fn dropping_host_shutdown_guard_wakes_waits_and_closes_socket() { + let (mut lanes, shutdown, _local, mut peer) = activate_host(); peer.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); - let receiver = thread::spawn(move || source.recv_request()); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); - drop(sink); drop(shutdown); assert_eq!( receiver.join().unwrap().unwrap_err().kind(), ErrorKind::ConnectionAborted ); + assert_eq!( + next.join().unwrap().err().map(|error| error.kind()), + Some(ErrorKind::ConnectionAborted) + ); let mut byte = [0]; assert_eq!(peer.read(&mut byte).unwrap(), 0); } #[test] - fn host_close_wakes_response_producer_blocked_on_full_ring() { - let (_source, sink, _shutdown, _requests, _responses, peer) = activate_host(); - for id in 0..CONTROL_RING_SLOT_COUNT { - sink.send_response(&response(RequestId(id))).unwrap(); - } - let blocked_sink = sink.clone(); - let blocked = thread::spawn(move || blocked_sink.send_response(&response(RequestId(99)))); + fn host_close_wakes_response_producer_blocked_on_full_lane() { + let (mut lane, _shutdown, _local, peer) = activate_host_lane(); + lane.send_response(&response(RequestId(1))).unwrap(); + let blocked = thread::spawn(move || lane.send_response(&response(RequestId(2)))); thread::sleep(Duration::from_millis(20)); + assert!(!blocked.is_finished()); drop(peer); assert_eq!( blocked.join().unwrap().unwrap_err().kind(), @@ -887,25 +993,22 @@ mod tests { #[test] fn host_reports_wrong_phase_ring_message_as_protocol_violation() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); write_payload( - &mut requests, + &mut local.lanes[0].request_producer, &encode_handshake_request(BrokerHandshakeRequest { protocol_version: BROKER_PROTOCOL_VERSION, }), ); - assert_eq!( - source.recv_request().unwrap(), - HostReceive::ProtocolViolation - ); + assert_eq!(lane.recv_request().unwrap(), HostReceive::ProtocolViolation); } #[test] fn malformed_host_ring_request_is_fatal_invalid_data() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &[u8::MAX]); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload(&mut local.lanes[0].request_producer, &[u8::MAX]); assert_eq!( - source.recv_request().unwrap_err().kind(), + lane.recv_request().unwrap_err().kind(), ErrorKind::InvalidData ); } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index d8009c6480..b64e43c143 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -11,8 +11,8 @@ use std::io::{Error, ErrorKind, Read, Result as IoResult}; use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::path::Path; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -30,17 +30,17 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; +use litebox_broker_transport::call_lane_pool::{LaneClaim, LocalCallLanePool}; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; -pub use litebox_broker_transport::pending_calls::MAX_PENDING_CALLS; use crate::memfd::MemfdSharedMemory; -use crate::pending_calls::{PendingCalls, pending_calls_error}; use crate::setup::{ copy_io_error, invalid_data, read_setup_frame, ring_error, shutdown_socket, wire_error, write_setup_frame, @@ -66,22 +66,22 @@ pub struct UnixControlRingLocalShutdown { association: Arc, } +type LocalLane = LocalControlRingLane; + /// State shared by every activated local endpoint of one association: the -/// request producer, the response consumer read by whichever caller holds the -/// pending-call reader role, the setup socket used for liveness and teardown, -/// pending call tracking, and the wake handles of all three ring directions. +/// call lanes and their demand, the setup socket used for liveness and +/// teardown, and the wake handles of every ring wait. struct LocalRingAssociation { - request_producer: Mutex>, - response_consumer: Mutex>, + /// The call lanes, which also record the failure that ended the + /// association. + lanes: LocalCallLanePool, + lane_demand: LocalLaneDemand, control_stream: UnixStream, - pending_calls: Arc, /// Whether the call channel was dropped, which ends the association as a /// local close rather than a failure. closed: AtomicBool, on_failure: Arc, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } /// Local notification receiver for a shared-ring Unix broker association. @@ -137,12 +137,14 @@ impl UnixStreamLocalSetupChannel { crate::memfd::receive_memfd(&mut self.stream, expected_len, deadline) } - /// Receives the control-ring memfd offered by the broker during setup. + /// Receives the control-ring memfd with `layout` offered by the broker + /// during setup. pub fn receive_control_ring( &mut self, + layout: ControlRingLayout, deadline: Option, ) -> IoResult { - crate::memfd::receive_control_ring_memfd(&mut self.stream, deadline) + crate::memfd::receive_control_ring_memfd(&mut self.stream, layout, deadline) } /// Receives the optional process image offered by the broker during @@ -159,7 +161,7 @@ impl UnixStreamLocalSetupChannel { /// monitor. /// /// The ring must be the control-ring memfd received during this - /// setup exchange. + /// setup exchange, with a layout of more than one lane. pub fn into_active( self, ring: ControlRing, @@ -175,38 +177,42 @@ impl UnixStreamLocalSetupChannel { )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_local(); + let wakes = endpoints.wake_handles(); + let LocalControlRingEndpoints { + lanes, + lane_demand: Some(lane_demand), + notification_consumer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let mut setup_stream = self.stream; - write_setup_frame(&mut setup_stream, CONTROL_RING_READY, self.setup_deadline)?; + write_setup_frame(&mut setup_stream, ready_token, self.setup_deadline)?; let Some(ready) = read_setup_frame(&mut setup_stream, self.setup_deadline)? else { return Err(Error::new( ErrorKind::UnexpectedEof, "broker closed before control-ring setup acknowledgement", )); }; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "broker sent an invalid control-ring setup acknowledgement", )); } let shutdown_stream = setup_stream.try_clone()?; - let litebox_broker_transport::control_ring::LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer, - } = ring.into_local(); - let pending_calls = Arc::new(PendingCalls::new()); - let on_failure: Arc = Arc::new(on_failure); let association = Arc::new(LocalRingAssociation { - request_wake: request_producer.wake_handle(), - request_producer: Mutex::new(request_producer), + lanes: LocalCallLanePool::new(lanes), + lane_demand, control_stream: shutdown_stream, - pending_calls: Arc::clone(&pending_calls), closed: AtomicBool::new(false), - on_failure, - response_wake: response_consumer.wake_handle(), - response_consumer: Mutex::new(response_consumer), - notification_wake: notification_consumer.wake_handle(), + on_failure: Arc::new(on_failure), + wakes, }); let monitor_association = Arc::clone(&association); if let Err(error) = thread::Builder::new() @@ -338,46 +344,25 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; - let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; - let request_frame = encode_request(request); - - let write_result = { - let mut producer = association - .request_producer - .lock() - .expect("broker request writer mutex poisoned"); - loop { - let write_status = association - .pending_calls - .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) - .map_err(pending_calls_error); - match write_status { - Ok(ControlRingWriteStatus::Written) => { - if let Err(error) = producer.wake_consumer() { - break Err(error); - } - break Ok(()); - } - Ok(ControlRingWriteStatus::Full { wait_epoch }) => { - if let Err(error) = producer.wait_for_capacity(wait_epoch) { - break Err(error); - } - } - Err(error) => break Err(error), - } + let mut lane = match association + .lanes + .claim(|lanes| association.demand_lanes(lanes)) + { + Ok(lane) => lane, + Err(error) => { + let result = Err(copy_io_error(&error)); + let _ = association.fail(error); + return result; } }; - if let Err(error) = write_result { - let _ = association.fail(error); + let result = association.call_on_lane(&mut lane, request); + // A failed call can leave its lane mid-exchange: fail the association + // before freeing the lane, so no other call uses it. + if let Err(error) = &result { + let _ = association.fail(copy_io_error(error)); } - - pending_call - .wait(|| association.read_response()) - .map_err(|error| copy_io_error(&error)) + drop(lane); + result } } @@ -386,8 +371,8 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { - if let Some(error) = self.association.pending_calls.current_failure() { - return self.association.end_notifications(&error); + if let Some(error) = self.association.lanes.current_failure() { + return self.association.end_notifications(error); } match self.consumer.try_read(decode_notification) { Ok(ControlRingReadStatus::Message(notification)) => { @@ -395,13 +380,13 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { .association .acknowledge_notification(&mut self.consumer) { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(error) = self.association.pending_calls.current_failure() { - return self.association.end_notifications(&error); + if let Some(error) = self.association.lanes.current_failure() { + return self.association.end_notifications(error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); @@ -429,11 +414,11 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { impl LocalRingAssociation { /// Ends notification receipt once the association has ended with /// `failure`: cleanly if the local side closed it. - fn end_notifications(&self, failure: &Error) -> IoResult> { + fn end_notifications(&self, failure: Error) -> IoResult> { if self.closed.load(Ordering::Acquire) { Ok(None) } else { - Err(copy_io_error(failure)) + Err(failure) } } @@ -441,15 +426,12 @@ impl LocalRingAssociation { &self, consumer: &mut ControlRingConsumer, ) -> IoResult<()> { - let result = self - .pending_calls - .run_if_live(|| { - consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - }) - .map_err(pending_calls_error); + let result = self.lanes.run_if_live(|| { + consumer + .publish_head() + .map_err(ring_error) + .and_then(|()| consumer.wake_producer()) + }); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -458,64 +440,74 @@ impl LocalRingAssociation { Ok(()) } - /// Reads one response for the caller holding the pending-call reader - /// role, blocking until one arrives or the association fails. - fn read_response(&self) { - let mut consumer = self - .response_consumer - .lock() - .expect("broker response reader mutex poisoned"); + /// Raises the lane demand to `lanes` and wakes the broker to serve them. + fn demand_lanes(&self, lanes: usize) -> IoResult<()> { + self.lane_demand.request(lanes).map_err(ring_error)?; + self.lane_demand.wake_host() + } + + /// Writes `request` to `lane` and reads the broker's response from it, + /// releasing the lane once the call succeeds. + fn call_on_lane( + &self, + lane: &mut LaneClaim<'_, LocalLane>, + request: BrokerRequest, + ) -> IoResult { + let request_id = request.request_id; + let request_frame = encode_request(request); + let producer = &mut lane.request_producer; loop { - match consumer.try_read(decode_response) { - Ok(ControlRingReadStatus::Message(response)) => { - if let Err(error) = consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - .and_then(|()| { - self.pending_calls - .complete(response) - .map_err(pending_calls_error) - }) - { - let _ = self.fail(error); - } - return; + let write_status = self + .lanes + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error))?; + match write_status { + ControlRingWriteStatus::Written => break, + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch)?; } + } + } + producer.wake_consumer()?; + + let consumer = &mut lane.response_consumer; + let response = loop { + match consumer.try_read(decode_response) { + Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if self.pending_calls.current_failure().is_some() { - return; + if let Some(error) = self.lanes.current_failure() { + return Err(error); } - if let Err(error) = consumer.wait_for_message(wait_epoch) { - let _ = self.fail(error); - return; - } - } - Err(ControlRingReadError::Ring(error)) => { - let _ = self.fail(ring_error(error)); - return; - } - Err(ControlRingReadError::Decode(error)) => { - let _ = self.fail(wire_error(error)); - return; + consumer.wait_for_message(wait_epoch)?; } + Err(ControlRingReadError::Ring(error)) => return Err(ring_error(error)), + Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } + }; + if response.request_id != request_id { + return Err(invalid_data("broker answered a different request")); } + // Accepting the response with the lane's release excludes failure + // recording, so no call succeeds once the association has failed. + lane.release_if_live(|lane| { + let consumer = &mut lane.response_consumer; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer() + })?; + Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - let first_failure = self.pending_calls.record_failure(Arc::new(error)); - let request_wake = self.request_wake.interrupt_wait(); - let response_wake = self.response_wake.interrupt_wait(); - let notification_wake = self.notification_wake.interrupt_wait(); + let first_failure = self.lanes.record_failure(error); + let wakes = self + .wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and); let shutdown_result = shutdown_socket(&self.control_stream); if first_failure { (self.on_failure)(); } - request_wake - .and(response_wake) - .and(notification_wake) - .and(shutdown_result) + wakes.and(shutdown_result) } } @@ -554,6 +546,9 @@ mod control_ring_tests { }; use litebox_broker_protocol::{ObjectHandle, RequestId}; use litebox_broker_transport::channel::{LocalCallChannel, LocalSetupChannel}; + use litebox_broker_transport::control_ring::{ + ControlRingDemandStatus, ControlRingProducer, HostControlRingEndpoints, + }; use rustix::fs::{OFlags, fcntl_getfl}; use std::io::{Read, Write}; use std::os::fd::AsFd; @@ -564,6 +559,9 @@ mod control_ring_tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; + type Broker = HostControlRingEndpoints; + + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; struct TestSocketPath(PathBuf); @@ -615,14 +613,15 @@ mod control_ring_tests { ControlRing, ControlRing, ) { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); ( - ControlRing::new(first).unwrap(), - ControlRing::new(second).unwrap(), + ControlRing::new(first, LAYOUT).unwrap(), + ControlRing::new(second, LAYOUT).unwrap(), ) } @@ -640,8 +639,7 @@ mod control_ring_tests { UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, - Producer, - Consumer, + Broker, UnixStream, ) { let (local_stream, peer_stream) = UnixStream::pair().unwrap(); @@ -649,25 +647,19 @@ mod control_ring_tests { let acknowledgement = thread::spawn(move || { assert_eq!( read_setup_frame(&mut ack_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); - write_setup_frame(&mut ack_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut ack_stream, LAYOUT.ready_token(), None).unwrap(); }); let (local_ring, broker_ring) = ring_pair(); let setup = negotiated_local(local_stream); let (channel, notifications, shutdown) = setup.into_active(local_ring, on_failure).unwrap(); acknowledgement.join().unwrap(); - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer: _, - } = broker_ring.into_broker(); ( channel, notifications, shutdown, - response_producer, - request_consumer, + broker_ring.into_host(), peer_stream, ) } @@ -759,9 +751,9 @@ mod control_ring_tests { let acknowledgement = thread::spawn(move || { assert_eq!( read_setup_frame(&mut host_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); - write_setup_frame(&mut host_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut host_stream, LAYOUT.ready_token(), None).unwrap(); host_stream }); let (ring, _) = ring_pair(); @@ -777,9 +769,21 @@ mod control_ring_tests { } #[test] - fn local_matches_out_of_order_ring_responses_without_socket_frames() { - let (channel, _notifications, _shutdown, mut responses, mut requests, mut peer) = - activate_local(|| {}); + fn single_lane_layouts_are_rejected() { + let layout = ControlRingLayout::SINGLE_LANE; + let memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); + let (local_stream, _peer) = UnixStream::pair().unwrap(); + let Err(error) = negotiated_local(local_stream) + .into_active(ControlRing::new(memory, layout).unwrap(), || {}) + else { + panic!("a single-lane layout was activated"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn concurrent_calls_use_separate_lanes_without_socket_frames() { + let (channel, _notifications, _shutdown, mut broker, mut peer) = activate_local(|| {}); peer.set_read_timeout(Some(Duration::from_millis(100))) .unwrap(); let channel = Arc::new(channel); @@ -787,17 +791,38 @@ mod control_ring_tests { let channel = Arc::clone(&channel); thread::spawn(move || channel.call(request(id))) }); - let first = read_request(&mut requests); - let second = read_request(&mut requests); + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + loop { + match demand.try_next_lane().unwrap() { + ControlRingDemandStatus::Lane(lane) => { + assert_eq!(lane, 1); + break; + } + ControlRingDemandStatus::Satisfied { wait_epoch } => { + demand.wait_for_demand(wait_epoch).unwrap(); + } + } + } + let first = read_request(&mut broker.lanes[0].request_consumer); + let second = read_request(&mut broker.lanes[1].request_consumer); + // The later lane answers first; neither call waits for the other. write_payload( - &mut responses, + &mut broker.lanes[1].response_producer, &encode_response(response(second.request_id)), ); - write_payload(&mut responses, &encode_response(response(first.request_id))); + write_payload( + &mut broker.lanes[0].response_producer, + &encode_response(response(first.request_id)), + ); for call in calls { assert!(call.join().unwrap().is_ok()); } + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); let mut byte = [0]; assert!(matches!( peer.read(&mut byte).unwrap_err().kind(), @@ -806,12 +831,38 @@ mod control_ring_tests { } #[test] - fn unknown_duplicate_and_malformed_responses_fail_closed() { + fn sequential_calls_reuse_the_first_lane() { + let (channel, _notifications, _shutdown, mut broker, _peer) = activate_local(|| {}); + let channel = Arc::new(channel); + for id in 1..=3 { + let caller = { + let channel = Arc::clone(&channel); + thread::spawn(move || channel.call(request(id))) + }; + let lane = &mut broker.lanes[0]; + let request = read_request(&mut lane.request_consumer); + assert_eq!(request.request_id, RequestId(id)); + write_payload( + &mut lane.response_producer, + &encode_response(response(request.request_id)), + ); + assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(id)); + } + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); + } + + #[test] + fn mismatched_and_malformed_responses_fail_closed() { for payload_kind in 0..3 { let failures = Arc::new(AtomicUsize::new(0)); let callback_failures = Arc::clone(&failures); let (failure_reported, wait_for_failure) = mpsc::channel(); - let (channel, _notifications, _shutdown, mut responses, mut requests, _peer) = + let (channel, _notifications, _shutdown, mut broker, _peer) = activate_local(move || { callback_failures.fetch_add(1, Ordering::SeqCst); failure_reported.send(()).unwrap(); @@ -821,22 +872,19 @@ mod control_ring_tests { let channel = Arc::clone(&channel); thread::spawn(move || channel.call(request(id))) }); - read_request(&mut requests); - read_request(&mut requests); + read_request(&mut broker.lanes[0].request_consumer); + let other = read_request(&mut broker.lanes[1].request_consumer); + let producer = &mut broker.lanes[0].response_producer; match payload_kind { - 0 => write_payload(&mut responses, &encode_response(response(RequestId(99)))), - 1 => { - let duplicate = encode_response(response(RequestId(1))); - write_payload(&mut responses, &duplicate); - write_payload(&mut responses, &duplicate); - } - _ => write_payload(&mut responses, &[u8::MAX]), + 0 => write_payload(producer, &encode_response(response(RequestId(99)))), + 1 => write_payload(producer, &encode_response(response(other.request_id))), + _ => write_payload(producer, &[u8::MAX]), } - let results = calls.map(|call| call.join().unwrap()); - let error_count = results.iter().filter(|result| result.is_err()).count(); - assert_eq!(error_count, if payload_kind == 1 { 1 } else { 2 }); + for call in calls { + assert!(call.join().unwrap().is_err()); + } wait_for_failure .recv_timeout(Duration::from_secs(1)) .expect("failure callback was not invoked"); @@ -845,25 +893,36 @@ mod control_ring_tests { } #[test] - fn local_socket_eof_and_shutdown_wake_pending_calls() { + fn local_socket_eof_and_shutdown_wake_waiting_calls() { for close_peer in [false, true] { - let (channel, _notifications, shutdown, _responses, mut requests, peer) = - activate_local(|| {}); - let caller = thread::spawn(move || channel.call(request(1))); - read_request(&mut requests); + let (channel, _notifications, shutdown, mut broker, peer) = activate_local(|| {}); + let channel = Arc::new(channel); + let call = |id| { + let channel = Arc::clone(&channel); + thread::spawn(move || channel.call(request(id))) + }; + let callers: Vec<_> = (0..LAYOUT.lane_count() as u64).map(call).collect(); + for lane in &mut broker.lanes { + read_request(&mut lane.request_consumer); + } + // Every lane is held, so this call waits to claim one. + let waiting = call(u64::MAX); + thread::sleep(Duration::from_millis(10)); if close_peer { drop(peer); } else { shutdown.shutdown().unwrap(); } - assert!(caller.join().unwrap().is_err()); + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + assert!(waiting.join().unwrap().is_err()); } } #[test] fn dropping_the_call_channel_ends_notifications_cleanly() { - let (channel, mut notifications, _shutdown, _responses, _requests, _peer) = - activate_local(|| {}); + let (channel, mut notifications, _shutdown, _broker, _peer) = activate_local(|| {}); let receiver = thread::spawn(move || notifications.recv_notification()); drop(channel); assert!(receiver.join().unwrap().unwrap().is_none()); @@ -973,37 +1032,9 @@ mod control_ring_tests { ); } - #[test] - fn completed_call_wins_over_later_failure_and_failure_wins_before_completion() { - let pending = PendingCalls::new(); - let completed = pending.register(RequestId(1)).unwrap(); - pending.complete(response(RequestId(1))).unwrap(); - pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); - assert_eq!( - completed.wait(|| unreachable!()).unwrap().request_id, - RequestId(1) - ); - - let pending = PendingCalls::new(); - let failed = pending.register(RequestId(2)).unwrap(); - pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); - assert!(pending.complete(response(RequestId(2))).is_err()); - assert_eq!( - failed.wait(|| unreachable!()).unwrap_err().kind(), - ErrorKind::ConnectionAborted - ); - } - #[test] fn failure_recording_waits_for_in_progress_publication() { - let pending = Arc::new(PendingCalls::new()); - let pending_call = pending.register(RequestId(1)).unwrap(); + let pending = Arc::new(LocalCallLanePool::new(vec![()])); let publication_state = Arc::new(AtomicUsize::new(0)); let (publication_started, wait_for_publication) = std::sync::mpsc::sync_channel(0); let (release_publication, publication_released) = std::sync::mpsc::sync_channel(0); @@ -1028,10 +1059,8 @@ mod control_ring_tests { let failure_state = Arc::clone(&publication_state); let failure = thread::spawn(move || { failure_started.send(()).unwrap(); - failure_pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); + failure_pending + .record_failure(Error::new(ErrorKind::ConnectionAborted, "test failure")); assert_eq!(failure_state.load(Ordering::Acquire), 2); failure_recorded.send(()).unwrap(); }); @@ -1047,24 +1076,9 @@ mod control_ring_tests { .recv_timeout(Duration::from_secs(1)) .unwrap(); failure.join().unwrap(); - assert_eq!( - pending_call.wait(|| unreachable!()).unwrap_err().kind(), - ErrorKind::ConnectionAborted - ); - } - - #[test] - fn duplicate_pending_registration_preserves_original() { - let pending = PendingCalls::new(); - let original = pending.register(RequestId(1)).unwrap(); - let Err(error) = pending.register(RequestId(1)).map_err(pending_calls_error) else { - panic!("duplicate registration unexpectedly succeeded"); - }; - assert_eq!(error.kind(), ErrorKind::InvalidData); - pending.complete(response(RequestId(1))).unwrap(); - assert_eq!( - original.wait(|| unreachable!()).unwrap().request_id, - RequestId(1) - ); + assert!(matches!( + pending.claim(|_| Ok(())), + Err(error) if error.kind() == ErrorKind::ConnectionAborted + )); } } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs b/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs index 43407e1409..77d7f65e98 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs @@ -17,11 +17,11 @@ mod host; mod local; pub use host::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, - validate_peer_process, + UnixControlRingHostCallLane, UnixControlRingHostCallLaneSource, + UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, + UnixStreamHostSetupChannel, validate_peer_process, }; pub use local::{ - MAX_PENDING_CALLS, UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, + UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, UnixStreamLocalSetupChannel, }; diff --git a/litebox_broker_transport_windows_userland/Cargo.toml b/litebox_broker_transport_windows_userland/Cargo.toml index 0cbe83e836..430ad4259c 100644 --- a/litebox_broker_transport_windows_userland/Cargo.toml +++ b/litebox_broker_transport_windows_userland/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } -litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0" } +litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0", features = ["std"] } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(windows)'.dependencies] diff --git a/litebox_broker_transport_windows_userland/src/control_ring.rs b/litebox_broker_transport_windows_userland/src/control_ring.rs index f8b57f0858..a6d58d950d 100644 --- a/litebox_broker_transport_windows_userland/src/control_ring.rs +++ b/litebox_broker_transport_windows_userland/src/control_ring.rs @@ -4,8 +4,8 @@ //! Active Windows-userland broker endpoints over a shared control ring. pub use crate::host::{ - WindowsControlRingHostNotificationChannel, WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, WindowsControlRingHostShutdown, + WindowsControlRingHostCallLane, WindowsControlRingHostCallLaneSource, + WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, }; pub use crate::local::{ WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, diff --git a/litebox_broker_transport_windows_userland/src/host.rs b/litebox_broker_transport_windows_userland/src/host.rs index 7218a32e58..0432432a15 100644 --- a/litebox_broker_transport_windows_userland/src/host.rs +++ b/litebox_broker_transport_windows_userland/src/host.rs @@ -17,12 +17,13 @@ use litebox_broker_protocol::wire::{ encode_notification, encode_response, }; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, PeerCredential, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, + ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + HostControlRingEndpoints, HostControlRingLane, HostLaneDemand, }; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::System::Pipes::GetNamedPipeClientProcessId; @@ -45,16 +46,20 @@ pub struct WindowsNamedPipeHostSetupChannel { negotiated: bool, } -/// Request-reading endpoint of an active host control-ring association. -pub struct WindowsControlRingHostRequestSource { - consumer: ControlRingConsumer, +type BrokerLane = HostControlRingLane; + +/// Source of the call lanes of an active host control-ring association, which +/// hands out each lane once the runner starts using it. +pub struct WindowsControlRingHostCallLaneSource { + /// Each lane, or `None` once handed out. + lanes: Vec>, + demand: HostLaneDemand, association: Arc, } -/// Shared response-writing endpoint of an active host control-ring association. -#[derive(Clone)] -pub struct WindowsControlRingHostResponseSink { - producer: Arc>>, +/// One call lane of an active host control-ring association. +pub struct WindowsControlRingHostCallLane { + lane: BrokerLane, association: Arc, } @@ -69,12 +74,13 @@ pub struct WindowsControlRingHostShutdown { association: Arc, } +/// State shared by every activated host endpoint of one association: terminal +/// status, the pipe liveness used for teardown, and the wake handles of every +/// ring wait. struct HostRingAssociation { status: Mutex, liveness: PipeLiveness, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } enum HostAssociationStatus { @@ -146,13 +152,14 @@ impl WindowsNamedPipeHostSetupChannel { ) } - /// Activates host request, response, and notification control-ring endpoints. + /// Activates host call-lane, notification, and shutdown control-ring endpoints. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( self, ring: ControlRing, ) -> IoResult<( - WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, + WindowsControlRingHostCallLaneSource, WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, )> { @@ -214,8 +221,7 @@ fn activate_host( setup_deadline: Option, ring: ControlRing, ) -> IoResult<( - WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, + WindowsControlRingHostCallLaneSource, WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, )> { @@ -224,43 +230,47 @@ fn activate_host( "broker host setup channel activated before negotiation completed", )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_host(); + let wakes = endpoints.wake_handles(); + let HostControlRingEndpoints { + lanes, + lane_demand: Some(demand), + notification_producer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let ready = read_frame(file_handle(&stream), setup_deadline)?.ok_or_else(|| { Error::new( ErrorKind::UnexpectedEof, "runner closed before control-ring setup acknowledgement", ) })?; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "runner sent an invalid control-ring setup acknowledgement", )); } - write_frame(file_handle(&stream), CONTROL_RING_READY, setup_deadline)?; + write_frame(file_handle(&stream), ready_token, setup_deadline)?; - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer, - } = ring.into_broker(); let stream = Arc::new(stream); let association = Arc::new(HostRingAssociation { liveness: PipeLiveness::new(Arc::clone(&stream), true)?, status: Mutex::new(HostAssociationStatus::Live), - request_wake: request_consumer.wake_handle(), - response_wake: response_producer.wake_handle(), - notification_wake: notification_producer.wake_handle(), + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() .name("litebox-runner-liveness".to_owned()) .spawn(move || monitor_host_pipe(&stream, &monitor_association))?; Ok(( - WindowsControlRingHostRequestSource { - consumer: request_consumer, - association: Arc::clone(&association), - }, - WindowsControlRingHostResponseSink { - producer: Arc::new(Mutex::new(response_producer)), + WindowsControlRingHostCallLaneSource { + lanes: lanes.into_iter().map(Some).collect(), + demand, association: Arc::clone(&association), }, WindowsControlRingHostNotificationChannel { @@ -298,23 +308,78 @@ impl Drop for WindowsControlRingHostShutdown { } } -impl WindowsControlRingHostRequestSource { - /// Receives one active broker request. +impl WindowsControlRingHostCallLaneSource { + /// Blocks until the runner starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` once the runner closed the association. + /// + /// # Panics + /// + /// Panics if the lane demand hands out a lane more than once. + pub fn next_lane(&mut self) -> IoResult> { + loop { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + match self.demand.try_next_lane() { + Ok(ControlRingDemandStatus::Lane(index)) => { + let lane = self.lanes[index] + .take() + .expect("lane demand hands out each lane once"); + return Ok(Some(WindowsControlRingHostCallLane { + lane, + association: Arc::clone(&self.association), + })); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) => { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + if let Err(error) = self.demand.wait_for_demand(wait_epoch) { + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + Err(error) => { + let error = ring_error(error); + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + } + } +} + +impl HostCallLaneSource for WindowsControlRingHostCallLaneSource { + type Error = Error; + type Lane = WindowsControlRingHostCallLane; + + fn next_lane(&mut self) -> IoResult> { + Self::next_lane(self) + } +} + +impl WindowsControlRingHostCallLane { + /// Receives this lane's next broker request. pub fn recv_request(&mut self) -> IoResult> { + let consumer = &mut self.lane.request_consumer; loop { if let Some(error) = self.association.current_failure() { return Err(error); } - match self.consumer.try_read(decode_request) { + match consumer.try_read(decode_request) { Ok(ControlRingReadStatus::Message(request)) => { - self.association.acknowledge_request(&mut self.consumer)?; + self.association.acknowledge_request(consumer)?; return Ok(HostReceive::Message(request)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(terminal) = self.association.request_terminal_result() { - return terminal; + if let Some(ended) = self.association.ended() { + return ended.map(|()| HostReceive::PeerClosed); } - if let Err(error) = self.consumer.wait_for_message(wait_epoch) { + if let Err(error) = consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); let _ = self.association.fail(error); return result; @@ -338,26 +403,13 @@ impl WindowsControlRingHostRequestSource { } } } -} - -impl HostRequestSource for WindowsControlRingHostRequestSource { - type Error = Error; - - fn recv_request(&mut self) -> IoResult> { - Self::recv_request(self) - } -} -impl WindowsControlRingHostResponseSink { - /// Serializes and sends one complete active broker response. - pub fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + /// Serializes and sends the response to this lane's last request. + pub fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { let frame = encode_response(response.clone()); - let mut producer = self - .producer - .lock() - .map_err(|_| Error::other("broker response writer mutex poisoned"))?; + let producer = &mut self.lane.response_producer; loop { - match self.association.try_publish(&mut producer, &frame)? { + match self.association.try_publish(producer, &frame)? { ControlRingWriteStatus::Written => return Ok(()), ControlRingWriteStatus::Full { wait_epoch } => { if let Err(error) = producer.wait_for_capacity(wait_epoch) { @@ -371,10 +423,14 @@ impl WindowsControlRingHostResponseSink { } } -impl HostResponseSink for WindowsControlRingHostResponseSink { +impl HostCallLane for WindowsControlRingHostCallLane { type Error = Error; - fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + fn recv_request(&mut self) -> IoResult> { + Self::recv_request(self) + } + + fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { Self::send_response(self, response) } } @@ -453,20 +509,22 @@ impl HostRingAssociation { } fn interrupt_waits(&self) -> IoResult<()> { - self.request_wake - .interrupt_wait() - .and(self.response_wake.interrupt_wait()) - .and(self.notification_wake.interrupt_wait()) + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) } - fn request_terminal_result(&self) -> Option>> { + /// Returns how the association ended, if it has: `Ok` if the runner + /// closed it. + fn ended(&self) -> Option> { match &*self .status .lock() .expect("broker host association mutex poisoned") { HostAssociationStatus::Live => None, - HostAssociationStatus::PeerClosed => Some(Ok(HostReceive::PeerClosed)), + HostAssociationStatus::PeerClosed => Some(Ok(())), HostAssociationStatus::Failed(error) => Some(Err(copy_io_error(error))), } } diff --git a/litebox_broker_transport_windows_userland/src/lib.rs b/litebox_broker_transport_windows_userland/src/lib.rs index 9422859139..18f14354ce 100644 --- a/litebox_broker_transport_windows_userland/src/lib.rs +++ b/litebox_broker_transport_windows_userland/src/lib.rs @@ -13,7 +13,6 @@ pub mod existing_runner; mod host; mod local; pub mod named_pipe; -mod pending_calls; pub mod process_image; mod setup; pub mod shared_memory; diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 8d64923454..f4b9820437 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -7,8 +7,8 @@ use std::ffi::OsStr; use std::fs::OpenOptions; use std::io::{Error, ErrorKind, Result as IoResult}; use std::os::windows::fs::OpenOptionsExt; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -20,19 +20,20 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; +use litebox_broker_transport::call_lane_pool::{LaneClaim, LocalCallLanePool}; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; use crate::control_ring::PipeLiveness; use crate::named_pipe::{TRANSFER_FRAME_TAG, WindowsNamedPipeStream}; -use crate::pending_calls::{PendingCalls, pending_calls_error}; use crate::process_image::WindowsReceivedProcessImage; use crate::setup::{ copy_io_error, invalid_data, read_frame, read_pipe_until_cancelled, ring_error, wire_error, @@ -60,18 +61,21 @@ pub struct WindowsControlRingLocalNotificationChannel { association: Arc, } +type LocalLane = LocalControlRingLane; + +/// State shared by every activated local endpoint of one association: the +/// call lanes and their demand, the pipe liveness used for teardown, and the +/// wake handles of every ring wait. struct LocalRingAssociation { - request_producer: Mutex>, - /// Read by whichever caller holds the pending-call reader role. - response_consumer: Mutex>, - pending_calls: Arc, + /// The call lanes, which also record the failure that ended the + /// association. + lanes: LocalCallLanePool, + lane_demand: LocalLaneDemand, /// Whether the call channel was dropped, which ends the association as a /// local close rather than a failure. closed: AtomicBool, liveness: PipeLiveness, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } impl WindowsNamedPipeLocalSetupChannel { @@ -123,8 +127,12 @@ impl WindowsNamedPipeLocalSetupChannel { unsafe { WindowsSharedMemory::from_transferred(transfer) } } - /// Receives and maps the shared control ring duplicated into this process. - pub fn receive_control_ring(&mut self) -> IoResult { + /// Receives the shared control ring the broker duplicated into this process and maps it with + /// `layout`. + pub fn receive_control_ring( + &mut self, + layout: ControlRingLayout, + ) -> IoResult { let frame = read_frame(file_handle(&self.stream), self.setup_deadline)?.ok_or_else(|| { Error::new( @@ -135,7 +143,7 @@ impl WindowsNamedPipeLocalSetupChannel { let transfer = decode_transfer(&frame)?; // SAFETY: The authenticated broker duplicated these handles into this process and encoded // their target-process values in the setup frame. - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) } + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, layout) } } /// Receives and maps the optional process image the broker sent with @@ -155,6 +163,8 @@ impl WindowsNamedPipeLocalSetupChannel { } /// Activates calls and notifications over the transferred shared control ring. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( self, ring: ControlRing, @@ -200,35 +210,40 @@ fn activate_local( "broker local setup channel activated before negotiation completed", )); } - write_frame(file_handle(&stream), CONTROL_RING_READY, setup_deadline)?; + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_local(); + let wakes = endpoints.wake_handles(); + let LocalControlRingEndpoints { + lanes, + lane_demand: Some(lane_demand), + notification_consumer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; + write_frame(file_handle(&stream), ready_token, setup_deadline)?; let ready = read_frame(file_handle(&stream), setup_deadline)?.ok_or_else(|| { Error::new( ErrorKind::UnexpectedEof, "broker closed before control-ring setup acknowledgement", ) })?; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "broker sent an invalid control-ring setup acknowledgement", )); } - let litebox_broker_transport::control_ring::LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer, - } = ring.into_local(); - let pending_calls = Arc::new(PendingCalls::new()); let stream = Arc::new(stream); let association = Arc::new(LocalRingAssociation { - liveness: PipeLiveness::new(Arc::clone(&stream), false)?, - request_wake: request_producer.wake_handle(), - request_producer: Mutex::new(request_producer), - response_wake: response_consumer.wake_handle(), - response_consumer: Mutex::new(response_consumer), - notification_wake: notification_consumer.wake_handle(), - pending_calls, + lanes: LocalCallLanePool::new(lanes), + lane_demand, closed: AtomicBool::new(false), + liveness: PipeLiveness::new(Arc::clone(&stream), false)?, + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() @@ -250,41 +265,25 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; - let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; - let frame = encode_request(request); - let write_result = { - let mut producer = association - .request_producer - .lock() - .map_err(|_| Error::other("broker request writer mutex poisoned"))?; - loop { - let status = association - .pending_calls - .run_if_live(|| producer.try_write(&frame).map_err(ring_error)) - .map_err(pending_calls_error); - match status { - Ok(ControlRingWriteStatus::Written) => { - break producer.wake_consumer(); - } - Ok(ControlRingWriteStatus::Full { wait_epoch }) => { - if let Err(error) = producer.wait_for_capacity(wait_epoch) { - break Err(error); - } - } - Err(error) => break Err(error), - } + let mut lane = match association + .lanes + .claim(|lanes| association.demand_lanes(lanes)) + { + Ok(lane) => lane, + Err(error) => { + let result = Err(copy_io_error(&error)); + let _ = association.fail(error); + return result; } }; - if let Err(error) = write_result { - let _ = association.fail(error); + let result = association.call_on_lane(&mut lane, request); + // A failed call can leave its lane mid-exchange: fail the association + // before freeing the lane, so no other call uses it. + if let Err(error) = &result { + let _ = association.fail(copy_io_error(error)); } - pending_call - .wait(|| association.read_response()) - .map_err(|error| copy_io_error(&error)) + drop(lane); + result } } @@ -303,8 +302,8 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { - if let Some(error) = self.association.pending_calls.current_failure() { - return self.association.end_notifications(&error); + if let Some(error) = self.association.lanes.current_failure() { + return self.association.end_notifications(error); } match self.consumer.try_read(decode_notification) { Ok(ControlRingReadStatus::Message(notification)) => { @@ -312,13 +311,13 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { .association .acknowledge_notification(&mut self.consumer) { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(error) = self.association.pending_calls.current_failure() { - return self.association.end_notifications(&error); + if let Some(error) = self.association.lanes.current_failure() { + return self.association.end_notifications(error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); @@ -346,11 +345,11 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { impl LocalRingAssociation { /// Ends notification receipt once the association has ended with /// `failure`: cleanly if the local side closed it. - fn end_notifications(&self, failure: &Error) -> IoResult> { + fn end_notifications(&self, failure: Error) -> IoResult> { if self.closed.load(Ordering::Acquire) { Ok(None) } else { - Err(copy_io_error(failure)) + Err(failure) } } @@ -358,15 +357,12 @@ impl LocalRingAssociation { &self, consumer: &mut ControlRingConsumer, ) -> IoResult<()> { - let result = self - .pending_calls - .run_if_live(|| { - consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - }) - .map_err(pending_calls_error); + let result = self.lanes.run_if_live(|| { + consumer + .publish_head() + .map_err(ring_error) + .and_then(|()| consumer.wake_producer()) + }); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -375,57 +371,68 @@ impl LocalRingAssociation { Ok(()) } - /// Reads one response for the caller holding the pending-call reader - /// role, blocking until one arrives or the association fails. - fn read_response(&self) { - let Ok(mut consumer) = self.response_consumer.lock() else { - let _ = self.fail(Error::other("broker response reader mutex poisoned")); - return; - }; + /// Raises the lane demand to `lanes` and wakes the broker to serve them. + fn demand_lanes(&self, lanes: usize) -> IoResult<()> { + self.lane_demand.request(lanes).map_err(ring_error)?; + self.lane_demand.wake_host() + } + + /// Writes `request` to `lane` and reads the broker's response from it, + /// releasing the lane once the call succeeds. + fn call_on_lane( + &self, + lane: &mut LaneClaim<'_, LocalLane>, + request: BrokerRequest, + ) -> IoResult { + let request_id = request.request_id; + let request_frame = encode_request(request); + let producer = &mut lane.request_producer; loop { - match consumer.try_read(decode_response) { - Ok(ControlRingReadStatus::Message(response)) => { - if let Err(error) = consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - .and_then(|()| { - self.pending_calls - .complete(response) - .map_err(pending_calls_error) - }) - { - let _ = self.fail(error); - } - return; + let write_status = self + .lanes + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error))?; + match write_status { + ControlRingWriteStatus::Written => break, + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch)?; } + } + } + producer.wake_consumer()?; + + let consumer = &mut lane.response_consumer; + let response = loop { + match consumer.try_read(decode_response) { + Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if self.pending_calls.current_failure().is_some() { - return; - } - if let Err(error) = consumer.wait_for_message(wait_epoch) { - let _ = self.fail(error); - return; + if let Some(error) = self.lanes.current_failure() { + return Err(error); } + consumer.wait_for_message(wait_epoch)?; } - Err(ControlRingReadError::Ring(error)) => { - let _ = self.fail(ring_error(error)); - return; - } - Err(ControlRingReadError::Decode(error)) => { - let _ = self.fail(wire_error(error)); - return; - } + Err(ControlRingReadError::Ring(error)) => return Err(ring_error(error)), + Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } + }; + if response.request_id != request_id { + return Err(invalid_data("broker answered a different request")); } + // Accepting the response with the lane's release excludes failure + // recording, so no call succeeds once the association has failed. + lane.release_if_live(|lane| { + let consumer = &mut lane.response_consumer; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer() + })?; + Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - self.pending_calls.record_failure(Arc::new(error)); - self.request_wake - .interrupt_wait() - .and(self.response_wake.interrupt_wait()) - .and(self.notification_wake.interrupt_wait()) + self.lanes.record_failure(error); + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) .and(self.liveness.shutdown()) } } diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index d796e31b83..493c3fef98 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -139,25 +139,43 @@ fn wide_string(value: &OsStr) -> IoResult> { mod tests { use super::*; use std::sync::Arc; + use std::thread; use std::time::{Duration, Instant}; - use crate::setup::{OwnedEvent, read_pipe_until_cancelled, write_frame}; + use crate::control_ring::{ + WindowsControlRingHostCallLaneSource, WindowsControlRingHostShutdown, + WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, + }; + use crate::setup::{OwnedEvent, read_frame, read_pipe_until_cancelled, write_frame}; use crate::shared_memory::WindowsSharedMemory; use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, ReadinessNotification, }; use litebox_broker_protocol::readiness::ReadinessFlags; + use litebox_broker_protocol::wire::{ + decode_request, decode_response, encode_handshake_response, encode_request, encode_response, + }; use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, ObjectHandle, RequestId}; use litebox_broker_transport::channel::{ HostNotificationChannel, HostReceive, HostSetupChannel, LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; - use litebox_broker_transport::control_ring::ControlRing; + use litebox_broker_transport::control_ring::{ + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingProducer, + ControlRingReadStatus, ControlRingWriteStatus, HostControlRingEndpoints, + LocalControlRingEndpoints, + }; use litebox_broker_transport::shared_memory::SharedMemory as _; use windows_sys::Win32::System::Threading::GetCurrentProcessId; + type Producer = ControlRingProducer; + type Consumer = ControlRingConsumer; + type Local = LocalControlRingEndpoints; + type Broker = HostControlRingEndpoints; + const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; fn pipe_name(suffix: &str) -> std::ffi::OsString { format!(r"\\.\pipe\litebox-broker-test-{}-{suffix}", unsafe { @@ -170,11 +188,11 @@ mod tests { ControlRing, ControlRing, ) { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let host_memory = local_memory.try_clone_view().unwrap(); ( - ControlRing::new(local_memory).unwrap(), - ControlRing::new(host_memory).unwrap(), + ControlRing::new(local_memory, LAYOUT).unwrap(), + ControlRing::new(host_memory, LAYOUT).unwrap(), ) } @@ -201,6 +219,178 @@ mod tests { } } + fn connect_local(name: &OsStr) -> WindowsNamedPipeLocalSetupChannel { + WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline( + name, + Instant::now() + Duration::from_secs(5), + ) + .unwrap() + } + + fn negotiated() -> BrokerHandshakeResponse { + BrokerHandshakeResponse::Negotiated { + broker_protocol_version: BROKER_PROTOCOL_VERSION, + process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, + } + } + + fn send_handshake_request(local: &mut WindowsNamedPipeLocalSetupChannel) { + local + .send_handshake_request(&BrokerHandshakeRequest { + protocol_version: BROKER_PROTOCOL_VERSION, + }) + .unwrap(); + } + + /// Negotiates one real association over a named pipe, so both halves reach + /// their negotiated state through the same handshake production uses. + fn negotiated_pair( + suffix: &str, + ) -> ( + WindowsNamedPipeLocalSetupChannel, + WindowsNamedPipeHostSetupChannel, + ) { + let name = pipe_name(suffix); + let listener = WindowsNamedPipeListener::bind(&name).unwrap(); + let mut local = connect_local(&name); + let mut host = accept_host_guaranteed(listener); + send_handshake_request(&mut local); + assert!(matches!( + host.recv_handshake_request().unwrap(), + HostReceive::Message(BrokerHandshakeRequest { .. }) + )); + host.send_handshake_response(&negotiated()).unwrap(); + assert!(matches!( + local.recv_handshake_response().unwrap(), + Some(BrokerHandshakeResponse::Negotiated { .. }) + )); + (local, host) + } + + /// Activates a host association against raw local ring endpoints and the + /// local half of its pipe. + fn activate_host( + suffix: &str, + ) -> ( + WindowsControlRingHostCallLaneSource, + WindowsControlRingHostShutdown, + Local, + WindowsNamedPipeLocalSetupChannel, + ) { + let (local, host) = negotiated_pair(suffix); + let (local_ring, host_ring) = control_rings(); + write_frame(local.stream.handle(), LAYOUT.ready_token(), None).unwrap(); + let (lanes, _notifications, shutdown) = host.into_active(host_ring).unwrap(); + assert_eq!( + read_frame(local.stream.handle(), None).unwrap().unwrap(), + LAYOUT.ready_token() + ); + (lanes, shutdown, local_ring.into_local(), local) + } + + /// Activates a local association against raw broker ring endpoints and the + /// host half of its pipe. + fn activate_local( + suffix: &str, + ) -> ( + WindowsControlRingLocalCallChannel, + WindowsControlRingLocalNotificationChannel, + Broker, + WindowsNamedPipeStream, + ) { + let name = pipe_name(suffix); + let listener = WindowsNamedPipeListener::bind(&name).unwrap(); + let mut local = connect_local(&name); + let host = accept_stream_guaranteed(listener); + send_handshake_request(&mut local); + read_frame(host.handle(), None).unwrap().unwrap(); + write_frame( + host.handle(), + &encode_handshake_response(negotiated()), + None, + ) + .unwrap(); + local.recv_handshake_response().unwrap().unwrap(); + let (local_ring, broker_ring) = control_rings(); + let (calls, notifications) = thread::scope(|scope| { + scope.spawn(|| { + assert_eq!( + read_frame(host.handle(), None).unwrap().unwrap(), + LAYOUT.ready_token() + ); + write_frame(host.handle(), LAYOUT.ready_token(), None).unwrap(); + }); + local.into_active(local_ring).unwrap() + }); + (calls, notifications, broker_ring.into_host(), host) + } + + fn demand_lanes(local: &Local, lanes: usize) { + let demand = local.lane_demand.as_ref().unwrap(); + demand.request(lanes).unwrap(); + demand.wake_host().unwrap(); + } + + fn read_request(consumer: &mut Consumer) -> BrokerRequest { + loop { + match consumer.try_read(decode_request).unwrap() { + ControlRingReadStatus::Message(request) => { + consumer.publish_head().unwrap(); + consumer.wake_producer().unwrap(); + return request; + } + ControlRingReadStatus::Empty { wait_epoch } => { + consumer.wait_for_message(wait_epoch).unwrap(); + } + } + } + } + + fn read_response(consumer: &mut Consumer) -> BrokerResponse { + loop { + match consumer.try_read(decode_response).unwrap() { + ControlRingReadStatus::Message(response) => { + consumer.publish_head().unwrap(); + consumer.wake_producer().unwrap(); + return response; + } + ControlRingReadStatus::Empty { wait_epoch } => { + consumer.wait_for_message(wait_epoch).unwrap(); + } + } + } + } + + fn write_payload(producer: &mut Producer, payload: &[u8]) { + loop { + match producer.try_write(payload).unwrap() { + ControlRingWriteStatus::Written => { + producer.wake_consumer().unwrap(); + return; + } + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch).unwrap(); + } + } + } + } + + fn request(id: u64) -> BrokerRequest { + BrokerRequest { + request_id: RequestId(id), + operation: BrokerOperation::CloseObject(ObjectHandle(id)), + } + } + + fn response(id: RequestId) -> BrokerResponse { + BrokerResponse { + request_id: id, + result: BrokerResult::ObjectClosed, + } + } + #[test] fn accept_without_client_would_block() { let mut listener = @@ -343,21 +533,20 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, responses, mut notifications, _shutdown) = - setup.into_active(host_ring).unwrap(); - let HostReceive::Message(request) = requests.recv_request().unwrap() else { + let (mut lanes, mut notifications, _shutdown) = setup.into_active(host_ring).unwrap(); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let HostReceive::Message(request) = lane.recv_request().unwrap() else { panic!("expected active request"); }; assert_eq!( request.operation, BrokerOperation::CloseObject(ObjectHandle(7)) ); - responses - .send_response(&BrokerResponse { - request_id: request.request_id, - result: BrokerResult::ObjectClosed, - }) - .unwrap(); + lane.send_response(&BrokerResponse { + request_id: request.request_id, + result: BrokerResult::ObjectClosed, + }) + .unwrap(); notifications .send_notification(&BrokerNotification::Readiness(ReadinessNotification { handle: ObjectHandle(7), @@ -402,7 +591,7 @@ mod tests { } #[test] - fn concurrent_calls_accept_responses_in_reverse_order() { + fn concurrent_calls_on_separate_lanes_complete_in_reverse_order() { let control_name = pipe_name("multiplex-control"); let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); let (local_ring, host_ring) = control_rings(); @@ -421,21 +610,20 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, responses, _notifications, _shutdown) = - setup.into_active(host_ring).unwrap(); - let HostReceive::Message(first) = requests.recv_request().unwrap() else { - panic!("expected first active request"); - }; - let HostReceive::Message(second) = requests.recv_request().unwrap() else { - panic!("expected second active request"); - }; - for request in [second, first] { - responses - .send_response(&BrokerResponse { - request_id: request.request_id, - result: BrokerResult::ObjectClosed, - }) - .unwrap(); + let (mut lanes, _notifications, _shutdown) = setup.into_active(host_ring).unwrap(); + // The second lane is handed out only once a concurrent call demands it. + let mut lanes = [ + lanes.next_lane().unwrap().unwrap(), + lanes.next_lane().unwrap().unwrap(), + ]; + let requests = lanes.each_mut().map(|lane| { + let HostReceive::Message(request) = lane.recv_request().unwrap() else { + panic!("expected active request"); + }; + request + }); + for (lane, request) in lanes.iter_mut().zip(requests).rev() { + lane.send_response(&response(request.request_id)).unwrap(); } wait_for_responses.recv().unwrap(); }); @@ -452,32 +640,19 @@ mod tests { setup.recv_handshake_response().unwrap().unwrap(); let (calls, _notifications) = setup.into_active(local_ring).unwrap(); let calls = Arc::new(calls); - let callers = [RequestId(11), RequestId(13)].map(|request_id| { + let callers = [11, 13].map(|id| { let calls = Arc::clone(&calls); - std::thread::spawn(move || { - calls - .call(BrokerRequest { - request_id, - operation: BrokerOperation::CloseObject(ObjectHandle(request_id.0)), - }) - .unwrap() - }) + std::thread::spawn(move || calls.call(request(id)).unwrap()) }); - for (caller, request_id) in callers.into_iter().zip([RequestId(11), RequestId(13)]) { - assert_eq!( - caller.join().unwrap(), - BrokerResponse { - request_id, - result: BrokerResult::ObjectClosed, - } - ); + for (caller, id) in callers.into_iter().zip([11, 13]) { + assert_eq!(caller.join().unwrap(), response(RequestId(id))); } responses_received.send(()).unwrap(); host.join().unwrap(); } #[test] - fn host_shutdown_interrupts_blocked_request_receive() { + fn host_shutdown_interrupts_blocked_request_and_lane_waits() { let control_name = pipe_name("shutdown-control"); let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); let (local_ring, host_ring) = control_rings(); @@ -495,16 +670,24 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, _responses, _notifications, shutdown) = - setup.into_active(host_ring).unwrap(); + let (mut lanes, _notifications, shutdown) = setup.into_active(host_ring).unwrap(); + let mut lane = lanes.next_lane().unwrap().unwrap(); let (reader_started, wait_for_reader) = std::sync::mpsc::sync_channel(0); let reader = std::thread::spawn(move || { reader_started.send(()).unwrap(); - requests.recv_request() + lane.recv_request() }); + let next = std::thread::spawn(move || lanes.next_lane()); wait_for_reader.recv().unwrap(); shutdown.shutdown().unwrap(); - assert!(reader.join().unwrap().is_err()); + assert_eq!( + reader.join().unwrap().unwrap_err().kind(), + ErrorKind::ConnectionAborted + ); + assert_eq!( + next.join().unwrap().err().map(|error| error.kind()), + Some(ErrorKind::ConnectionAborted) + ); }); let deadline = Instant::now() + Duration::from_secs(5); @@ -520,4 +703,126 @@ mod tests { let (_calls, _notifications) = setup.into_active(local_ring).unwrap(); host.join().unwrap(); } + + #[test] + fn single_lane_layouts_are_rejected() { + let layout = ControlRingLayout::SINGLE_LANE; + let (local, host) = negotiated_pair("single-lane"); + let local_memory = WindowsSharedMemory::create_control_ring(layout).unwrap(); + let host_memory = local_memory.try_clone_view().unwrap(); + + let Err(error) = host.into_active(ControlRing::new(host_memory, layout).unwrap()) else { + panic!("the host activated a single-lane layout"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + let Err(error) = local.into_active(ControlRing::new(local_memory, layout).unwrap()) else { + panic!("the local side activated a single-lane layout"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn host_hands_out_lanes_on_demand_and_each_lane_serves_its_own_calls() { + let (mut lanes, _shutdown, mut local, peer) = activate_host("lanes-on-demand"); + let mut first = lanes.next_lane().unwrap().unwrap(); + let next = thread::spawn(move || lanes.next_lane().map(|lane| (lane, lanes))); + thread::sleep(Duration::from_millis(20)); + assert!(!next.is_finished()); + demand_lanes(&local, 2); + let (second, mut lanes) = next.join().unwrap().unwrap(); + let mut second = second.unwrap(); + + for (index, lane) in [&mut first, &mut second].into_iter().enumerate() { + let id = u64::try_from(index).unwrap(); + write_payload( + &mut local.lanes[index].request_producer, + &encode_request(request(id)), + ); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { + panic!("expected ring request"); + }; + assert_eq!(received.request_id, RequestId(id)); + lane.send_response(&response(received.request_id)).unwrap(); + } + for (index, lane) in local.lanes.iter_mut().take(2).enumerate() { + assert_eq!( + read_response(&mut lane.response_consumer).request_id, + RequestId(u64::try_from(index).unwrap()) + ); + } + + // Each remaining lane is handed out once; then the source waits until + // the peer closes. + demand_lanes(&local, LAYOUT.lane_count()); + for _ in 2..LAYOUT.lane_count() { + assert!(lanes.next_lane().unwrap().is_some()); + } + let last = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + assert!(!last.is_finished()); + drop(peer); + assert!(last.join().unwrap().unwrap().is_none()); + } + + #[test] + fn host_clean_close_wakes_request_and_lane_waits_as_peer_closed() { + let (mut lanes, _shutdown, _local, peer) = activate_host("clean-close"); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + drop(peer); + assert_eq!(receiver.join().unwrap().unwrap(), HostReceive::PeerClosed); + assert!(next.join().unwrap().unwrap().is_none()); + } + + #[test] + fn mismatched_and_malformed_responses_fail_closed() { + for payload_kind in 0..3 { + let (calls, _notifications, mut broker, _peer) = + activate_local(&format!("local-bad-response-{payload_kind}")); + let calls = Arc::new(calls); + let callers = [1, 2].map(|id| { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }); + read_request(&mut broker.lanes[0].request_consumer); + let other = read_request(&mut broker.lanes[1].request_consumer); + + let producer = &mut broker.lanes[0].response_producer; + match payload_kind { + 0 => write_payload(producer, &encode_response(response(RequestId(99)))), + 1 => write_payload(producer, &encode_response(response(other.request_id))), + _ => write_payload(producer, &[u8::MAX]), + } + + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + // The association is failed, so it refuses further calls. + assert!(calls.call(request(3)).is_err()); + } + } + + #[test] + fn broker_pipe_close_wakes_waiting_calls() { + let (calls, _notifications, mut broker, peer) = activate_local("local-pipe-close"); + let calls = Arc::new(calls); + let call = |id| { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }; + let callers: Vec<_> = (0..LAYOUT.lane_count() as u64).map(call).collect(); + for lane in &mut broker.lanes { + read_request(&mut lane.request_consumer); + } + // Every lane is held, so this call waits to claim one. + let waiting = call(u64::MAX); + thread::sleep(Duration::from_millis(10)); + drop(peer); + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + assert!(waiting.join().unwrap().is_err()); + } } diff --git a/litebox_broker_transport_windows_userland/src/pending_calls.rs b/litebox_broker_transport_windows_userland/src/pending_calls.rs deleted file mode 100644 index c430dc5820..0000000000 --- a/litebox_broker_transport_windows_userland/src/pending_calls.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::pending_calls::{ - PendingCalls as GenericPendingCalls, PendingCallsCondvar, PendingCallsError, PendingCallsMutex, - PendingCallsSync, -}; - -use crate::setup::{copy_io_error, invalid_data}; - -pub(crate) type PendingCalls = GenericPendingCalls; - -pub(crate) struct StdPendingCallsSync; - -pub(crate) struct StdPendingCallsMutex(Mutex); - -pub(crate) struct StdPendingCallsCondvar(Condvar, PhantomData); - -impl PendingCallsMutex for StdPendingCallsMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker pending mutex poisoned") - } -} - -impl PendingCallsCondvar for StdPendingCallsCondvar { - type Mutex = StdPendingCallsMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0.wait(guard).expect("broker pending mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl PendingCallsSync for StdPendingCallsSync { - type Mutex = StdPendingCallsMutex; - type Condvar = StdPendingCallsCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdPendingCallsMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdPendingCallsCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn pending_calls_error(error: PendingCallsError) -> Error { - match error { - PendingCallsError::AssociationFailed(error) => copy_io_error(&error), - PendingCallsError::Operation(error) => error, - PendingCallsError::DuplicateRequestId => invalid_data("duplicate broker request ID"), - PendingCallsError::UnknownResponseId => { - invalid_data("broker returned an unknown response ID") - } - } -} diff --git a/litebox_broker_transport_windows_userland/src/shared_memory.rs b/litebox_broker_transport_windows_userland/src/shared_memory.rs index e1a46f44bb..4e196db31e 100644 --- a/litebox_broker_transport_windows_userland/src/shared_memory.rs +++ b/litebox_broker_transport_windows_userland/src/shared_memory.rs @@ -8,8 +8,7 @@ use std::ptr::NonNull; use std::sync::Arc; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingDirection, - MemoryAccessPolicy, WaitableSharedMemory, + CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingLayout, MemoryAccessPolicy, WaitableSharedMemory, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_platform::buffer::RawBytes; @@ -29,15 +28,6 @@ use windows_sys::Win32::System::Threading::{ const CONTROL_RING_WAIT_RECHECK_TIMEOUT_MS: u32 = 100; -const CONTROL_RING_WAKE_OFFSETS: [usize; 6] = [ - ControlRingDirection::Requests.producer_epoch_offset(), - ControlRingDirection::Requests.consumer_epoch_offset(), - ControlRingDirection::Responses.producer_epoch_offset(), - ControlRingDirection::Responses.consumer_epoch_offset(), - ControlRingDirection::Notifications.producer_epoch_offset(), - ControlRingDirection::Notifications.consumer_epoch_offset(), -]; - pub(crate) struct OwnedHandle(pub(crate) HANDLE); impl Drop for OwnedHandle { @@ -60,7 +50,9 @@ pub(crate) struct TransferredSharedMemory { /// One mapped view of a page-file-backed Windows section. pub struct WindowsSharedMemory { mapping: Arc, - wake_handles: Option>, + /// One event per waitable control-ring word, in + /// [`ControlRingLayout::waitable_word_index`] order. + wake_handles: Option>, view: NonNull, length: usize, policy: MemoryAccessPolicy, @@ -80,9 +72,12 @@ impl WindowsSharedMemory { Self::create_with_policy(length, MemoryAccessPolicy::Bytes) } - /// Creates and maps shared memory for one control ring. - pub fn create_control_ring() -> IoResult { - Self::create_with_policy(CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy::ControlRing) + /// Creates and maps shared memory for one control ring with `layout`. + pub fn create_control_ring(layout: ControlRingLayout) -> IoResult { + Self::create_with_policy( + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), + ) } fn create_with_policy(length: usize, policy: MemoryAccessPolicy) -> IoResult { @@ -118,7 +113,9 @@ impl WindowsSharedMemory { } let wake_handles = match policy { MemoryAccessPolicy::Bytes => None, - MemoryAccessPolicy::ControlRing => Some(Arc::new(create_wake_handles()?)), + MemoryAccessPolicy::ControlRing(layout) => { + Some(create_wake_handles(layout.waitable_word_count())?) + } }; Self::map(Arc::new(OwnedHandle(mapping)), wake_handles, length, policy) } @@ -135,7 +132,7 @@ impl WindowsSharedMemory { fn map( mapping: Arc, - wake_handles: Option>, + wake_handles: Option>, length: usize, policy: MemoryAccessPolicy, ) -> IoResult { @@ -170,7 +167,7 @@ impl WindowsSharedMemory { .ok_or_else(|| Error::other("NtMapViewOfSection returned a null view"))?; let spin_limit = match policy { MemoryAccessPolicy::Bytes => 0, - MemoryAccessPolicy::ControlRing => parallel_spin_limit(), + MemoryAccessPolicy::ControlRing(_) => parallel_spin_limit(), }; Ok(Self { mapping, @@ -223,24 +220,29 @@ impl WindowsSharedMemory { unsafe { Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::Bytes) } } - /// Reconstructs control-ring memory from handles duplicated into the current process. + /// Reconstructs control-ring memory with `layout` from handles duplicated into the current + /// process. /// /// # Safety /// /// Every handle must be live, owned by the caller, and valid in the current process. The first - /// handle must name a mapping of exactly [`CONTROL_RING_MEMORY_SIZE`] bytes, followed by the six - /// event handles in the transport-defined order. + /// handle must name a mapping of exactly [`ControlRingLayout::memory_size`] bytes, followed by + /// one event handle per waitable word of `layout`, in + /// [`ControlRingLayout::waitable_word_index`] order. pub(crate) unsafe fn control_ring_from_transferred( transfer: TransferredSharedMemory, + layout: ControlRingLayout, ) -> IoResult { - if transfer.length != CONTROL_RING_MEMORY_SIZE { + if transfer.length != layout.memory_size() { return Err(Error::new( ErrorKind::InvalidData, "invalid transferred control-ring length", )); } // SAFETY: The caller upholds the handle validity requirements documented above. - unsafe { Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::ControlRing) } + unsafe { + Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::ControlRing(layout)) + } } unsafe fn from_transferred_with_policy( @@ -249,7 +251,7 @@ impl WindowsSharedMemory { ) -> IoResult { let expected_handles = match policy { MemoryAccessPolicy::Bytes => 1, - MemoryAccessPolicy::ControlRing => 1 + CONTROL_RING_WAKE_OFFSETS.len(), + MemoryAccessPolicy::ControlRing(layout) => 1 + layout.waitable_word_count(), }; if transfer.length == 0 || transfer.handles.len() != expected_handles { return Err(Error::new( @@ -269,16 +271,7 @@ impl WindowsSharedMemory { )); } let mapping = Arc::new(handles.remove(0)); - let wake_handles = if handles.is_empty() { - None - } else { - Some(Arc::new(handles.try_into().map_err(|_| { - Error::new( - ErrorKind::InvalidData, - "invalid transferred control-ring wake handle count", - ) - })?)) - }; + let wake_handles = (!handles.is_empty()).then(|| Arc::from(handles)); Self::map(mapping, wake_handles, transfer.length, policy) } @@ -317,16 +310,18 @@ impl WindowsSharedMemory { } fn wake_handle(&self, offset: usize) -> IoResult { - let index = CONTROL_RING_WAKE_OFFSETS - .iter() - .position(|candidate| *candidate == offset) - .ok_or_else(|| { - Error::new(ErrorKind::InvalidInput, "invalid control-ring wait offset") - })?; - self.wake_handles - .as_ref() - .map(|handles| handles[index].0) - .ok_or_else(|| Error::new(ErrorKind::InvalidInput, "shared memory is not waitable")) + let (MemoryAccessPolicy::ControlRing(layout), Some(handles)) = + (self.policy, &self.wake_handles) + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "shared memory is not waitable", + )); + }; + let index = layout.waitable_word_index(offset).ok_or_else(|| { + Error::new(ErrorKind::InvalidInput, "invalid control-ring wait offset") + })?; + Ok(handles[index].0) } } @@ -339,19 +334,18 @@ fn parallel_spin_limit() -> u32 { } } -fn create_wake_handles() -> IoResult<[OwnedHandle; CONTROL_RING_WAKE_OFFSETS.len()]> { - let mut handles = Vec::with_capacity(CONTROL_RING_WAKE_OFFSETS.len()); - for _ in CONTROL_RING_WAKE_OFFSETS { - // SAFETY: The event is unnamed, auto-reset, initially nonsignaled, and has no custom ACL. - let handle = unsafe { CreateEventW(std::ptr::null(), 0, 0, std::ptr::null()) }; - if handle.is_null() { - return Err(Error::last_os_error()); - } - handles.push(OwnedHandle(handle)); - } - handles - .try_into() - .map_err(|_| Error::other("incorrect control-ring wake handle count")) +fn create_wake_handles(count: usize) -> IoResult> { + (0..count) + .map(|_| { + // SAFETY: The event is unnamed, auto-reset, initially nonsignaled, and has no custom + // ACL. + let handle = unsafe { CreateEventW(std::ptr::null(), 0, 0, std::ptr::null()) }; + if handle.is_null() { + return Err(Error::last_os_error()); + } + Ok(OwnedHandle(handle)) + }) + .collect() } /// Duplicates `handle` into `target_process` with `access`, or with the same access if `None`, @@ -558,7 +552,7 @@ impl WaitableSharedMemory for WindowsSharedMemory { mod tests { use super::*; use litebox_broker_transport::control_ring::{ - ControlRing, ControlRingReadStatus, ControlRingWriteStatus, + ControlRing, ControlRingDemandStatus, ControlRingReadStatus, ControlRingWriteStatus, }; use windows_sys::Win32::System::Memory::{ FILE_MAP_ALL_ACCESS, MEM_COMMIT, MEM_FREE, MEMORY_BASIC_INFORMATION, MapViewOfFile, @@ -641,13 +635,14 @@ mod tests { #[test] fn native_clones_do_not_inherit_association_views() { let pool = WindowsSharedMemory::create(64).unwrap(); - let ring = WindowsSharedMemory::create_control_ring().unwrap(); + let ring = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let transfer = ring .duplicate_to_process(unsafe { GetCurrentProcess() }) .unwrap(); // SAFETY: DuplicateHandle created every transferred handle in this process. let received_ring = - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) }.unwrap(); + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, LAYOUT) } + .unwrap(); // SAFETY: `pool.mapping` is a live section handle at least 64 bytes long. let inherited = unsafe { MapViewOfFile(pool.mapping.0, FILE_MAP_ALL_ACCESS, 0, 0, 64) }; assert!(!inherited.Value.is_null(), "{}", Error::last_os_error()); @@ -672,19 +667,24 @@ mod tests { assert_eq!(byte, [5]); } + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + #[test] fn cloned_views_share_control_ring_messages() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = local_memory.try_clone_view().unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); + let mut broker = ControlRing::new(broker_memory, LAYOUT).unwrap().into_host(); assert_eq!( - local.request_producer.try_write(b"request").unwrap(), + local.lanes[0] + .request_producer + .try_write(b"request") + .unwrap(), ControlRingWriteStatus::Written ); assert_eq!( - broker + broker.lanes[0] .request_consumer .try_read(|frame| Ok::<_, ()>(frame.to_vec())), Ok(ControlRingReadStatus::Message(b"request".to_vec())) @@ -693,10 +693,21 @@ mod tests { #[test] fn cloned_views_wake_blocked_control_ring_consumer() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = local_memory.try_clone_view().unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + // The last lane waits on different events than lane 0. + let mut local = ControlRing::new(local_memory, LAYOUT) + .unwrap() + .into_local() + .lanes + .pop() + .unwrap(); + let mut broker = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_host() + .lanes + .pop() + .unwrap(); let wait_epoch = match broker.request_consumer.try_read(|_| Ok::<_, ()>(())) { Ok(ControlRingReadStatus::Empty { wait_epoch }) => wait_epoch, @@ -722,10 +733,64 @@ mod tests { writer.join().unwrap(); } + #[test] + fn cloned_views_wake_broker_waiting_for_lane_demand() { + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); + let broker_memory = local_memory.try_clone_view().unwrap(); + let local_demand = ControlRing::new(local_memory, LAYOUT) + .unwrap() + .into_local() + .lane_demand + .unwrap(); + let mut broker_demand = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_host() + .lane_demand + .unwrap(); + + assert_eq!( + broker_demand.try_next_lane(), + Ok(ControlRingDemandStatus::Lane(0)) + ); + let demander = std::thread::spawn(move || { + local_demand.request(2).unwrap(); + local_demand.wake_host().unwrap(); + }); + loop { + match broker_demand.try_next_lane().unwrap() { + ControlRingDemandStatus::Lane(lane) => { + assert_eq!(lane, 1); + break; + } + ControlRingDemandStatus::Satisfied { wait_epoch } => { + broker_demand.wait_for_demand(wait_epoch).unwrap(); + } + } + } + demander.join().unwrap(); + } + + #[test] + fn every_waitable_control_ring_word_has_its_own_event() { + for layout in [ + ControlRingLayout::SINGLE_LANE, + ControlRingLayout::MULTI_LANE, + ] { + let memory = WindowsSharedMemory::create_control_ring(layout).unwrap(); + let mut events = (0..layout.memory_size()) + .filter(|offset| layout.waitable_word_index(*offset).is_some()) + .map(|offset| memory.wake_handle(offset).unwrap()) + .collect::>(); + events.sort_unstable(); + events.dedup(); + assert_eq!(events.len(), layout.waitable_word_count()); + } + } + #[test] fn event_wait_returns_without_peer_cooperation() { - let memory = WindowsSharedMemory::create_control_ring().unwrap(); - let epoch_offset = CONTROL_RING_WAKE_OFFSETS[0]; + let memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); + let epoch_offset = LAYOUT.requests(0).producer_epoch_offset(); let start = std::time::Instant::now(); memory.wait_while_equal(epoch_offset, 0).unwrap(); @@ -735,8 +800,8 @@ mod tests { #[test] fn byte_memory_does_not_infer_control_ring_from_length() { - let memory = WindowsSharedMemory::create(CONTROL_RING_MEMORY_SIZE).unwrap(); - let epoch_offset = CONTROL_RING_WAKE_OFFSETS[0]; + let memory = WindowsSharedMemory::create(LAYOUT.memory_size()).unwrap(); + let epoch_offset = LAYOUT.requests(0).producer_epoch_offset(); assert_eq!( memory.wake_one(epoch_offset).unwrap_err().kind(), @@ -752,23 +817,28 @@ mod tests { #[test] fn duplicated_handles_reconstruct_control_ring_view() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let transfer = local_memory .duplicate_to_process(unsafe { GetCurrentProcess() }) .unwrap(); + assert_eq!(transfer.handles.len(), 1 + LAYOUT.waitable_word_count()); // SAFETY: DuplicateHandle created every transferred handle in this process, and the source // mapping remains alive for the duration of the reconstructed view. let broker_memory = - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) }.unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, LAYOUT) } + .unwrap(); + let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); + let mut broker = ControlRing::new(broker_memory, LAYOUT).unwrap().into_host(); assert_eq!( - local.request_producer.try_write(b"transferred").unwrap(), + local.lanes[0] + .request_producer + .try_write(b"transferred") + .unwrap(), ControlRingWriteStatus::Written ); assert_eq!( - broker + broker.lanes[0] .request_consumer .try_read(|frame| Ok::<_, ()>(frame.to_vec())), Ok(ControlRingReadStatus::Message(b"transferred".to_vec())) diff --git a/litebox_broker_userland/src/lib.rs b/litebox_broker_userland/src/lib.rs index a63d519c89..927a694ccb 100644 --- a/litebox_broker_userland/src/lib.rs +++ b/litebox_broker_userland/src/lib.rs @@ -30,10 +30,3 @@ pub mod random; pub mod stdio; mod terminal; mod timer; - -/// The maximum number of worker threads that serve one association's requests. -/// -/// An association starts with a single worker, its own thread. Another worker is started only -/// when a request is about to wait, such as for a child process to start, and no worker is idle -/// to receive the next request. -const WORKER_COUNT: usize = 8; diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 4c2951817f..28ce1db0c7 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -6,23 +6,22 @@ //! This module owns association setup, request dispatch, readiness //! publication, cancellation, and teardown for one broker association. //! It is transport-neutral: it depends only on the [`HostSetupChannel`], -//! [`HostRequestSource`], [`HostResponseSink`], [`HostNotificationChannel`], -//! and [`HostAssociationShutdown`] contracts from +//! [`HostCallLaneSource`], [`HostCallLane`], [`HostNotificationChannel`], and +//! [`HostAssociationShutdown`] contracts from //! `litebox_broker_transport::channel`, so a deployment supplies the //! transport-specific setup channel, activation, and shared-memory plumbing //! and this runtime serves the association the same way regardless of //! transport. //! -//! Concurrency and worker sizing are deliberately not part of the public -//! surface. Both association entry points delegate to one internal runtime -//! that owns the worker count. +//! Concurrency is deliberately not part of the public surface. Both +//! association entry points delegate to one internal runtime that chooses the +//! control-ring layout and serves each of its call lanes with one worker. use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::sync::{ - Arc, Mutex, MutexGuard, + Arc, Mutex, atomic::{AtomicBool, Ordering}, }; -use std::thread::{JoinHandle, Thread}; use litebox_broker_core::BrokerCore; use litebox_broker_host::{ @@ -30,13 +29,12 @@ use litebox_broker_host::{ setup_connection, }; use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest, ProcessRequest}; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, }; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; use crate::process_launcher::{PendingRunnerAssociation, UserlandProcessLauncher}; @@ -62,10 +60,11 @@ pub(crate) fn is_peer_closed_error(error: &IoError) -> bool { /// `control_channel` must already be configured with whatever deadline and /// peer authentication its transport requires; this function only negotiates /// the broker protocol and control-ring setup over it. `create_shared_memory` -/// and `create_control_memory` allocate the transport's shared-memory -/// resources, `send_shared_memory` transfers them to the peer, and `activate` -/// consumes the negotiated setup channel and control ring into the transport's -/// active request, response, notification, and shutdown endpoints. +/// and `create_control_memory` (for the given control-ring layout) allocate +/// the transport's shared-memory resources, `send_shared_memory` transfers +/// them to the peer, and `activate` consumes the negotiated setup channel and +/// control ring into the transport's active call lanes, notification, and +/// shutdown endpoints. /// /// Returns once the association ends, whether by a clean peer close or a /// failure. Layout mismatches and broker setup failures are mapped to a @@ -73,28 +72,27 @@ pub(crate) fn is_peer_closed_error(error: &IoError) -> bool { pub fn serve_in_process_runner_association< Memory, SetupChannel, - RequestSource, - ResponseSink, + LaneSource, NotificationChannel, Shutdown, >( broker: &BrokerCore, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, ) -> IoResult<()> where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { serve_association_inner( broker, @@ -117,8 +115,7 @@ where pub(crate) fn serve_out_of_process_runner_association< Memory, SetupChannel, - RequestSource, - ResponseSink, + LaneSource, NotificationChannel, Shutdown, >( @@ -126,21 +123,21 @@ pub(crate) fn serve_out_of_process_runner_association< broker: BrokerCore, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, launcher: Arc, ) -> AssociationOutcome where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { let outcome = serve_association_inner( &broker, @@ -168,33 +165,26 @@ where } #[allow(clippy::too_many_arguments)] -fn serve_association_inner< - Memory, - SetupChannel, - RequestSource, - ResponseSink, - NotificationChannel, - Shutdown, ->( +fn serve_association_inner( broker: &BrokerCore, mut control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, launcher: Option>, startup: Option, ) -> IoResult where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { let (process, startup) = match startup { Some(startup) => { @@ -209,8 +199,9 @@ where SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?, ); - let control_memory = create_control_memory()?; - let control_ring = ControlRing::new(control_memory) + let layout = ControlRingLayout::MULTI_LANE; + let control_memory = create_control_memory(layout)?; + let control_ring = ControlRing::new(control_memory, layout) .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; let readiness = Arc::new(ReadinessPublisherRuntime::new()); let association = match setup_connection( @@ -245,7 +236,7 @@ where )); } }; - let (request_source, response_sink, notification_channel, shutdown) = + let (lane_source, notification_channel, shutdown) = match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { @@ -258,8 +249,7 @@ where Ok(dispatch_requests( association, readiness, - request_source, - response_sink, + lane_source, notification_channel, shutdown, launcher, @@ -377,12 +367,12 @@ impl> /// Fails the association if readiness publication unwinds. /// -/// The request reader owns association termination but does not depend on the -/// publisher, so an unwinding publisher would otherwise leave a live +/// The association thread owns association termination but does not depend on +/// the publisher, so an unwinding publisher would otherwise leave a live /// association with no notification source. The join that turns that panic into -/// a reported failure is reached only once the reader has returned, and a peer -/// that is waiting for a readiness change it will never be told about does not -/// return it. Failing the association here ends that wait instead. +/// a reported failure is reached only once the association has ended, and a +/// peer that is waiting for a readiness change it will never be told about does +/// not end it. Failing the association here ends it instead. struct PublisherPanicGuard<'association, Shutdown: HostAssociationShutdown> { failure_coordinator: &'association HostAssociationFailureCoordinator, } @@ -404,9 +394,10 @@ impl> Drop /// panic out of the association, and both states it can rest in have to end for /// that join to complete. Closing publication returns a publisher parked for /// work, and ending the transport returns one blocked on notification capacity -/// that a local endpoint stopped draining. The failure coordinator owns the -/// association until `dispatch_requests` returns, which is after that join, so -/// an unwind cannot leave ending the transport to dropping it. +/// that a local endpoint stopped draining, as well as any worker waiting for a +/// request. The failure coordinator owns the association until +/// `dispatch_requests` returns, which is after that join, so an unwind cannot +/// leave ending the transport to dropping it. struct ReadinessPublicationGuard<'association, Shutdown: HostAssociationShutdown> { readiness: &'association ReadinessPublisherRuntime, failure_coordinator: &'association HostAssociationFailureCoordinator, @@ -421,28 +412,41 @@ impl> Drop } } -/// Requests cancellation before an association scope joins its workers. -struct AssociationCancellationGuard<'association, Memory: SharedMemory> { +/// Ends an association once its end is observed, including on an unwind: +/// cancels the requests still executing and applies owner-death handling, which +/// fails the children still starting. +struct AssociationEndingGuard<'association, Memory: SharedMemory> { association: &'association BrokerHostAssociation, } -impl Drop for AssociationCancellationGuard<'_, Memory> { +impl Drop for AssociationEndingGuard<'_, Memory> { fn drop(&mut self) { self.association.request_cancellation(); + self.association.association_ending(); } } /// Serves one association until it ends, then reports its first failure. /// +/// Each call lane gets its own worker once the runner first uses it, so the +/// association has as many workers as the runner has had calls in flight at +/// once. Calls on different lanes never wait for each other, and back-to-back +/// calls stay on the first lane's worker. +/// +/// This thread serves the first lane: having run setup with the runner, it +/// tends to stay near the runner, whereas a new thread can land across a cache +/// boundary. An observer thread starts and joins the workers of further lanes +/// and ends the association as soon as it observes its end, so requests still +/// executing see that promptly. +/// /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network /// reactor is currently its production source. #[allow(clippy::too_many_arguments)] -fn dispatch_requests( +fn dispatch_requests( association: BrokerHostAssociation, readiness: Arc, - request_source: RequestSource, - response_sink: ResponseSink, + mut lane_source: LaneSource, mut notification_channel: NotificationChannel, shutdown: Shutdown, launcher: Option>, @@ -450,13 +454,12 @@ fn dispatch_requests AssociationOutcome where Memory: SharedMemory, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { - let association = Arc::new(association); - let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(shutdown)); + let failure_coordinator = HostAssociationFailureCoordinator::new(shutdown); if let Err(error) = association.activate_process() { return AssociationOutcome { result: Err(IoError::other(format!( @@ -465,31 +468,26 @@ where abnormal: true, }; } - let workers = Arc::new(Workers { - association: Arc::clone(&association), - requests: Mutex::new(request_source), - response_sink, - failure_coordinator: Arc::clone(&failure_coordinator), - launcher, - role: ReceivingRole::new(), - }); + let context = AssociationContext { + association: &association, + failure_coordinator: &failure_coordinator, + launcher: launcher.as_ref(), + }; std::thread::scope(|scope| { - let publisher_readiness = Arc::clone(&readiness); - let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() .name("litebox-broker-notifier".to_owned()) - .spawn_scoped(scope, move || { + .spawn_scoped(scope, || { let _panicking = PublisherPanicGuard { - failure_coordinator: &publisher_failure_coordinator, + failure_coordinator: &failure_coordinator, }; - // The request workers own association termination. A failing - // notification transport must fail the association before - // returning an error, so a worker still receiving observes and - // reports the same failure. Reporting here would instead turn - // a clean peer close into an error when its transport teardown - // releases a blocked notification send. - let _ = publisher_readiness.run(&mut notification_channel); + // The association thread owns association termination. A + // failing notification transport must fail the association + // before returning an error, so the association thread + // observes and reports the same failure. Reporting here would + // instead turn a clean peer close into an error when its + // transport teardown releases a blocked notification send. + let _ = readiness.run(&mut notification_channel); }); let publisher = match publisher { Ok(publisher) => Some(publisher), @@ -507,26 +505,28 @@ where readiness: &readiness, failure_coordinator: &failure_coordinator, }; - let cancellation = AssociationCancellationGuard { - association: &association, - }; - - // This thread is the first worker; others start when requests wait. - workers.run(false); - drop(cancellation); - for worker in workers.role.take_started() { - if worker.join().is_err() { - failure_coordinator.report_panic(IoError::other("broker request worker panicked")); + if let Some(first_lane) = context.next_lane(&mut lane_source) { + let observer = std::thread::Builder::new() + .name("litebox-broker-lanes".to_owned()) + .spawn_scoped(scope, move || context.serve_further_lanes(lane_source)); + match observer { + Ok(observer) => { + context.serve(first_lane); + if observer.join().is_err() { + failure_coordinator + .report_panic(IoError::other("broker call-lane observer panicked")); + } + } + Err(error) => failure_coordinator.report(error), } } - // Readiness publication lives exactly as long as the association. The - // request reader returns only once the association is over, but workers - // keep draining already-queued requests after that, so publication must - // outlive them or a late readiness change would be discarded. Ending it + // Ends the association if no observer did; ending it again is + // harmless. + association.association_ending(); + // Readiness publication lives exactly as long as the association, and + // workers can still change readiness until they are joined. Ending it // here rather than leaving it to the scope orders it before the join - // that observes a panicking publisher, and dropping the guard is what - // ends both states the publisher can rest in without depending on the - // reader having failed the association already. + // that observes a panicking publisher. drop(publication); if let Some(publisher) = publisher && publisher.join().is_err() @@ -535,14 +535,10 @@ where } }); - drop(workers); let result = match failure_coordinator.take_error() { Some(error) => Err(error), None => Ok(()), }; - let Ok(association) = Arc::try_unwrap(association) else { - panic!("all broker association workers must be joined before teardown"); - }; let panicked = failure_coordinator.panicked(); let abnormal = failure_coordinator.abnormal(); if panicked || !finish_process { @@ -553,242 +549,112 @@ where AssociationOutcome { result, abnormal } } -/// The workers serving one association's requests. -/// -/// The association's own thread is the first worker; [`ReceivingRole::pass`] -/// starts others only when a request is about to wait while no worker is -/// idle. -struct Workers { - association: Arc>, - /// Locked only by the worker holding `role`. - requests: Mutex, - response_sink: ResponseSink, - failure_coordinator: Arc>, - launcher: Option>, - role: ReceivingRole, -} - -/// The right to receive an association's next request. -/// -/// One worker at a time receives, and it keeps the role while it executes the -/// request it received, so the requests of a mostly sequential peer keep going -/// to one worker with a warm cache instead of rotating through ones that have -/// gone cold. A worker passes the role on only before executing a request that -/// waits (see [`Workers::run`]), so a slow request does not hold up later ones. -struct ReceivingRole { - state: Mutex, -} - -struct Receivers { - /// Whether the receiving worker has observed the end of the requests. - ended: bool, - /// Whether a worker holds the receiving role. - held: bool, - /// Workers waiting for the receiving role, most recently idle last. - /// - /// A worker waits only while another one holds the role, so this is empty - /// whenever `held` is false. - idle: Vec, - /// Workers started by [`ReceivingRole::pass`]. - started: Vec>, +/// What the threads serving an association share. +struct AssociationContext<'association, Memory: SharedMemory, Shutdown> { + association: &'association BrokerHostAssociation, + failure_coordinator: &'association HostAssociationFailureCoordinator, + launcher: Option<&'association Arc>, } -impl ReceivingRole { - fn new() -> Self { - Self { - state: Mutex::new(Receivers { - ended: false, - held: false, - idle: Vec::with_capacity(crate::WORKER_COUNT), - started: Vec::with_capacity(crate::WORKER_COUNT), - }), - } +impl Clone for AssociationContext<'_, Memory, Shutdown> { + fn clone(&self) -> Self { + *self } +} - fn state(&self) -> MutexGuard<'_, Receivers> { - self.state - .lock() - .expect("broker receiving role mutex poisoned") - } +impl Copy for AssociationContext<'_, Memory, Shutdown> {} - /// Waits for the role, or returns `false` once requests have ended. - fn take(&self) -> bool { - let mut state = self.state(); - if state.ended { - return false; - } - if !state.held { - state.held = true; - return true; - } - let worker = std::thread::current(); - let id = worker.id(); - state.idle.push(worker); - loop { - if state.ended { - return false; +impl AssociationContext<'_, Memory, Shutdown> +where + Memory: SharedMemory, + Shutdown: HostAssociationShutdown, +{ + /// Returns the next lane the runner uses, or `None` once the association + /// has ended, reporting why it ended abnormally. + fn next_lane>( + self, + lane_source: &mut LaneSource, + ) -> Option { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| lane_source.next_lane())) { + Ok(Ok(lane)) => lane, + Ok(Err(error)) => { + self.failure_coordinator.report(error); + None } - // Only a worker passing the role removes a waiting one. - if !state.idle.iter().any(|idle| idle.id() == id) { - return true; + Err(_) => { + self.failure_coordinator + .report_panic(IoError::other("broker call-lane source panicked")); + None } - drop(state); - std::thread::park(); - state = self.state(); - } - } - - /// Passes the role to the most recently idle worker or, while there are - /// fewer than [`crate::WORKER_COUNT`] workers, to one that `start` starts - /// already holding it given its index. - /// - /// Without either, the role is left for the next worker to finish. - fn pass(&self, start: impl FnOnce(usize) -> IoResult>) { - let mut state = self.state(); - if let Some(next) = state.idle.pop() { - drop(state); - next.unpark(); - return; - } - let index = state.started.len() + 1; - // Starting under the lock records the new worker before it can end - // the requests, so the association thread joins it. - if index < crate::WORKER_COUNT - && let Ok(worker) = start(index) - { - state.started.push(worker); - } else { - state.held = false; - } - } - - /// Ends receiving and returns every idle worker. - fn end(&self) { - let idle = { - let mut state = self.state(); - state.ended = true; - std::mem::take(&mut state.idle) - }; - for worker in idle { - worker.unpark(); } } - /// Takes the workers [`Self::pass`] started. - /// - /// Once requests have ended, no worker holds the role to start another. - fn take_started(&self) -> Vec> { - let mut state = self.state(); - debug_assert!(state.ended, "broker requests must end before joining"); - std::mem::take(&mut state.started) - } -} - -impl - Workers -where - Memory: SharedMemory, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, -{ - /// Serves requests until the association's requests end, starting with - /// the receiving role if `holds_role`. - /// - /// Every worker takes its turn receiving and executes each request it - /// receives. Once every worker is busy, requests wait in the transport - /// until one finishes. - fn run(self: &Arc, mut holds_role: bool) { - loop { - if !holds_role && !self.role.take() { - return; - } - let Some(request) = self.next_request() else { - return; + /// Starts a worker for each further lane the runner uses until the + /// association ends, then ends the association and joins the workers. + fn serve_further_lanes(self, mut lane_source: LaneSource) + where + LaneSource: HostCallLaneSource, + LaneSource::Lane: Send, + Self: Send, + { + std::thread::scope(|scope| { + // Dropped before the scope joins the workers, including on an + // unwind, so the join never waits on a worker blocked in a request. + let _ending = AssociationEndingGuard { + association: self.association, }; - // Starting a child process waits for its runner's setup, so another - // worker receives meanwhile. Other requests wait at most briefly for - // broker threads such as the socket reactor, and never for the - // guest, which learns of readiness through notifications instead. - holds_role = !matches!( - request.operation, - BrokerOperation::Process( - ProcessRequest::StartChild(_) | ProcessRequest::StartChildWithExistingRunner(_) - ) - ); - if !holds_role { - self.pass_role(); + let mut workers = 0; + while let Some(lane) = self.next_lane(&mut lane_source) { + workers += 1; + let started = std::thread::Builder::new() + .name(format!("litebox-broker-worker-{workers}")) + .spawn_scoped(scope, move || self.serve(lane)); + if let Err(error) = started { + self.failure_coordinator.report(error); + break; + } } - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.association.execute_request_with( + }); + } + + /// Serves one lane's requests until the association ends. + fn serve>(self, mut lane: Lane) { + let served = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + while !self.failure_coordinator.failed() { + let request = match lane.recv_request() { + Ok(HostReceive::Message(request)) => request, + Ok(HostReceive::PeerClosed) => return, + Ok(HostReceive::ProtocolViolation) => { + self.failure_coordinator.report(IoError::new( + ErrorKind::InvalidData, + "runner sent a request for the wrong protocol phase", + )); + return; + } + Err(error) => { + self.failure_coordinator.report(error); + return; + } + }; + match self.association.execute_request_with( request, |process, operation, shared_buffers| { - self.launcher.as_ref().and_then(|launcher| { + self.launcher.and_then(|launcher| { handle_process_operation(launcher, process, operation, shared_buffers) }) }, - |response| self.response_sink.send_response(response), - ) - })) { - Ok(Ok(()) | Err(BrokerHostError::AssociationFailed)) => {} - Ok(Err(error)) => self.failure_coordinator.report(map_host_error(error)), - Err(_) => { - self.failure_coordinator - .report_panic(IoError::other("broker request worker panicked")); + |response| lane.send_response(response), + ) { + Ok(()) | Err(BrokerHostError::AssociationFailed) => {} + Err(error) => self.failure_coordinator.report(map_host_error(error)), } } + })); + if served.is_err() { + self.failure_coordinator + .report_panic(IoError::other("broker request worker panicked")); } } - - /// Passes the receiving role on before executing a request that waits. - /// - /// See [`ReceivingRole::pass`] for which worker takes it. - fn pass_role(self: &Arc) { - self.role.pass(|index| { - let workers = Arc::clone(self); - std::thread::Builder::new() - .name(format!("litebox-broker-worker-{index}")) - .spawn(move || workers.run(true)) - }); - } - - /// Receives the next request while holding the receiving role, or returns - /// `None` once the association's requests have ended. - /// - /// The worker that observes the end reports why and starts association - /// teardown, so workers still executing requests see cancellation - /// promptly. - fn next_request(&self) -> Option { - let received = (!self.failure_coordinator.failed()).then(|| { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.requests - .lock() - .expect("broker request source mutex poisoned") - .recv_request() - })) - }); - if let Some(Ok(Ok(HostReceive::Message(request)))) = received { - return Some(request); - } - self.role.end(); - match received { - Some(Ok(Ok(HostReceive::ProtocolViolation))) => { - self.failure_coordinator.report(IoError::new( - ErrorKind::InvalidData, - "runner sent a request for the wrong protocol phase", - )); - } - Some(Ok(Err(error))) => self.failure_coordinator.report(error), - Some(Err(_)) => { - self.failure_coordinator - .report_panic(IoError::other("broker request reader panicked")); - } - Some(Ok(Ok(HostReceive::PeerClosed | HostReceive::Message(_)))) | None => {} - } - self.association.request_cancellation(); - self.association.association_ending(); - None - } } #[cfg(all(test, target_os = "linux"))] @@ -807,16 +673,16 @@ mod tests { use litebox_broker_protocol::message::{BrokerHandshakeResponse, BrokerNotification}; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_transport::channel::{ - HostNotificationChannel, HostReceive, HostSetupChannel, LocalSetupChannel, + HostNotificationChannel, HostSetupChannel, LocalSetupChannel, }; - use litebox_broker_transport::control_ring::ControlRing; + use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport::shared_memory::SharedBufferPool; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, - UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, - UnixControlRingLocalShutdown, UnixStreamHostSetupChannel, UnixStreamLocalSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixControlRingLocalCallChannel, + UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, + UnixStreamHostSetupChannel, UnixStreamLocalSetupChannel, }; use super::*; @@ -826,6 +692,8 @@ mod tests { /// the userland binary chooses for real runner processes. const TEST_SETUP_TIMEOUT: Duration = Duration::from_secs(5); + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + #[test] fn host_errors_preserve_io_categories() { for (error, expected) in [ @@ -863,10 +731,9 @@ mod tests { /// One live host association: the endpoints teardown acts on, and the rest /// held open so the association stays up for the duration of a test. struct LiveAssociation { - request_source: UnixControlRingHostRequestSource, + lane_source: UnixControlRingHostCallLaneSource, notifications: UnixControlRingHostNotificationChannel, shutdown: UnixControlRingHostShutdown, - _response_sink: UnixControlRingHostResponseSink, _local: ( UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -887,26 +754,42 @@ mod tests { }) .unwrap(); local_setup.recv_handshake_response().unwrap().unwrap(); - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let local_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let host_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let local_ring = ControlRing::new(local_memory).unwrap(); - let host_ring = ControlRing::new(host_memory).unwrap(); + let local_ring = ControlRing::new(local_memory, LAYOUT).unwrap(); + let host_ring = ControlRing::new(host_memory, LAYOUT).unwrap(); let local_activation = std::thread::spawn(move || local_setup.into_active(local_ring, || {}).unwrap()); - let (request_source, response_sink, notifications, shutdown) = + let (lane_source, notifications, shutdown) = control_channel.into_active(host_ring).unwrap(); LiveAssociation { - request_source, + lane_source, notifications, shutdown, - _response_sink: response_sink, _local: local_activation.join().unwrap(), } } + /// Waits for a call lane beyond the always-demanded first one, as the lane + /// observer does. + fn wait_for_more_lanes( + mut lane_source: UnixControlRingHostCallLaneSource, + ) -> Receiver> { + let first = lane_source.next_lane().unwrap().unwrap(); + let (result_sender, result_receiver) = sync_channel(1); + std::thread::spawn(move || { + let _first = first; + result_sender + .send(lane_source.next_lane().map(|lane| lane.is_some())) + .unwrap(); + }); + result_receiver + } + /// A notification channel that accepts every send and keeps nothing. struct DiscardingChannel; @@ -931,13 +814,14 @@ mod tests { UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - IoError::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = + ControlRing::new(control_memory, LAYOUT).map_err(|error| { + IoError::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; let (call_channel, notifications, shutdown) = setup.into_active(control_ring, || {})?; Ok(( @@ -979,8 +863,8 @@ mod tests { let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let shared_buffers = Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let control_ring = ControlRing::new(control_memory).unwrap(); + let control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let control_ring = ControlRing::new(control_memory, LAYOUT).unwrap(); let mut control = UnixStreamHostSetupChannel::from_host_guaranteed( host_stream, Instant::now() + TEST_SETUP_TIMEOUT, @@ -1000,16 +884,14 @@ mod tests { ) .unwrap() .unwrap(); - let (request_source, response_sink, notifications, shutdown) = - control.into_active(control_ring).unwrap(); + let (lane_source, notifications, shutdown) = control.into_active(control_ring).unwrap(); started.recv().unwrap(); outcome_sender .send( dispatch_requests( association, readiness, - request_source, - response_sink, + lane_source, notifications, shutdown, None, @@ -1103,18 +985,14 @@ mod tests { } #[test] - fn a_panicking_publisher_ends_a_blocked_request_reader() { + fn a_panicking_publisher_ends_the_wait_for_lanes() { let association = live_association(); - let mut request_source = association.request_source; let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(association.shutdown)); - let (result_sender, result_receiver) = sync_channel(1); - let reader = std::thread::spawn(move || { - result_sender.send(request_source.recv_request()).unwrap(); - }); + let result_receiver = wait_for_more_lanes(association.lane_source); - // The peer sends nothing and never closes, so the reader returns only if - // the publisher's unwind fails the association. + // The peer sends nothing and never closes, so the wait ends only if the + // publisher's unwind fails the association. let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::spawn(move || { let _panicking = PublisherPanicGuard { @@ -1123,38 +1001,26 @@ mod tests { panic!("readiness publication panicked"); }); - let receive_result = result_receiver + let wait_result = result_receiver .recv_timeout(TEST_SETUP_TIMEOUT) - .expect("a panicking publisher must end a blocked request reader"); - assert!(matches!( - receive_result, - Ok(HostReceive::PeerClosed) | Err(_) - )); - reader.join().unwrap(); + .expect("a panicking publisher must end the wait for lanes"); + assert!(matches!(wait_result, Ok(false) | Err(_))); assert!(publisher.join().is_err()); assert!(failure_coordinator.panicked()); assert!(failure_coordinator.take_error().is_some()); } #[test] - fn first_failure_is_preserved_and_unblocks_request_reading() { + fn first_failure_is_preserved_and_ends_the_wait_for_lanes() { let association = live_association(); - let mut request_source = association.request_source; let failure_coordinator = HostAssociationFailureCoordinator::new(association.shutdown); - let (result_sender, result_receiver) = sync_channel(1); - let reader = std::thread::spawn(move || { - result_sender.send(request_source.recv_request()).unwrap(); - }); + let result_receiver = wait_for_more_lanes(association.lane_source); failure_coordinator.report(IoError::new(ErrorKind::TimedOut, "first failure")); failure_coordinator.report(IoError::other("second failure")); - let receive_result = result_receiver.recv_timeout(TEST_SETUP_TIMEOUT); - reader.join().unwrap(); + let wait_result = result_receiver.recv_timeout(TEST_SETUP_TIMEOUT); - assert!(matches!( - receive_result.unwrap(), - Ok(HostReceive::PeerClosed) | Err(_) - )); + assert!(matches!(wait_result.unwrap(), Ok(false) | Err(_))); let error = failure_coordinator.take_error().unwrap(); assert!(!failure_coordinator.panicked()); assert_eq!(error.kind(), ErrorKind::TimedOut); @@ -1225,6 +1091,38 @@ mod tests { host.join().unwrap(); } + #[test] + fn dispatching_requests_serves_more_concurrent_calls_than_lanes() { + use litebox_broker_local::BrokerLocalError; + + let readiness = Arc::new(ReadinessPublisherRuntime::new()); + let (local, _notifications, _shutdown, outcome, host) = spawn_dispatch(readiness); + + // Callers beyond the lane count wait for a lane, and every call is + // answered on the lane it was made on. + std::thread::scope(|scope| { + for _ in 0..2 * LAYOUT.lane_count() { + scope.spawn(|| { + for _ in 0..50 { + assert!(matches!( + local.check_readiness(litebox_broker_protocol::ObjectHandle(u64::MAX)), + Err(BrokerLocalError::Broker(ErrorCode::UnknownObject)) + )); + } + }); + } + }); + + // Every started worker is waiting on its lane, and a clean local close + // must end them all along with dispatch. + drop(local); + outcome + .recv_timeout(TEST_SETUP_TIMEOUT) + .expect("a clean local close must end dispatch") + .unwrap(); + host.join().unwrap(); + } + #[test] fn dispatching_requests_fails_when_its_readiness_publisher_panics() { // Publication is one-shot, so a runtime that has already run makes the @@ -1234,7 +1132,7 @@ mod tests { readiness.run(&mut DiscardingChannel).unwrap(); // The local half stays connected and idle, so nothing but the panic can - // release the request reader that owns association termination. + // release the association thread that owns association termination. let (local, _notifications, _shutdown, outcome, host) = spawn_dispatch(Arc::clone(&readiness)); let error = outcome diff --git a/litebox_broker_userland/tests/notification_runtime.rs b/litebox_broker_userland/tests/notification_runtime.rs index a6f1738a6a..af54b7efc5 100644 --- a/litebox_broker_userland/tests/notification_runtime.rs +++ b/litebox_broker_userland/tests/notification_runtime.rs @@ -12,7 +12,7 @@ use std::time::Duration; use litebox_broker_core::test_support::TestBrokerCoreBuilder; use litebox_broker_core::{ObjectRights, PolicyEngine}; -use litebox_broker_host::{ConnectionTermination, setup_connection}; +use litebox_broker_host::{BrokerHostAssociation, setup_connection}; use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::ObjectHandle; use litebox_broker_protocol::fs::FileOpenFlags; @@ -20,11 +20,14 @@ use litebox_broker_protocol::message::{BrokerNotification, ReadinessNotification use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_transport::channel::{HostNotificationChannel, HostReceive}; -use litebox_broker_transport::control_ring::{CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRing}; +use litebox_broker_transport::control_ring::{ + CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRing, ControlRingLayout, +}; use litebox_broker_transport::shared_memory::SharedBufferPool; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixStreamHostSetupChannel, }; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -38,11 +41,33 @@ const TEST_TIMEOUT: Duration = Duration::from_secs(10); const BLOCK_DELAY: Duration = Duration::from_millis(50); /// More objects than the notification ring holds, so publication must block. const OVERSUBSCRIBED_OBJECT_COUNT: u64 = CONTROL_RING_NOTIFICATION_SLOT_COUNT * 3; +/// Layout of every test association's control ring. +const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + +/// Serves each call lane the local endpoint starts using on its own thread, +/// until the association ends: `Ok` if the local endpoint closed it. +fn serve_lanes( + mut lanes: UnixControlRingHostCallLaneSource, + association: &BrokerHostAssociation, +) -> std::io::Result<()> { + std::thread::scope(|scope| { + while let Some(mut lane) = lanes.next_lane()? { + scope.spawn(move || { + while let Ok(HostReceive::Message(request)) = lane.recv_request() { + association + .execute_request(request, |response| lane.send_response(response)) + .unwrap(); + } + }); + } + Ok(()) + }) +} /// Runs one host association whose only traffic is readiness notifications. /// -/// The reader thread mirrors production: it is the endpoint that observes local -/// termination and interrupts every ring wait of the association. +/// Its call lanes are served as in production, so the association ends when +/// the local endpoint closes it. fn spawn_host( stream: UnixStream, host: impl FnOnce(UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown) @@ -58,8 +83,8 @@ fn spawn_host( let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let shared_buffers = Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let control_ring = ControlRing::new(control_memory).unwrap(); + let control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let control_ring = ControlRing::new(control_memory, LAYOUT).unwrap(); let mut control = UnixStreamHostSetupChannel::from_accepted(stream); let association = setup_connection( &broker, @@ -76,15 +101,12 @@ fn spawn_host( ) .unwrap() .unwrap(); - let (mut request_source, response_sink, notifications, shutdown) = - control.into_active(control_ring).unwrap(); + let (lanes, notifications, shutdown) = control.into_active(control_ring).unwrap(); std::thread::scope(|scope| { + // Dropping `shutdown` when `host` returns can end the association + // before the local endpoint closes it. scope.spawn(|| { - while let Ok(HostReceive::Message(request)) = request_source.recv_request() { - association - .execute_request(request, |response| response_sink.send_response(response)) - .unwrap(); - } + let _ = serve_lanes(lanes, &association); }); host(notifications, shutdown); }); @@ -102,8 +124,8 @@ fn negotiate_local( UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = ControlRing::new(control_memory, LAYOUT).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid test control ring: {error:?}"), @@ -145,8 +167,8 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { let host_shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let host_shared_buffers = Arc::new(SharedBufferPool::new(host_shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let host_control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let host_control_ring = ControlRing::new(host_control_memory).unwrap(); + let host_control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let host_control_ring = ControlRing::new(host_control_memory, LAYOUT).unwrap(); let notification = BrokerNotification::Readiness(ReadinessNotification { handle: ObjectHandle(7), readiness: ReadinessFlags::READ, @@ -170,28 +192,17 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { ) .unwrap() .unwrap(); - let (mut request_source, response_sink, mut notifications, _shutdown) = - control.into_active(host_control_ring).unwrap(); + let (lanes, mut notifications, _shutdown) = control.into_active(host_control_ring).unwrap(); notifications.send_notification(&host_notification).unwrap(); - loop { - match request_source.recv_request().unwrap() { - HostReceive::Message(request) => association - .execute_request(request, |response| response_sink.send_response(response)) - .unwrap(), - HostReceive::PeerClosed => return ConnectionTermination::PeerClosed, - HostReceive::ProtocolViolation => { - return ConnectionTermination::ProtocolViolation; - } - } - } + serve_lanes(lanes, &association) }); let (local, _startup, notification_channel) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(local_control), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = ControlRing::new(control_memory, LAYOUT).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid test control ring: {error:?}"), @@ -216,10 +227,10 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { assert_eq!(local.add_event(handle, 1).unwrap(), readiness); drop(local); - assert_eq!( - host_thread.join().unwrap(), - ConnectionTermination::PeerClosed - ); + host_thread + .join() + .unwrap() + .expect("the host must see a clean close"); } #[test] diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 2856d5ceb9..a0f4ff3c5f 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -20,7 +20,7 @@ use litebox_broker_protocol::shared_buffer::{ #[cfg(target_os = "linux")] use litebox_broker_protocol::socket::{ReceiveFromFlags, SendFlags, SocketConnectionStatus}; use litebox_broker_transport::channel::LocalCallChannel; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; #[cfg(target_os = "linux")] use litebox_broker_transport_linux_userland::unix_socket::UnixStreamLocalSetupChannel; #[cfg(windows)] @@ -159,8 +159,10 @@ fn run_fake_runner(args: &[OsString]) { SHARED_BUFFER_POOL_SIZE, Some(Instant::now() + Duration::from_secs(5)), )?; - let control_memory = - setup.receive_control_ring(Some(Instant::now() + Duration::from_secs(5)))?; + let control_memory = setup.receive_control_ring( + ControlRingLayout::MULTI_LANE, + Some(Instant::now() + Duration::from_secs(5)), + )?; let process_image = setup.receive_process_image(Some(Instant::now() + Duration::from_secs(5)))?; (shared_memory, control_memory, process_image) @@ -168,17 +170,18 @@ fn run_fake_runner(args: &[OsString]) { #[cfg(windows)] let (shared_memory, control_memory, process_image) = { let shared_memory = setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?; - let control_memory = setup.receive_control_ring()?; + let control_memory = setup.receive_control_ring(ControlRingLayout::MULTI_LANE)?; let process_image = setup.receive_process_image()?; (shared_memory, control_memory, process_image) }; assert!(process_image.is_none()); - let control_ring = ControlRing::new(control_memory).map_err(|error| { - std::io::Error::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; + let control_ring = ControlRing::new(control_memory, ControlRingLayout::MULTI_LANE) + .map_err(|error| { + std::io::Error::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; #[cfg(target_os = "linux")] let (call_channel, _notifications, _shutdown) = setup.into_active(control_ring, || {})?; #[cfg(windows)] diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index de10598ebb..fcc0d6c47c 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -80,9 +80,8 @@ use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER use litebox_broker_protocol::wire; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, CONTROL_RING_MEMORY_SIZE, ControlRing, ControlRingConsumer, - ControlRingDirection, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, - MemoryAccessPolicy, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingProducer, + ControlRingReadStatus, ControlRingWriteStatus, HostControlRingEndpoints, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool}; use litebox_common_vm_abi::{Status, UserRange, WireFrame}; @@ -99,8 +98,12 @@ use timers::KernelTimers; /// Size of a process's broker shared memory. pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; +/// Layout of a process's broker control ring: the kernel serves one call at a +/// time, so a single lane. +pub const CONTROL_RING_LAYOUT: ControlRingLayout = ControlRingLayout::SINGLE_LANE; + /// Size of a process's broker control ring. -pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; +pub const CONTROL_RING_SIZE: usize = CONTROL_RING_LAYOUT.memory_size(); /// What the broker waits for: the runner's event sources (e.g., device /// interrupts and a timer). @@ -272,7 +275,8 @@ impl Broker { ) -> Association { let memory = PinnedControlRing::new(control_ring_pages).expect("the pages hold the control ring"); - let ring = ControlRing::new(memory).expect("a pinned control ring has the exact size"); + let ring = ControlRing::new(memory, CONTROL_RING_LAYOUT) + .expect("a pinned control ring has the exact size"); Association { broker: self.clone(), shared_memory, @@ -568,16 +572,18 @@ impl Association { return Err(Status::Denied); } let words = ring.memory().clone(); - let BrokerControlRingEndpoints { - request_consumer, - response_producer, + let HostControlRingEndpoints { + lanes, + lane_demand: _, notification_producer, - } = ring.into_broker(); + } = ring.into_host(); + let [lane] = <[_; 1]>::try_from(lanes) + .unwrap_or_else(|_| unreachable!("a single-lane layout has one lane")); self.readiness.activate(notification_producer); *state = State::Active(Box::new(Active { association: host, - requests: request_consumer, - responses: response_producer, + requests: lane.request_consumer, + responses: lane.response_producer, words, overflow: None, })); @@ -600,12 +606,12 @@ impl Association { if !matches!(*self.state.borrow(), State::Active(_)) { return Err(EnterError::Status(Status::Denied)); } - if !MemoryAccessPolicy::ControlRing.permits_u32(offset) { + if !CONTROL_RING_LAYOUT.permits_u32(offset) { return Err(EnterError::Status(Status::InvalidArgument)); } let wait = match op { RingFutex::Wake => { - if offset != ControlRingDirection::Requests.producer_epoch_offset() { + if offset != CONTROL_RING_LAYOUT.requests(0).producer_epoch_offset() { // The broker waits on no other word. return Ok(Entered::Done); } @@ -616,7 +622,7 @@ impl Association { expected, deadline, notifications: offset - == ControlRingDirection::Notifications.producer_epoch_offset(), + == CONTROL_RING_LAYOUT.notifications().producer_epoch_offset(), }), }; self.submitted.set(true); @@ -1079,14 +1085,17 @@ impl ProcessLauncher for KernelProcessLauncher { #[cfg(test)] mod tests { - use litebox_broker_transport::control_ring::CONTROL_RING_READY; + use super::CONTROL_RING_LAYOUT; /// `ABI_VERSION` stands in for the userland broker's ring-layout token; /// change both together. #[test] fn ring_layout_change_needs_an_abi_version_change() { assert_eq!( - (CONTROL_RING_READY, litebox_common_vm_abi::ABI_VERSION), + ( + CONTROL_RING_LAYOUT.ready_token(), + litebox_common_vm_abi::ABI_VERSION + ), (&b"litebox-control-ring-ready-v1"[..], 1) ); } diff --git a/litebox_broker_vm_kernel/src/memory.rs b/litebox_broker_vm_kernel/src/memory.rs index bbf2d1012f..0813fa01e0 100644 --- a/litebox_broker_vm_kernel/src/memory.rs +++ b/litebox_broker_vm_kernel/src/memory.rs @@ -4,11 +4,11 @@ //! The association's shared memory: the shared buffers in the process's lazy //! mapping, and the control ring in pinned pages. +use crate::{CONTROL_RING_LAYOUT, CONTROL_RING_SIZE}; use alloc::sync::Arc; use core::ops::Range; use litebox::mm::exception_table::memcpy_fallible; use litebox::platform::common_providers::userspace_pointers::ValidateAccess as _; -use litebox_broker_transport::control_ring::{CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy}; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_common_vm_abi::UserRange; use litebox_platform::buffer::Fault; @@ -71,14 +71,14 @@ impl SharedMemory for UserSharedMemory { } /// The control ring at the start of pinned pages, through the kernel's -/// mapping. Accesses keep to [`MemoryAccessPolicy::ControlRing`]. +/// mapping. Accesses keep to [`CONTROL_RING_LAYOUT`]. #[derive(Clone)] pub(crate) struct PinnedControlRing(Arc); impl PinnedControlRing { /// `None` if `pages` cannot hold the ring. pub(crate) fn new(pages: PinnedUserPages) -> Option { - (pages.len() >= CONTROL_RING_MEMORY_SIZE).then_some(Self(Arc::new(pages))) + (pages.len() >= CONTROL_RING_SIZE).then_some(Self(Arc::new(pages))) } fn word(&self, offset: usize, permitted: bool) -> Result<*mut T, SharedMemoryError> { @@ -97,11 +97,11 @@ impl PinnedControlRing { } fn u32_at(&self, offset: usize) -> Result<*mut u32, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u32(offset)) + self.word(offset, CONTROL_RING_LAYOUT.permits_u32(offset)) } fn u64_at(&self, offset: usize) -> Result<*mut u64, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u64(offset)) + self.word(offset, CONTROL_RING_LAYOUT.permits_u64(offset)) } /// Calls `copy` for each page's part of `offset..offset + len`, with that @@ -112,7 +112,7 @@ impl PinnedControlRing { len: usize, mut copy: impl FnMut(*mut u8, Range), ) -> Result<(), SharedMemoryError> { - if !MemoryAccessPolicy::ControlRing.permits_byte_range(offset, len) { + if !CONTROL_RING_LAYOUT.permits_byte_range(offset, len) { return Err(SharedMemoryError::InvalidRange); } let mut done = 0; @@ -133,7 +133,7 @@ impl PinnedControlRing { // (checked), and disjoint from private memory. impl SharedMemory for PinnedControlRing { fn len(&self) -> usize { - CONTROL_RING_MEMORY_SIZE + CONTROL_RING_SIZE } fn read(&self, offset: usize, destination: &mut [u8]) -> Result<(), SharedMemoryError> { diff --git a/litebox_common_vm_abi/src/lib.rs b/litebox_common_vm_abi/src/lib.rs index e3c6a70e11..1b80fd047a 100644 --- a/litebox_common_vm_abi/src/lib.rs +++ b/litebox_common_vm_abi/src/lib.rs @@ -194,8 +194,8 @@ //! no further request. //! - A malformed ring or request kills the process. Before the handshake, a //! futex operation on the control ring is [`Status::Denied`]; on the ring, -//! only the words that `litebox_broker_transport`'s -//! `MemoryAccessPolicy::ControlRing` lets a peer access as `u32` are valid. +//! only the words that `litebox_broker_transport`'s single-lane +//! `ControlRingLayout` lets a peer access as `u32` are valid. #![no_std] diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index e03f6eb803..61a28dff65 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -1757,6 +1757,22 @@ fn spawn_test_broker_with_mode( } } +/// Shuts a test broker association down if one of its lane threads, or the thread starting them, +/// panics, ending every wait on it so the test fails instead of hanging. +#[cfg(target_os = "linux")] +struct ShutdownOnPanic<'a>( + &'a litebox_broker_transport_linux_userland::unix_socket::UnixControlRingHostShutdown, +); + +#[cfg(target_os = "linux")] +impl Drop for ShutdownOnPanic<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + let _ = self.0.shutdown(); + } + } +} + #[cfg(target_os = "linux")] fn run_test_broker_connection( broker: &litebox_broker_core::BrokerCore, @@ -1774,11 +1790,15 @@ fn run_test_broker_connection( ) .expect("failed to attach broker test shared-buffer layout"), ); + let layout = litebox_broker_transport::control_ring::ControlRingLayout::MULTI_LANE; let control_memory = - litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory::create_control_ring() - .expect("failed to create broker test control ring"); - let control_ring = litebox_broker_transport::control_ring::ControlRing::new(control_memory) - .expect("failed to attach broker test control ring"); + litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory::create_control_ring( + layout, + ) + .expect("failed to create broker test control ring"); + let control_ring = + litebox_broker_transport::control_ring::ControlRing::new(control_memory, layout) + .expect("failed to attach broker test control ring"); let setup_deadline = std::time::Instant::now() + BROKER_HELPER_TIMEOUT; let mut channel = litebox_broker_transport_linux_userland::unix_socket::UnixStreamHostSetupChannel::from_host_guaranteed( @@ -1803,57 +1823,73 @@ fn run_test_broker_connection( ) .expect("broker host setup failed") .expect("broker setup terminated before activation"); - let (mut request_source, response_sink, mut notifications, _shutdown) = channel + let (mut lanes, mut notifications, shutdown) = channel .into_active(control_ring) .expect("failed to activate broker test control ring"); let publisher_readiness = readiness.clone(); let publisher = std::thread::spawn(move || publisher_readiness.run(&mut notifications)); - let mut close_object_count = 0; - // Files and the process's signals are not counted, since every runner opens them. - let mut uncounted_handles = Vec::new(); - let termination = loop { - match request_source - .recv_request() - .expect("failed to receive broker test request") + // The number of objects closed, and the open file and signal handles excluded from it since + // every runner opens them. + let counts = std::sync::Mutex::new((0, Vec::new())); + // Serves each call lane the runner starts using on its own thread, until the runner closes the + // association. + std::thread::scope(|scope| { + let _shutdown_on_panic = ShutdownOnPanic(&shutdown); + while let Some(mut lane) = lanes + .next_lane() + .expect("failed to receive a broker test call lane") { - litebox_broker_transport::channel::HostReceive::Message(request) => { - if let litebox_broker_protocol::message::BrokerOperation::CloseObject(handle) = - &request.operation - { - if let Some(index) = uncounted_handles.iter().position(|value| value == handle) + let (association, counts, shutdown) = (&association, &counts, &shutdown); + scope.spawn(move || { + let _shutdown_on_panic = ShutdownOnPanic(shutdown); + loop { + match lane + .recv_request() + .expect("failed to receive broker test request") { - uncounted_handles.swap_remove(index); - } else { - close_object_count += 1; + litebox_broker_transport::channel::HostReceive::Message(request) => { + if let litebox_broker_protocol::message::BrokerOperation::CloseObject( + handle, + ) = &request.operation + { + let (close_object_count, uncounted_handles) = + &mut *counts.lock().unwrap(); + if let Some(index) = + uncounted_handles.iter().position(|value| value == handle) + { + uncounted_handles.swap_remove(index); + } else { + *close_object_count += 1; + } + } + association + .execute_request(request, |response| { + match &response.result { + litebox_broker_protocol::message::BrokerResult::File( + litebox_broker_protocol::message::FileResponse::Open( + open, + ), + ) => counts.lock().unwrap().1.push(open.handle), + litebox_broker_protocol::message::BrokerResult::Signal( + litebox_broker_protocol::message::SignalResponse::Open( + open, + ), + ) => counts.lock().unwrap().1.push(open.handle), + _ => {} + } + lane.send_response(response) + }) + .expect("failed to execute broker test request"); + } + litebox_broker_transport::channel::HostReceive::PeerClosed => return, + litebox_broker_transport::channel::HostReceive::ProtocolViolation => { + panic!("broker test request violated the protocol"); + } } } - association - .execute_request(request, |response| { - match &response.result { - litebox_broker_protocol::message::BrokerResult::File( - litebox_broker_protocol::message::FileResponse::Open(open), - ) => uncounted_handles.push(open.handle), - litebox_broker_protocol::message::BrokerResult::Signal( - litebox_broker_protocol::message::SignalResponse::Open(open), - ) => uncounted_handles.push(open.handle), - _ => {} - } - response_sink.send_response(response) - }) - .expect("failed to execute broker test request"); - } - litebox_broker_transport::channel::HostReceive::PeerClosed => { - break litebox_broker_host::ConnectionTermination::PeerClosed; - } - litebox_broker_transport::channel::HostReceive::ProtocolViolation => { - break litebox_broker_host::ConnectionTermination::ProtocolViolation; - } + }); } - }; - assert_eq!( - termination, - litebox_broker_host::ConnectionTermination::PeerClosed - ); + }); readiness.close(); // The runner's clean close can fail a notification send already in flight, which the broker // runtime likewise ignores; any other publication failure is a real error. @@ -1873,6 +1909,7 @@ fn run_test_broker_connection( ); } association.finish(); + let (close_object_count, _) = counts.into_inner().unwrap(); close_object_count_tx .send(close_object_count) .expect("failed to report broker close-object count");