From fa088d4ecacdbae43700de94c3ede5766be5a56e Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Tue, 20 Oct 2026 23:07:48 -0700 Subject: [PATCH 01/12] Serve broker calls on per-call lanes Replace the userland broker's shared request/response rings and receiving-role hand-off with call lanes. A control ring now holds several single-slot request/response lanes; a runner call claims the lowest free lane, and the broker serves each lane on its own worker, started when the runner first uses the lane. The association thread serves the first lane itself, and an observer thread starts further workers and cancels requests once the association ends. The VM transport keeps the single-lane layout, which is byte-identical to the previous one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- dev_tests/src/ratchet.rs | 2 +- litebox_broker_local_userland/src/linux.rs | 47 +- litebox_broker_local_userland/src/windows.rs | 7 +- litebox_broker_local_vm_userland/src/lib.rs | 51 +- .../src/call_lane_pool.rs | 207 +++ litebox_broker_transport/src/channel.rs | 52 +- litebox_broker_transport/src/control_ring.rs | 1234 +++++++++++------ litebox_broker_transport/src/lib.rs | 2 +- litebox_broker_transport/src/pending_calls.rs | 316 ----- .../src/call_lane_pool.rs | 172 +++ .../src/lib.rs | 2 +- .../src/memfd.rs | 98 +- .../src/pending_calls.rs | 153 -- .../src/unix_socket/host.rs | 429 +++--- .../src/unix_socket/local.rs | 437 +++--- .../src/unix_socket/mod.rs | 8 +- .../src/call_lane_pool.rs | 172 +++ .../src/control_ring.rs | 4 +- .../src/host.rs | 192 ++- .../src/lib.rs | 2 +- .../src/local.rs | 217 +-- .../src/named_pipe.rs | 477 ++++++- .../src/pending_calls.rs | 76 - .../src/shared_memory.rs | 230 +-- litebox_broker_userland/src/lib.rs | 7 - litebox_broker_userland/src/runtime.rs | 651 ++++----- .../tests/notification_runtime.rs | 85 +- .../tests/userland_broker.rs | 23 +- litebox_broker_vm_kernel/src/lib.rs | 37 +- litebox_broker_vm_kernel/src/memory.rs | 14 +- litebox_common_vm_abi/src/lib.rs | 4 +- litebox_runner_linux_userland/tests/run.rs | 113 +- 32 files changed, 3282 insertions(+), 2239 deletions(-) create mode 100644 litebox_broker_transport/src/call_lane_pool.rs delete mode 100644 litebox_broker_transport/src/pending_calls.rs create mode 100644 litebox_broker_transport_linux_userland/src/call_lane_pool.rs delete mode 100644 litebox_broker_transport_linux_userland/src/pending_calls.rs create mode 100644 litebox_broker_transport_windows_userland/src/call_lane_pool.rs delete mode 100644 litebox_broker_transport_windows_userland/src/pending_calls.rs diff --git a/dev_tests/src/ratchet.rs b/dev_tests/src/ratchet.rs index 94aa8c11b0..11ff1f42d3 100644 --- a/dev_tests/src/ratchet.rs +++ b/dev_tests/src/ratchet.rs @@ -41,7 +41,7 @@ fn ratchet_pub_uses() -> Result<()> { ("litebox_broker_local_userland/", 2), ("litebox_broker_platform_linux_userland/", 3), ("litebox_broker_platform_windows_userland/", 1), - ("litebox_broker_transport_linux_userland/", 3), + ("litebox_broker_transport_linux_userland/", 2), ("litebox_broker_transport_windows_userland/", 4), ("litebox_common_linux/", 1), ("litebox_common_macos/", 2), diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index 8e526586b1..6a54d944c7 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -17,7 +17,7 @@ use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, UnixStreamLocalSetupChannel, @@ -84,13 +84,14 @@ fn negotiate( BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; - let control_memory = setup.receive_control_ring(Some(setup_deadline))?; + let layout = ControlRingLayout::MULTI_LANE; + let control_memory = setup.receive_control_ring(layout, Some(setup_deadline))?; let process_image = setup.receive_process_image(Some(setup_deadline))?; let positional_io_fds = [ shared_memory.as_fd().as_raw_fd(), control_memory.as_fd().as_raw_fd(), ]; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_ring = ControlRing::new(control_memory, layout).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid broker control ring: {error:?}"), @@ -268,8 +269,8 @@ mod tests { }; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixStreamHostSetupChannel, }; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -318,18 +319,19 @@ mod tests { UnixControlRingLocalCallChannel, UnixControlRingLocalShutdown, UnixControlRingLocalNotificationChannel, - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, ) { - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let layout = ControlRingLayout::MULTI_LANE; + let local_memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); let host_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.as_fd().try_clone_to_owned().unwrap(), + layout, ) .unwrap(); - let local_ring = ControlRing::new(local_memory).unwrap(); - let host_ring = ControlRing::new(host_memory).unwrap(); + let local_ring = ControlRing::new(local_memory, layout).unwrap(); + let host_ring = ControlRing::new(host_memory, layout).unwrap(); let host_activation = std::thread::spawn(move || host_channel.into_active(host_ring).unwrap()); let weak_association_coordinator = Arc::downgrade(association_coordinator); @@ -340,14 +342,12 @@ mod tests { } }) .unwrap(); - let (request_source, response_sink, host_notifications, host_shutdown) = - host_activation.join().unwrap(); + let (host_lanes, host_notifications, host_shutdown) = host_activation.join().unwrap(); ( local_call, local_shutdown, local_notifications, - request_source, - response_sink, + host_lanes, host_notifications, host_shutdown, ) @@ -362,8 +362,7 @@ mod tests { active_channel, association_shutdown, notification_channel, - host_request_source, - host_response_sink, + host_lanes, _host_notifications, host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); @@ -380,8 +379,7 @@ mod tests { .unwrap(); host_shutdown.shutdown().unwrap(); - drop(host_request_source); - drop(host_response_sink); + drop(host_lanes); failure_receiver .recv_timeout(Duration::from_secs(1)) @@ -401,8 +399,7 @@ mod tests { active_channel, association_shutdown, _notification_channel, - mut host_request_source, - _host_response_sink, + mut host_lanes, _host_notifications, _host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); @@ -419,10 +416,9 @@ mod tests { let (failure_sender, failure_receiver) = mpsc::sync_channel(1); association_coordinator.install_dispatch(move || failure_sender.send(()).unwrap()); failure_receiver.try_recv().unwrap(); - assert_eq!( - host_request_source.recv_request().unwrap(), - HostReceive::PeerClosed - ); + // Lane 0 is always demanded, so the broker may hand it out before it + // sees the close. + while host_lanes.next_lane().unwrap().is_some() {} drop(active_channel); } @@ -435,8 +431,7 @@ mod tests { active_channel, association_shutdown, notification_channel, - _host_request_source, - _host_response_sink, + _host_lanes, mut host_notifications, host_shutdown, ) = activate_control_channel(local_setup, host_control, &association_coordinator); diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index 1f33f25a37..bcdf1423f7 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -10,7 +10,7 @@ use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport_windows_userland::control_ring::{ WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, }; @@ -43,9 +43,10 @@ pub fn connect(control_pipe: &OsStr) -> Result<(BrokerConnection, Option Option { let start: usize = region.start.trunc(); - (region.len >= CONTROL_RING_MEMORY_SIZE as u64 && start.is_multiple_of(size_of::())) + (region.len >= LAYOUT.memory_size() as u64 && start.is_multiple_of(size_of::())) .then_some(Self { start }) } @@ -257,18 +258,18 @@ impl KernelControlRing { } fn u32_at(&self, offset: usize) -> Result<*mut u32, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u32(offset)) + self.word(offset, LAYOUT.permits_u32(offset)) } fn u64_at(&self, offset: usize) -> Result<*mut u64, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u64(offset)) + self.word(offset, LAYOUT.permits_u64(offset)) } fn bytes_at(&self, offset: usize, len: usize) -> Result { - if !MemoryAccessPolicy::ControlRing.permits_byte_range(offset, len) { + if !LAYOUT.permits_byte_range(offset, len) { return Err(SharedMemoryError::InvalidRange); } - checked(self.start, CONTROL_RING_MEMORY_SIZE, offset, len) + checked(self.start, LAYOUT.memory_size(), offset, len) } } @@ -276,7 +277,7 @@ impl KernelControlRing { // process's lifetime and disjoint from private memory. impl SharedMemory for KernelControlRing { fn len(&self) -> usize { - CONTROL_RING_MEMORY_SIZE + LAYOUT.memory_size() } fn read(&self, offset: usize, destination: &mut [u8]) -> Result<(), SharedMemoryError> { @@ -395,12 +396,16 @@ pub fn connect( .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange)) }; let LocalControlRingEndpoints { - request_producer, - response_consumer, + lanes, + lane_demand: _, notification_consumer, - } = ControlRing::new(ring()?) + } = ControlRing::new(ring()?, LAYOUT) .map_err(ChannelError::Ring)? .into_local(); + let [lane] = <[_; 1]>::try_from(lanes) + .unwrap_or_else(|_| unreachable!("a single-lane layout has one lane")); + let (request_producer, response_consumer) = + (lane.request_producer, lane.response_consumer); let notifications = Notifications { consumer: notification_consumer, words: ring()?, @@ -442,7 +447,7 @@ impl Notifications { if received != 0 { return Ok(received); } - let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + let epoch = LAYOUT.notifications().producer_epoch_offset(); match broker_wait(self.words.start, epoch, wait_epoch, deadline) { Ok(()) => {} Err(Status::TimedOut) => return Ok(0), @@ -661,7 +666,7 @@ fn futex( /// Has the kernel serve published broker requests: a wake on the requests' /// producer epoch of the control ring at `start`. fn broker_wake(start: usize) -> Result<(), Status> { - let offset = ControlRingDirection::Requests.producer_epoch_offset(); + let offset = LAYOUT.requests(0).producer_epoch_offset(); futex(start, |addr| FutexRequest::wake(addr, 1), offset) } @@ -691,8 +696,8 @@ fn broker_wait( /// /// On a failed check. pub fn self_check(notifications: &Notifications, platform: &VmUserland) { - let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); - let served = ControlRingDirection::Requests.consumer_epoch_offset(); + let epoch = LAYOUT.notifications().producer_epoch_offset(); + let served = LAYOUT.requests(0).consumer_epoch_offset(); let now = || platform.now(); let current = |offset| { notifications @@ -745,17 +750,17 @@ mod tests { fn ring(memory: &mut [u64]) -> (Notifications, ControlRingProducer) { let region = UserRange { start: memory.as_mut_ptr() as u64, - len: CONTROL_RING_MEMORY_SIZE as u64, + len: LAYOUT.memory_size() as u64, }; let ring = || KernelControlRing::new(region).expect("room for the ring"); let notifications = Notifications { - consumer: ControlRing::new(ring()) + consumer: ControlRing::new(ring(), LAYOUT) .expect("ring") .into_local() .notification_consumer, words: ring(), }; - let producer = ControlRing::new(ring()) + let producer = ControlRing::new(ring(), LAYOUT) .expect("ring") .into_broker() .notification_producer; @@ -770,7 +775,7 @@ mod tests { } fn ring_memory() -> Vec { - alloc::vec![0u64; CONTROL_RING_MEMORY_SIZE / size_of::()] + alloc::vec![0u64; LAYOUT.memory_size() / size_of::()] } #[test] diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs new file mode 100644 index 0000000000..f37d3079ca --- /dev/null +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Portable call-lane admission over caller-supplied synchronization. + +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::ops::{Deref, DerefMut}; + +/// A mutex usable by [`CallLanePool`]. +pub trait CallLanePoolMutex { + /// Guard granting access to the protected value. + type Guard<'a>: Deref + DerefMut + where + Self: 'a, + T: 'a; + + /// Locks the mutex. + fn lock(&self) -> Self::Guard<'_>; +} + +/// A condition variable paired with a [`CallLanePoolMutex`]. +pub trait CallLanePoolCondvar { + /// Mutex type whose guard this condition variable waits on. + type Mutex: CallLanePoolMutex; + + /// Atomically releases the guard and waits for a notification. + fn wait<'a>( + &self, + guard: >::Guard<'a>, + ) -> >::Guard<'a> + where + Self::Mutex: 'a, + T: 'a; + + /// Wakes one waiter. + fn notify_one(&self); + + /// Wakes all waiters. + fn notify_all(&self); +} + +/// Supplies synchronization primitives for [`CallLanePool`]. +pub trait CallLanePoolSync { + /// Mutex protecting a value of type `T`. + type Mutex: CallLanePoolMutex; + + /// Condition variable paired with [`Self::Mutex`]. + type Condvar: CallLanePoolCondvar>; + + /// Creates a mutex protecting `value`. + fn mutex(value: T) -> Self::Mutex; + + /// Creates a condition variable. + fn condvar() -> Self::Condvar; +} + +/// Failure from call-lane coordination. +#[derive(Debug)] +pub enum CallLanePoolError { + /// The association had already failed. + AssociationFailed(Arc), + /// The guarded operation failed. + Operation(Error), +} + +/// Admission of concurrent calls to a fixed set of call lanes. +/// +/// A call holds a lane from writing its request until reading its response, +/// so calls on different lanes never wait for each other. Calls claim the +/// lowest free lane, so an association uses only as many lanes as it has +/// concurrent calls; callers beyond the lane count wait for a lane to free up. +/// Lane 0 is always demanded; before a call first claims any higher lane, the +/// claim raises the demand so the peer starts serving that lane. +pub struct CallLanePool { + state: Sync::Mutex>, + lane_freed: Sync::Condvar>, +} + +struct CallLanePoolInner { + /// Each lane, or `None` while a call holds it. + lanes: Vec>, + /// Number of lanes demanded so far. + demanded: usize, + /// Number of claims waiting for a free lane. Freeing a lane notifies only + /// if one waits, since a notification can cost a system call even with no + /// waiters. + waiting: usize, + failure: Option>, +} + +/// A lane held by one call; dropping it frees the lane. +pub struct LaneClaim<'lanes, Sync: CallLanePoolSync, Lane, Error> { + lanes: &'lanes CallLanePool, + index: usize, + lane: Option, +} + +impl Deref for LaneClaim<'_, Sync, Lane, Error> { + type Target = Lane; + + fn deref(&self) -> &Lane { + self.lane + .as_ref() + .expect("claimed lane is present until drop") + } +} + +impl DerefMut for LaneClaim<'_, Sync, Lane, Error> { + fn deref_mut(&mut self) -> &mut Lane { + self.lane + .as_mut() + .expect("claimed lane is present until drop") + } +} + +impl Drop for LaneClaim<'_, Sync, Lane, Error> { + fn drop(&mut self) { + let mut state = self.lanes.state.lock(); + state.lanes[self.index] = self.lane.take(); + if state.waiting > 0 { + self.lanes.lane_freed.notify_one(); + } + } +} + +impl CallLanePool { + /// Creates live admission over `lanes`, of which only lane 0 is demanded. + /// + /// # Panics + /// + /// Panics if `lanes` is empty. + pub fn new(lanes: Vec) -> Self { + assert!(!lanes.is_empty(), "call lanes need at least one lane"); + Self { + state: Sync::mutex(CallLanePoolInner { + lanes: lanes.into_iter().map(Some).collect(), + demanded: 1, + waiting: 0, + failure: None, + }), + lane_freed: Sync::condvar(), + } + } + + /// Claims the lowest free lane, blocking while every lane is held. + /// + /// When the claimed lane is the first beyond those demanded so far, + /// `demand` is called with the new lane count, under the admission lock so + /// demand only rises. If it fails, the lane stays free and undemanded. + pub fn claim( + &self, + demand: impl FnOnce(usize) -> Result<(), Error>, + ) -> Result, CallLanePoolError> { + let mut state = self.state.lock(); + let index = loop { + if let Some(error) = state.failure.as_ref() { + return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); + } + if let Some(index) = state.lanes.iter().position(Option::is_some) { + break index; + } + state.waiting += 1; + state = self.lane_freed.wait(state); + state.waiting -= 1; + }; + if index >= state.demanded { + demand(index + 1).map_err(CallLanePoolError::Operation)?; + state.demanded = index + 1; + } + Ok(LaneClaim { + lanes: self, + index, + lane: state.lanes[index].take(), + }) + } + + /// Records the first terminal failure and wakes every waiting claim. + /// + /// Returns whether this call recorded the first failure. + pub fn record_failure(&self, error: Arc) -> bool { + let mut state = self.state.lock(); + if state.failure.is_some() { + return false; + } + state.failure = Some(error); + self.lane_freed.notify_all(); + true + } + + /// Returns the association's terminal failure, if one was recorded. + pub fn current_failure(&self) -> Option> { + self.state.lock().failure.as_ref().map(Arc::clone) + } + + /// Runs an operation while excluding failure recording. + pub fn run_if_live( + &self, + operation: impl FnOnce() -> Result, + ) -> Result> { + let state = self.state.lock(); + if let Some(error) = state.failure.as_ref() { + return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); + } + operation().map_err(CallLanePoolError::Operation) + } +} diff --git a/litebox_broker_transport/src/channel.rs b/litebox_broker_transport/src/channel.rs index b344898404..711fdb6527 100644 --- a/litebox_broker_transport/src/channel.rs +++ b/litebox_broker_transport/src/channel.rs @@ -129,40 +129,52 @@ pub trait HostNotificationChannel { /// /// An error is terminal when this channel is part of an active /// association. Before returning it, the transport must terminate that - /// association and unblock its request, response, notification, and - /// shutdown endpoints. This lets the association runtime preserve a clean - /// peer close while relying on the request endpoint to report transport - /// failures. + /// association and unblock its call-lane, notification, and shutdown + /// endpoints. This lets the association runtime preserve a clean peer + /// close while relying on the call lanes to report transport failures. fn send_notification(&mut self, notification: &BrokerNotification) -> Result<(), Self::Error>; } -/// Host-side endpoint that reads active broker requests from an association's -/// peer. +/// Host-side serving end of one call lane of an active association. /// -/// This is the request-reading half of an active association's control -/// transport. It is transport-neutral so a generic association runtime, -/// such as the one in `litebox_broker_userland`, can dispatch requests without -/// depending on a concrete transport crate. -pub trait HostRequestSource { +/// A lane carries one call at a time: the broker reads a request from it and +/// answers that request on the same lane before reading the next. Calls on +/// different lanes are independent, so a broker can serve each lane on its +/// own thread. The trait is transport-neutral so a generic association +/// runtime, such as the one in `litebox_broker_userland`, can serve calls +/// without depending on a concrete transport crate. +pub trait HostCallLane { /// Channel-specific error type. type Error; - /// Receives one active broker request. + /// Receives this lane's next broker request. fn recv_request(&mut self) -> Result, Self::Error>; + + /// Serializes and sends the response to this lane's last request. + fn send_response(&mut self, response: &BrokerResponse) -> Result<(), Self::Error>; } -/// Host-side endpoint that publishes active broker responses to an -/// association's peer. +/// Host-side source of the call lanes of an active association. /// -/// This is the response-writing half of an active association's control -/// transport. Implementations are typically cheaply [`Clone`]able so -/// concurrent request workers can each hold one. -pub trait HostResponseSink { +/// The peer starts using lanes as its concurrent calls grow, and this source +/// hands out each lane once the peer starts using it, so a broker serves only +/// lanes in use. The peer always uses the first lane, so the first +/// [`next_lane`](Self::next_lane) returns it without blocking unless the +/// association has already ended. +pub trait HostCallLaneSource { /// Channel-specific error type. type Error; - /// Serializes and sends one complete active broker response. - fn send_response(&self, response: &BrokerResponse) -> Result<(), Self::Error>; + /// Lane type handed out by this source. + type Lane: HostCallLane; + + /// Blocks until the peer starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` when the peer closed the association cleanly. An + /// error is terminal: before returning it, the transport must terminate + /// the association and unblock every endpoint. + fn next_lane(&mut self) -> Result, Self::Error>; } /// RAII-independent shutdown handle for an active broker association. diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index ed001f447b..83c7c07d90 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -9,6 +9,7 @@ //! rather than interpreting the same shared memory differently. use alloc::sync::Arc; +use alloc::vec::Vec; use core::mem::size_of; use core::ops::Range; use core::sync::atomic::{Ordering, fence}; @@ -18,9 +19,6 @@ use crate::shared_memory::{ControlRingMemory, SharedMemoryError}; /// Size of one shared control-ring slot. pub const CONTROL_RING_SLOT_SIZE: usize = 128; -/// Versioned token exchanged before endpoints activate this control-ring ABI. -pub const CONTROL_RING_READY: &[u8] = b"litebox-control-ring-ready-v1"; - /// Size of the fixed metadata at the start of a control-ring slot. pub const CONTROL_RING_SLOT_HEADER_SIZE: usize = 16; @@ -36,9 +34,6 @@ const _: () = assert!( ); const _: () = assert!(CONTROL_RING_SLOT_SIZE.is_multiple_of(size_of::())); -/// Number of slots in each request or response direction. -pub const CONTROL_RING_SLOT_COUNT: u64 = 64; - /// Number of slots in the broker-to-local notification direction. pub const CONTROL_RING_NOTIFICATION_SLOT_COUNT: u64 = 64; @@ -47,21 +42,10 @@ pub const CONTROL_RING_NOTIFICATION_SLOT_COUNT: u64 = 64; /// yet is far shorter than a scheduler time slice. pub const CONTROL_RING_PARALLEL_SPIN_LIMIT: u32 = 4000; -/// Exact shared-memory size required for all association control directions. -pub const CONTROL_RING_MEMORY_SIZE: usize = - CONTROL_RING_DATA_SIZE + CONTROL_RING_SYNC_DIRECTION_SIZE * 3; - -// The fixed count is representable by `usize` on every supported target. -#[allow(clippy::cast_possible_truncation)] -const CONTROL_RING_DIRECTION_SIZE: usize = - CONTROL_RING_SLOT_SIZE * CONTROL_RING_SLOT_COUNT as usize; // The fixed count is representable by `usize` on every supported target. #[allow(clippy::cast_possible_truncation)] const CONTROL_RING_NOTIFICATION_DIRECTION_SIZE: usize = CONTROL_RING_SLOT_SIZE * CONTROL_RING_NOTIFICATION_SLOT_COUNT as usize; -const CONTROL_RING_DATA_SIZE: usize = - CONTROL_RING_DIRECTION_SIZE * 2 + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE; -const CONTROL_RING_SYNC_DIRECTION_SIZE: usize = 24; const PRODUCER_EPOCH_OFFSET: usize = 0; const CONSUMER_EPOCH_OFFSET: usize = 4; const CONSUMER_HEAD_OFFSET: usize = 8; @@ -69,7 +53,230 @@ const CONSUMER_HEAD_OFFSET: usize = 8; const CONSUMER_WAITING_OFFSET: usize = 16; /// Nonzero while the producer may block on the consumer epoch. const PRODUCER_WAITING_OFFSET: usize = 20; -const _: () = assert!(CONTROL_RING_SYNC_DIRECTION_SIZE.is_multiple_of(size_of::())); +/// Size of the synchronization words of one direction. +const SYNC_WORDS_SIZE: usize = 24; +/// Cache-line size assumed when separating words that different threads poll. +const CACHE_LINE_SIZE: usize = 64; +/// Number of lanes the local peer asks the broker to serve. +const REQUESTED_LANES_OFFSET: usize = 0; +/// Incremented whenever the local peer raises its requested lane count. +const DEMAND_EPOCH_OFFSET: usize = 4; +const DEMAND_WORDS_SIZE: usize = 8; + +/// Shape of a control-ring mapping: its call lanes and their capacity. +/// +/// A mapping holds one or more *lanes*, each a request direction and a +/// response direction of the same slot count, and one broker-to-local +/// notification direction. Calls on different lanes never queue behind each +/// other, so a deployment that serves each lane separately can run as many +/// calls in parallel as it has lanes. +/// +/// The mapping starts with the slots of every direction, in the order lane 0 +/// requests, lane 0 responses, lane 1 requests, and so on, then notifications. +/// The synchronization words of each direction follow in the same order, one +/// block of `sync_stride` bytes per direction. With more than one lane, the +/// lane-demand words come last: the local peer raises the number of lanes it +/// uses there, so the broker can start serving a lane only once it is needed. +/// Lane 0 is always in use. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ControlRingLayout { + lanes: usize, + call_slots: u64, + sync_stride: usize, + ready: &'static [u8], +} + +const _: () = assert!(ControlRingLayout::SINGLE_LANE.memory_size() == 24_648); +const _: () = assert!(ControlRingLayout::MULTI_LANE.memory_size() == 11_336); + +impl ControlRingLayout { + /// One lane of 64 slots per direction, served by one broker thread. + pub const SINGLE_LANE: Self = Self { + lanes: 1, + call_slots: 64, + sync_stride: SYNC_WORDS_SIZE, + ready: b"litebox-control-ring-ready-v1", + }; + + /// Eight single-slot lanes, each carrying one call at a time. + /// + /// The synchronization words of each direction have a cache line to + /// themselves, so neither the two directions of a call nor lanes in use at + /// the same time contend for one. + pub const MULTI_LANE: Self = Self { + lanes: 8, + call_slots: 1, + sync_stride: CACHE_LINE_SIZE, + ready: b"litebox-control-ring-lanes-ready-v1", + }; + + /// Returns the number of lanes. + pub const fn lane_count(self) -> usize { + self.lanes + } + + /// Returns the versioned token endpoints exchange before activating a + /// mapping with this layout. + pub const fn ready_token(self) -> &'static [u8] { + self.ready + } + + /// Returns the exact size of a mapping with this layout. + pub const fn memory_size(self) -> usize { + self.demand_offset() + if self.lanes > 1 { DEMAND_WORDS_SIZE } else { 0 } + } + + /// Returns the local-to-broker request direction of `lane`. + /// + /// # Panics + /// + /// Panics if `lane` is not less than [`Self::lane_count`]. + pub const fn requests(self, lane: usize) -> ControlRingDirection { + assert!(lane < self.lanes, "control-ring lane out of range"); + self.direction(2 * lane) + } + + /// Returns the broker-to-local response direction of `lane`. + /// + /// # Panics + /// + /// Panics if `lane` is not less than [`Self::lane_count`]. + pub const fn responses(self, lane: usize) -> ControlRingDirection { + assert!(lane < self.lanes, "control-ring lane out of range"); + self.direction(2 * lane + 1) + } + + /// Returns the broker-to-local notification direction. + pub const fn notifications(self) -> ControlRingDirection { + self.direction(2 * self.lanes) + } + + /// Returns whether this layout permits a byte-copy range. + /// + /// Concrete shared-memory implementations use this together with + /// [`Self::permits_u32`] and [`Self::permits_u64`] to keep their own byte + /// and typed-word accesses disjoint. A peer can bypass these checks through + /// its backing-resource alias, so implementations must remain memory-safe + /// under arbitrary peer writes. + pub const fn permits_byte_range(self, offset: usize, length: usize) -> bool { + let Some(end) = offset.checked_add(length) else { + return false; + }; + if end > self.memory_size() { + return false; + } + if length == 0 { + return true; + } + let slot_offset = offset % CONTROL_RING_SLOT_SIZE; + end <= self.data_size() + && slot_offset >= size_of::() + && slot_offset + length <= CONTROL_RING_SLOT_SIZE + } + + /// Returns whether this layout permits a `u32` access at `offset`. + pub const fn permits_u32(self, offset: usize) -> bool { + let Some(relative) = offset.checked_sub(self.data_size()) else { + return false; + }; + if relative < self.sync_size() { + return matches!( + relative % self.sync_stride, + PRODUCER_EPOCH_OFFSET + | CONSUMER_EPOCH_OFFSET + | CONSUMER_WAITING_OFFSET + | PRODUCER_WAITING_OFFSET + ); + } + self.lanes > 1 + && matches!( + relative - self.sync_size(), + REQUESTED_LANES_OFFSET | DEMAND_EPOCH_OFFSET + ) + } + + /// Returns whether this layout permits a `u64` access at `offset`. + pub const fn permits_u64(self, offset: usize) -> bool { + if offset < self.data_size() { + return offset.is_multiple_of(CONTROL_RING_SLOT_SIZE); + } + let relative = offset - self.data_size(); + relative < self.sync_size() && relative % self.sync_stride == CONSUMER_HEAD_OFFSET + } + + /// Returns the number of `u32` words an endpoint may block on: the + /// producer and consumer epochs of every direction and, with more than one + /// lane, the lane-demand epoch. + pub const fn waitable_word_count(self) -> usize { + 2 * self.direction_count() + if self.lanes > 1 { 1 } else { 0 } + } + + /// Returns the position of the waitable `u32` at `offset` among the + /// [`Self::waitable_word_count`] words, or `None` if no endpoint blocks + /// on it. + pub const fn waitable_word_index(self, offset: usize) -> Option { + let Some(relative) = offset.checked_sub(self.data_size()) else { + return None; + }; + if relative < self.sync_size() { + let direction = relative / self.sync_stride; + return match relative % self.sync_stride { + PRODUCER_EPOCH_OFFSET => Some(2 * direction), + CONSUMER_EPOCH_OFFSET => Some(2 * direction + 1), + _ => None, + }; + } + if self.lanes > 1 && relative - self.sync_size() == DEMAND_EPOCH_OFFSET { + Some(2 * self.direction_count()) + } else { + None + } + } + + const fn direction_count(self) -> usize { + 2 * self.lanes + 1 + } + + // Layout slot counts are small constants representable by `usize`. + #[allow(clippy::cast_possible_truncation)] + const fn call_direction_size(self) -> usize { + CONTROL_RING_SLOT_SIZE * self.call_slots as usize + } + + const fn data_size(self) -> usize { + 2 * self.lanes * self.call_direction_size() + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE + } + + const fn sync_size(self) -> usize { + self.direction_count() * self.sync_stride + } + + const fn demand_offset(self) -> usize { + self.data_size() + self.sync_size() + } + + /// Returns the direction at `index` in mapping order, the notification + /// direction coming after every lane's. + const fn direction(self, index: usize) -> ControlRingDirection { + ControlRingDirection { + data_offset: index * self.call_direction_size(), + slot_count: if index == 2 * self.lanes { + CONTROL_RING_NOTIFICATION_SLOT_COUNT + } else { + self.call_slots + }, + sync_offset: self.data_size() + index * self.sync_stride, + } + } +} + +const _: () = assert!(SYNC_WORDS_SIZE.is_multiple_of(size_of::())); +const _: () = assert!(CACHE_LINE_SIZE >= SYNC_WORDS_SIZE); +const _: () = assert!( + ControlRingLayout::MULTI_LANE + .data_size() + .is_multiple_of(CACHE_LINE_SIZE) +); /// Access model enforced by a concrete shared-memory mapping. /// @@ -79,8 +286,9 @@ const _: () = assert!(CONTROL_RING_SYNC_DIRECTION_SIZE.is_multiple_of(size_of::< pub enum MemoryAccessPolicy { /// General shared memory supporting only byte copies. Bytes, - /// Control-ring memory supporting its ABI-defined byte and word accesses. - ControlRing, + /// Control-ring memory supporting the byte and word accesses its layout + /// defines. + ControlRing(ControlRingLayout), } impl MemoryAccessPolicy { @@ -88,163 +296,72 @@ impl MemoryAccessPolicy { pub const fn permits_byte_range(self, offset: usize, length: usize) -> bool { match self { Self::Bytes => true, - Self::ControlRing => memory_permits_byte_range(offset, length), + Self::ControlRing(layout) => layout.permits_byte_range(offset, length), } } /// Returns whether this policy permits a `u32` access at `offset`. pub const fn permits_u32(self, offset: usize) -> bool { - matches!(self, Self::ControlRing) && memory_permits_u32(offset) + match self { + Self::Bytes => false, + Self::ControlRing(layout) => layout.permits_u32(offset), + } } /// Returns whether this policy permits a `u64` access at `offset`. pub const fn permits_u64(self, offset: usize) -> bool { - matches!(self, Self::ControlRing) && memory_permits_u64(offset) - } -} - -/// Returns whether the control-ring ABI permits a byte-copy range. -/// -/// Concrete shared-memory implementations use this together with -/// [`memory_permits_u32`] and [`memory_permits_u64`] to keep their own byte and -/// typed-word accesses disjoint. A peer can bypass these checks through its -/// backing-resource alias, so implementations must remain memory-safe under -/// arbitrary peer writes. -pub const fn memory_permits_byte_range(offset: usize, length: usize) -> bool { - let Some(end) = offset.checked_add(length) else { - return false; - }; - if end > CONTROL_RING_MEMORY_SIZE { - return false; - } - if length == 0 { - return true; - } - direction_permits_byte_range(offset, length, 0, CONTROL_RING_SLOT_COUNT) - || direction_permits_byte_range( - offset, - length, - CONTROL_RING_DIRECTION_SIZE, - CONTROL_RING_SLOT_COUNT, - ) - || direction_permits_byte_range( - offset, - length, - CONTROL_RING_DIRECTION_SIZE * 2, - CONTROL_RING_NOTIFICATION_SLOT_COUNT, - ) -} - -/// Returns whether the control-ring ABI permits a `u32` access at `offset`. -pub const fn memory_permits_u32(offset: usize) -> bool { - let Some(relative) = offset.checked_sub(CONTROL_RING_DATA_SIZE) else { - return false; - }; - relative < CONTROL_RING_SYNC_DIRECTION_SIZE * 3 - && matches!( - relative % CONTROL_RING_SYNC_DIRECTION_SIZE, - PRODUCER_EPOCH_OFFSET - | CONSUMER_EPOCH_OFFSET - | CONSUMER_WAITING_OFFSET - | PRODUCER_WAITING_OFFSET - ) -} - -/// Returns whether the control-ring ABI permits a `u64` access at `offset`. -pub const fn memory_permits_u64(offset: usize) -> bool { - if offset < CONTROL_RING_DATA_SIZE { - return offset.is_multiple_of(CONTROL_RING_SLOT_SIZE); - } - let relative = offset - CONTROL_RING_DATA_SIZE; - relative < CONTROL_RING_SYNC_DIRECTION_SIZE * 3 - && relative % CONTROL_RING_SYNC_DIRECTION_SIZE == CONSUMER_HEAD_OFFSET -} - -const fn direction_permits_byte_range( - offset: usize, - length: usize, - direction_start: usize, - slot_count: u64, -) -> bool { - let Some(relative) = offset.checked_sub(direction_start) else { - return false; - }; - #[allow(clippy::cast_possible_truncation)] - let direction_size = CONTROL_RING_SLOT_SIZE * slot_count as usize; - if relative >= direction_size { - return false; + match self { + Self::Bytes => false, + Self::ControlRing(layout) => layout.permits_u64(offset), + } } - let slot_offset = relative % CONTROL_RING_SLOT_SIZE; - let Some(slot_end) = slot_offset.checked_add(length) else { - return false; - }; - slot_offset >= size_of::() && slot_end <= CONTROL_RING_SLOT_SIZE } -/// One direction in the shared control-ring mapping. +/// One direction in a control-ring mapping, as placed by its +/// [`ControlRingLayout`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ControlRingDirection { - /// Local-to-broker request ring. - Requests, - /// Broker-to-local response ring. - Responses, - /// Broker-to-local asynchronous notification ring. - Notifications, +pub struct ControlRingDirection { + data_offset: usize, + slot_count: u64, + sync_offset: usize, } impl ControlRingDirection { fn slot_range(self, slot: u64) -> Range { - debug_assert!(slot < self.slot_count()); + debug_assert!(slot < self.slot_count); let slot = usize::try_from(slot).expect("control-ring slot index is bounded"); - let data_offset = match self { - Self::Requests => 0, - Self::Responses => CONTROL_RING_DIRECTION_SIZE, - Self::Notifications => CONTROL_RING_DIRECTION_SIZE * 2, - }; - let start = data_offset + slot * CONTROL_RING_SLOT_SIZE; + let start = self.data_offset + slot * CONTROL_RING_SLOT_SIZE; start..start + CONTROL_RING_SLOT_SIZE } /// Returns the number of slots in this direction. pub const fn slot_count(self) -> u64 { - match self { - Self::Requests | Self::Responses => CONTROL_RING_SLOT_COUNT, - Self::Notifications => CONTROL_RING_NOTIFICATION_SLOT_COUNT, - } + self.slot_count } /// Returns the `u32` epoch incremented when the producer publishes work for /// this direction. pub const fn producer_epoch_offset(self) -> usize { - self.sync_offset() + PRODUCER_EPOCH_OFFSET + self.sync_offset + PRODUCER_EPOCH_OFFSET } /// Returns the `u32` epoch incremented when the consumer publishes progress /// for this direction. pub const fn consumer_epoch_offset(self) -> usize { - self.sync_offset() + CONSUMER_EPOCH_OFFSET + self.sync_offset + CONSUMER_EPOCH_OFFSET } /// Returns the `u64` consumer-head offset for this direction. pub const fn consumer_head_offset(self) -> usize { - self.sync_offset() + CONSUMER_HEAD_OFFSET + self.sync_offset + CONSUMER_HEAD_OFFSET } const fn consumer_waiting_offset(self) -> usize { - self.sync_offset() + CONSUMER_WAITING_OFFSET + self.sync_offset + CONSUMER_WAITING_OFFSET } const fn producer_waiting_offset(self) -> usize { - self.sync_offset() + PRODUCER_WAITING_OFFSET - } - - const fn sync_offset(self) -> usize { - CONTROL_RING_DATA_SIZE - + match self { - Self::Requests => 0, - Self::Responses => CONTROL_RING_SYNC_DIRECTION_SIZE, - Self::Notifications => CONTROL_RING_SYNC_DIRECTION_SIZE * 2, - } + self.sync_offset + PRODUCER_WAITING_OFFSET } } @@ -342,42 +459,95 @@ pub enum ControlRingReadError { Decode(DecodeError), } -/// Exact-size shared memory containing request, response, and notification rings. +/// Exact-size shared memory holding the directions of a [`ControlRingLayout`]. pub struct ControlRing { memory: Memory, + layout: ControlRingLayout, } -/// Role-bound association ring endpoints owned by the local peer. -pub struct LocalControlRingEndpoints { +/// The local peer's ends of one call lane. +pub struct LocalControlRingLane { /// Local-to-broker request producer. pub request_producer: ControlRingProducer, /// Broker-to-local response consumer. pub response_consumer: ControlRingConsumer, - /// Broker-to-local notification consumer. - pub notification_consumer: ControlRingConsumer, } -/// Role-bound association ring endpoints owned by the broker peer. -pub struct BrokerControlRingEndpoints { +/// The broker peer's ends of one call lane. +pub struct BrokerControlRingLane { /// Local-to-broker request consumer. pub request_consumer: ControlRingConsumer, /// Broker-to-local response producer. pub response_producer: ControlRingProducer, +} + +/// Role-bound association ring endpoints owned by the local peer. +pub struct LocalControlRingEndpoints { + /// Call lanes, in lane order. + pub lanes: Vec>, + /// Lane-demand publisher, present when the layout has more than one lane. + pub lane_demand: Option>, + /// Broker-to-local notification consumer. + pub notification_consumer: ControlRingConsumer, +} + +/// Role-bound association ring endpoints owned by the broker peer. +pub struct BrokerControlRingEndpoints { + /// Call lanes, in lane order. + pub lanes: Vec>, + /// Lane-demand observer, present when the layout has more than one lane. + pub lane_demand: Option>, /// Broker-to-local notification producer. pub notification_producer: ControlRingProducer, } +impl LocalControlRingEndpoints { + /// Returns handles that interrupt every wait these endpoints offer. + pub fn wake_handles(&self) -> Vec> { + self.lanes + .iter() + .flat_map(|lane| { + [ + lane.request_producer.wake_handle(), + lane.response_consumer.wake_handle(), + ] + }) + .chain([self.notification_consumer.wake_handle()]) + .collect() + } +} + +impl BrokerControlRingEndpoints { + /// Returns handles that interrupt every wait these endpoints offer. + pub fn wake_handles(&self) -> Vec> { + self.lanes + .iter() + .flat_map(|lane| { + [ + lane.request_consumer.wake_handle(), + lane.response_producer.wake_handle(), + ] + }) + .chain([self.notification_producer.wake_handle()]) + .chain(self.lane_demand.as_ref().map(BrokerLaneDemand::wake_handle)) + .collect() + } +} + impl ControlRing { - /// Attaches to an exact-size shared control-ring mapping. - pub fn new(memory: Memory) -> Result { + /// Attaches to an exact-size shared control-ring mapping with `layout`. + pub fn new(memory: Memory, layout: ControlRingLayout) -> Result { let actual = memory.len(); - if actual != CONTROL_RING_MEMORY_SIZE { - return Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual, - }); + let expected = layout.memory_size(); + if actual != expected { + return Err(ControlRingError::MemoryLengthMismatch { expected, actual }); } - Ok(Self { memory }) + Ok(Self { memory, layout }) + } + + /// Returns the layout of this mapping. + pub const fn layout(&self) -> ControlRingLayout { + self.layout } /// Returns the copy-only shared-memory resource. @@ -387,39 +557,50 @@ impl ControlRing { /// Consumes the mapping into endpoints owned by the local peer. pub fn into_local(self) -> LocalControlRingEndpoints { + let layout = self.layout; let ring = Arc::new(self); LocalControlRingEndpoints { - request_producer: ControlRingProducer::new( - Arc::clone(&ring), - ControlRingDirection::Requests, - ), - response_consumer: ControlRingConsumer::new( - Arc::clone(&ring), - ControlRingDirection::Responses, - ), - notification_consumer: ControlRingConsumer::new( - ring, - ControlRingDirection::Notifications, - ), + lanes: (0..layout.lane_count()) + .map(|lane| LocalControlRingLane { + request_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }) + .collect(), + lane_demand: (layout.lane_count() > 1).then(|| LocalLaneDemand { + ring: Arc::clone(&ring), + }), + notification_consumer: ControlRingConsumer::new(ring, layout.notifications()), } } /// Consumes the mapping into endpoints owned by the broker peer. pub fn into_broker(self) -> BrokerControlRingEndpoints { + let layout = self.layout; let ring = Arc::new(self); BrokerControlRingEndpoints { - request_consumer: ControlRingConsumer::new( - Arc::clone(&ring), - ControlRingDirection::Requests, - ), - response_producer: ControlRingProducer::new( - Arc::clone(&ring), - ControlRingDirection::Responses, - ), - notification_producer: ControlRingProducer::new( - ring, - ControlRingDirection::Notifications, - ), + lanes: (0..layout.lane_count()) + .map(|lane| BrokerControlRingLane { + request_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }) + .collect(), + lane_demand: (layout.lane_count() > 1).then(|| BrokerLaneDemand { + ring: Arc::clone(&ring), + started: 0, + }), + notification_producer: ControlRingProducer::new(ring, layout.notifications()), } } @@ -506,6 +687,118 @@ impl ControlRing { } } +/// Local-peer end of the lane-demand words of a multi-lane control ring. +/// +/// The local peer raises the number of lanes it uses before it first writes a +/// request to a lane, and the broker starts serving a lane once it is +/// demanded. Lane 0 is always demanded. +pub struct LocalLaneDemand { + ring: Arc>, +} + +impl LocalLaneDemand { + /// Publishes that the local peer uses the first `lanes` lanes. + /// + /// The count must not decrease within an association. + /// + /// # Panics + /// + /// Panics if `lanes` exceeds the layout's lane count. + pub fn request(&self, lanes: usize) -> Result<(), ControlRingError> { + let layout = self.ring.layout; + assert!(lanes <= layout.lane_count(), "lane demand out of range"); + let lanes = u32::try_from(lanes).expect("lane count is bounded by the layout"); + let demand = layout.demand_offset(); + self.ring + .memory + .store_u32_release(demand + REQUESTED_LANES_OFFSET, lanes.to_le())?; + self.ring + .memory + .increment_u32_release(demand + DEMAND_EPOCH_OFFSET)?; + Ok(()) + } +} + +impl LocalLaneDemand { + /// Wakes the broker after [`Self::request`]. + /// + /// The wake is unconditional: demand rises at most once per lane, so + /// there is nothing to gain from an announcement word. + pub fn wake_broker(&self) -> Result<(), Memory::Error> { + self.ring + .memory + .wake_one(self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET) + } +} + +/// Result of polling a [`BrokerLaneDemand`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ControlRingDemandStatus { + /// The local peer demands this lane, which the broker has yet to serve. + Lane(usize), + /// The broker serves every demanded lane. + Satisfied { + /// Demand epoch sampled before the demand was read. + wait_epoch: u32, + }, +} + +/// Broker-peer end of the lane-demand words of a multi-lane control ring. +/// +/// It hands out each lane, in lane order, once the local peer demands it. +/// The local peer can publish any count; the broker clamps it to the layout's +/// lanes and never takes a lane back. +pub struct BrokerLaneDemand { + ring: Arc>, + started: usize, +} + +impl BrokerLaneDemand { + /// Returns the next demanded lane the broker has yet to serve, if any. + pub fn try_next_lane(&mut self) -> Result { + let layout = self.ring.layout; + let demand = layout.demand_offset(); + let wait_epoch = self + .ring + .memory + .load_u32_acquire(demand + DEMAND_EPOCH_OFFSET)?; + let requested = u32::from_le( + self.ring + .memory + .load_u32_acquire(demand + REQUESTED_LANES_OFFSET)?, + ); + let demanded = usize::try_from(requested) + .unwrap_or(usize::MAX) + .clamp(1, layout.lane_count()); + if self.started < demanded { + let lane = self.started; + self.started += 1; + return Ok(ControlRingDemandStatus::Lane(lane)); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) + } + + /// Returns a handle that can interrupt [`Self::wait_for_demand`]. + pub fn wake_handle(&self) -> ControlRingWakeHandle { + ControlRingWakeHandle::new( + &self.ring, + self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET, + ) + } +} + +impl BrokerLaneDemand { + /// Waits for new demand after [`ControlRingDemandStatus::Satisfied`]. + /// + /// The caller must poll again after this possibly spurious wakeup. + pub fn wait_for_demand(&mut self, wait_epoch: u32) -> Result<(), Memory::Error> { + self.ring.memory.wait_while_equal( + self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET, + wait_epoch, + ) + } +} + /// Trusted endpoint-local state for one control-ring producer. pub struct ControlRingProducer { ring: Arc>, @@ -524,7 +817,7 @@ pub struct ControlRingProducer { /// without gaining access to ring memory. pub struct ControlRingWakeHandle { ring: Arc>, - wait_epoch: ControlRingWaitEpoch, + epoch_offset: usize, } /// Shared-memory wait and wake support for control-ring epoch words. @@ -570,33 +863,25 @@ pub trait WaitableSharedMemory: ControlRingMemory { } } -#[derive(Clone, Copy)] -enum ControlRingWaitEpoch { - Producer(ControlRingDirection), - Consumer(ControlRingDirection), -} - -impl ControlRingWaitEpoch { - const fn offset(self) -> usize { - match self { - Self::Producer(direction) => direction.producer_epoch_offset(), - Self::Consumer(direction) => direction.consumer_epoch_offset(), - } - } -} - impl Clone for ControlRingWakeHandle { fn clone(&self) -> Self { Self { ring: Arc::clone(&self.ring), - wait_epoch: self.wait_epoch, + epoch_offset: self.epoch_offset, } } } impl ControlRingWakeHandle { + fn new(ring: &Arc>, epoch_offset: usize) -> Self { + Self { + ring: Arc::clone(ring), + epoch_offset, + } + } + pub(crate) const fn wait_epoch_offset(&self) -> usize { - self.wait_epoch.offset() + self.epoch_offset } pub(crate) fn memory(&self) -> &Memory { @@ -694,10 +979,7 @@ impl ControlRingProducer { /// Returns a handle that can interrupt a wait on this producer's ring /// direction. pub fn wake_handle(&self) -> ControlRingWakeHandle { - ControlRingWakeHandle { - ring: Arc::clone(&self.ring), - wait_epoch: ControlRingWaitEpoch::Consumer(self.direction), - } + ControlRingWakeHandle::new(&self.ring, self.direction.consumer_epoch_offset()) } #[cfg(test)] @@ -810,10 +1092,7 @@ impl ControlRingConsumer { /// Returns a handle that can interrupt a wait on this consumer's ring /// direction. pub fn wake_handle(&self) -> ControlRingWakeHandle { - ControlRingWakeHandle { - ring: Arc::clone(&self.ring), - wait_epoch: ControlRingWaitEpoch::Producer(self.direction), - } + ControlRingWakeHandle::new(&self.ring, self.direction.producer_epoch_offset()) } #[cfg(test)] @@ -951,170 +1230,358 @@ mod tests { const SEQUENCE_RANGE: core::ops::Range = 0..8; const LENGTH_RANGE: core::ops::Range = 8..12; const RESERVED_RANGE: core::ops::Range = 12..16; + const LAYOUT: ControlRingLayout = ControlRingLayout::SINGLE_LANE; + const LAYOUTS: [ControlRingLayout; 2] = [ + ControlRingLayout::SINGLE_LANE, + ControlRingLayout::MULTI_LANE, + ]; + const MEMORY_SIZE: usize = LAYOUT.memory_size(); + const REQUESTS: ControlRingDirection = LAYOUT.requests(0); + const RESPONSES: ControlRingDirection = LAYOUT.responses(0); + const NOTIFICATIONS: ControlRingDirection = LAYOUT.notifications(); + const SLOT_COUNT: u64 = REQUESTS.slot_count(); #[test] fn mapping_requires_the_exact_control_ring_size() { - assert_eq!(CONTROL_RING_MEMORY_SIZE, 24_648); - assert!(matches!( - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE - 1)), - Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual - }) if actual == CONTROL_RING_MEMORY_SIZE - 1 - )); - assert!(matches!( - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE + 1)), - Err(ControlRingError::MemoryLengthMismatch { - expected: CONTROL_RING_MEMORY_SIZE, - actual - }) if actual == CONTROL_RING_MEMORY_SIZE + 1 - )); - assert!(ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)).is_ok()); + assert_eq!(ControlRingLayout::SINGLE_LANE.memory_size(), 24_648); + for layout in LAYOUTS { + let size = layout.memory_size(); + assert!(matches!( + ControlRing::new(TestMemory::new(size - 1), layout), + Err(ControlRingError::MemoryLengthMismatch { expected, actual }) + if expected == size && actual == size - 1 + )); + assert!(matches!( + ControlRing::new(TestMemory::new(size + 1), layout), + Err(ControlRingError::MemoryLengthMismatch { expected, actual }) + if expected == size && actual == size + 1 + )); + assert!(ControlRing::new(TestMemory::new(size), layout).is_ok()); + } } #[test] fn memory_layout_separates_byte_and_word_regions() { - assert!(memory_permits_u64(0)); - assert!(!memory_permits_u32(0)); - assert!(!memory_permits_byte_range(0, 1)); - assert!(memory_permits_byte_range( - size_of::(), - CONTROL_RING_SLOT_SIZE - size_of::() - )); - assert!(!memory_permits_byte_range( - size_of::(), - CONTROL_RING_SLOT_SIZE - size_of::() + 1 - )); + for layout in LAYOUTS { + assert!(layout.permits_u64(0)); + assert!(!layout.permits_u32(0)); + assert!(!layout.permits_byte_range(0, 1)); + assert!( + layout.permits_byte_range( + size_of::(), + CONTROL_RING_SLOT_SIZE - size_of::() + ) + ); + assert!(!layout.permits_byte_range( + size_of::(), + CONTROL_RING_SLOT_SIZE - size_of::() + 1 + )); - let sync_start = CONTROL_RING_DATA_SIZE; - assert!(memory_permits_u32(sync_start + PRODUCER_EPOCH_OFFSET)); - assert!(memory_permits_u32(sync_start + CONSUMER_EPOCH_OFFSET)); - assert!(memory_permits_u64(sync_start + CONSUMER_HEAD_OFFSET)); - assert!(memory_permits_u32(sync_start + CONSUMER_WAITING_OFFSET)); - assert!(memory_permits_u32(sync_start + PRODUCER_WAITING_OFFSET)); - assert!(!memory_permits_u64(sync_start + CONSUMER_WAITING_OFFSET)); - assert!(!memory_permits_u32(sync_start + CONSUMER_HEAD_OFFSET)); - assert!(!memory_permits_byte_range(sync_start, 1)); - assert!(!memory_permits_u32(CONTROL_RING_MEMORY_SIZE)); - assert!(!memory_permits_u64(CONTROL_RING_MEMORY_SIZE)); - assert!(!memory_permits_byte_range(CONTROL_RING_MEMORY_SIZE, 1)); + let sync_start = layout.data_size(); + assert!(!layout.permits_byte_range(sync_start - 1, 2)); + assert!(layout.permits_u32(sync_start + PRODUCER_EPOCH_OFFSET)); + assert!(layout.permits_u32(sync_start + CONSUMER_EPOCH_OFFSET)); + assert!(layout.permits_u64(sync_start + CONSUMER_HEAD_OFFSET)); + assert!(layout.permits_u32(sync_start + CONSUMER_WAITING_OFFSET)); + assert!(layout.permits_u32(sync_start + PRODUCER_WAITING_OFFSET)); + assert!(!layout.permits_u64(sync_start + CONSUMER_WAITING_OFFSET)); + assert!(!layout.permits_u32(sync_start + CONSUMER_HEAD_OFFSET)); + assert!(!layout.permits_byte_range(sync_start, 1)); + let size = layout.memory_size(); + assert!(!layout.permits_u32(size)); + assert!(!layout.permits_u64(size)); + assert!(!layout.permits_byte_range(size, 1)); + assert!(layout.permits_byte_range(size, 0)); + } + + let multi_lane = ControlRingLayout::MULTI_LANE; + let padding = multi_lane.data_size() + SYNC_WORDS_SIZE; + assert!(!multi_lane.permits_u32(padding)); + assert!(!multi_lane.permits_u64(padding)); + let demand = multi_lane.demand_offset(); + assert!(multi_lane.permits_u32(demand + REQUESTED_LANES_OFFSET)); + assert!(multi_lane.permits_u32(demand + DEMAND_EPOCH_OFFSET)); + assert!(!multi_lane.permits_u64(demand)); + assert!(!multi_lane.permits_byte_range(demand, 1)); } #[test] fn directions_occupy_disjoint_data_and_sync_ranges() { - let directions = [ - ControlRingDirection::Requests, - ControlRingDirection::Responses, - ControlRingDirection::Notifications, - ]; - let mut next_data_offset = 0; - for direction in directions { - let first = direction.slot_range(0); - let last = direction.slot_range(direction.slot_count() - 1); - assert_eq!(first.start, next_data_offset); + for layout in LAYOUTS { + let directions: Vec<_> = (0..layout.lane_count()) + .flat_map(|lane| [layout.requests(lane), layout.responses(lane)]) + .chain([layout.notifications()]) + .collect(); + let mut next_data_offset = 0; + for direction in &directions { + let first = direction.slot_range(0); + let last = direction.slot_range(direction.slot_count() - 1); + assert_eq!(first.start, next_data_offset); + assert_eq!( + last.end - first.start, + usize::try_from(direction.slot_count()).unwrap() * CONTROL_RING_SLOT_SIZE + ); + next_data_offset = last.end; + } + assert_eq!(next_data_offset, layout.data_size()); + + for (index, direction) in directions.iter().enumerate() { + assert_eq!( + direction.producer_epoch_offset(), + layout.data_size() + index * layout.sync_stride + ); + assert_eq!( + direction.consumer_epoch_offset(), + direction.producer_epoch_offset() + size_of::() + ); + assert_eq!( + direction.consumer_head_offset(), + direction.producer_epoch_offset() + size_of::() + ); + assert!( + direction + .consumer_head_offset() + .is_multiple_of(align_of::()) + ); + assert_eq!( + direction.consumer_waiting_offset(), + direction.consumer_head_offset() + size_of::() + ); + assert_eq!( + direction.producer_waiting_offset(), + direction.consumer_waiting_offset() + size_of::() + ); + } assert_eq!( - last.end - first.start, - usize::try_from(direction.slot_count()).unwrap() * CONTROL_RING_SLOT_SIZE + layout.notifications().sync_offset + layout.sync_stride, + layout.demand_offset() ); - next_data_offset = last.end; } - assert_eq!(next_data_offset, CONTROL_RING_DATA_SIZE); + assert_eq!( + ControlRingLayout::SINGLE_LANE.demand_offset(), + ControlRingLayout::SINGLE_LANE.memory_size() + ); + assert_eq!( + ControlRingLayout::MULTI_LANE.demand_offset() + DEMAND_WORDS_SIZE, + ControlRingLayout::MULTI_LANE.memory_size() + ); + } - for (index, direction) in directions.into_iter().enumerate() { - assert_eq!( - direction.producer_epoch_offset(), - CONTROL_RING_DATA_SIZE + index * CONTROL_RING_SYNC_DIRECTION_SIZE - ); - assert_eq!( - direction.consumer_epoch_offset(), - direction.producer_epoch_offset() + size_of::() + #[test] + fn waitable_words_are_indexed_densely() { + for layout in LAYOUTS { + let mut indices: Vec<_> = (0..layout.memory_size()) + .filter_map(|offset| layout.waitable_word_index(offset)) + .collect(); + assert_eq!(indices.len(), layout.waitable_word_count()); + indices.sort_unstable(); + indices.dedup(); + assert_eq!(indices.len(), layout.waitable_word_count()); + assert!( + indices + .iter() + .all(|&index| index < layout.waitable_word_count()) ); + for offset in 0..layout.memory_size() { + if layout.waitable_word_index(offset).is_some() { + assert!(layout.permits_u32(offset)); + } + } + let mut wait_offsets: Vec<_> = (0..layout.lane_count()) + .flat_map(|lane| [layout.requests(lane), layout.responses(lane)]) + .chain([layout.notifications()]) + .flat_map(|direction| { + [ + direction.producer_epoch_offset(), + direction.consumer_epoch_offset(), + ] + }) + .collect(); + if layout.lane_count() > 1 { + wait_offsets.push(layout.demand_offset() + DEMAND_EPOCH_OFFSET); + } + assert_eq!(wait_offsets.len(), layout.waitable_word_count()); + for offset in wait_offsets { + assert!(layout.waitable_word_index(offset).is_some()); + } + } + assert_eq!(ControlRingLayout::SINGLE_LANE.waitable_word_count(), 6); + } + + #[test] + fn endpoint_roles_bind_opposite_directions_to_one_mapping() { + for layout in LAYOUTS { + let local = test_ring_with(layout).into_local(); + assert_eq!(local.lanes.len(), layout.lane_count()); + assert_eq!(local.lane_demand.is_some(), layout.lane_count() > 1); + for (index, lane) in local.lanes.iter().enumerate() { + assert_eq!(lane.request_producer.direction(), layout.requests(index)); + assert_eq!(lane.response_consumer.direction(), layout.responses(index)); + assert!(Arc::ptr_eq( + &lane.request_producer.ring, + &local.notification_consumer.ring + )); + assert!(Arc::ptr_eq( + &lane.response_consumer.ring, + &local.notification_consumer.ring + )); + } assert_eq!( - direction.consumer_head_offset(), - direction.producer_epoch_offset() + size_of::() - ); - assert!( - direction - .consumer_head_offset() - .is_multiple_of(align_of::()) + local.notification_consumer.direction(), + layout.notifications() ); + assert_eq!(local.wake_handles().len(), 2 * layout.lane_count() + 1); + + let broker = test_ring_with(layout).into_broker(); + assert_eq!(broker.lanes.len(), layout.lane_count()); + assert_eq!(broker.lane_demand.is_some(), layout.lane_count() > 1); + for (index, lane) in broker.lanes.iter().enumerate() { + assert_eq!(lane.request_consumer.direction(), layout.requests(index)); + assert_eq!(lane.response_producer.direction(), layout.responses(index)); + assert!(Arc::ptr_eq( + &lane.request_consumer.ring, + &broker.notification_producer.ring + )); + assert!(Arc::ptr_eq( + &lane.response_producer.ring, + &broker.notification_producer.ring + )); + } assert_eq!( - direction.consumer_waiting_offset(), - direction.consumer_head_offset() + size_of::() + broker.notification_producer.direction(), + layout.notifications() ); assert_eq!( - direction.producer_waiting_offset(), - direction.consumer_waiting_offset() + size_of::() + broker.wake_handles().len(), + 2 * layout.lane_count() + 1 + usize::from(layout.lane_count() > 1) ); } - assert_eq!( - ControlRingDirection::Notifications.producer_waiting_offset() + size_of::(), - CONTROL_RING_MEMORY_SIZE - ); } #[test] - fn endpoint_roles_bind_opposite_directions_to_one_mapping() { - let local = test_ring().into_local(); - assert_eq!( - local.request_producer.direction(), - ControlRingDirection::Requests - ); - assert_eq!( - local.response_consumer.direction(), - ControlRingDirection::Responses - ); - assert_eq!( - local.notification_consumer.direction(), - ControlRingDirection::Notifications + fn single_slot_lanes_carry_one_call_at_a_time() { + let layout = ControlRingLayout::MULTI_LANE; + let ring = Arc::new(test_ring_with(layout)); + let mut lanes: Vec<_> = (0..layout.lane_count()) + .map(|lane| { + let local = LocalControlRingLane { + request_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }; + let broker = BrokerControlRingLane { + request_consumer: ControlRingConsumer::new( + Arc::clone(&ring), + layout.requests(lane), + ), + response_producer: ControlRingProducer::new( + Arc::clone(&ring), + layout.responses(lane), + ), + }; + (local, broker) + }) + .collect(); + for round in 0..3_u8 { + for (index, (local, broker)) in lanes.iter_mut().enumerate() { + let request = [round, u8::try_from(index).unwrap()]; + assert_eq!( + local.request_producer.try_write(&request), + Ok(ControlRingWriteStatus::Written) + ); + assert!(matches!( + local.request_producer.try_write(&request), + Ok(ControlRingWriteStatus::Full { .. }) + )); + assert_eq!( + broker.request_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Message(request.to_vec())) + ); + assert!(matches!( + broker.request_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Empty { .. }) + )); + broker.request_consumer.publish_head().unwrap(); + let response = [round, u8::try_from(index).unwrap(), 0xff]; + assert_eq!( + broker.response_producer.try_write(&response), + Ok(ControlRingWriteStatus::Written) + ); + assert_eq!( + local.response_consumer.try_read(owned_bytes), + Ok(ControlRingReadStatus::Message(response.to_vec())) + ); + local.response_consumer.publish_head().unwrap(); + } + } + } + + #[test] + fn broker_hands_out_lanes_in_order_as_demand_rises() { + let ring = Arc::new(test_ring_with(ControlRingLayout::MULTI_LANE)); + let local = LocalLaneDemand { + ring: Arc::clone(&ring), + }; + let mut broker = BrokerLaneDemand { + ring: Arc::clone(&ring), + started: 0, + }; + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + let Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) = broker.try_next_lane() else { + panic!("only lane 0 is demanded initially"); + }; + local.request(3).unwrap(); + let epoch_offset = ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET; + assert_ne!( + ring.memory.load_u32_acquire(epoch_offset).unwrap(), + wait_epoch ); - assert!(Arc::ptr_eq( - &local.request_producer.ring, - &local.response_consumer.ring - )); - assert!(Arc::ptr_eq( - &local.request_producer.ring, - &local.notification_consumer.ring + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(1))); + assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(2))); + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); - let broker = test_ring().into_broker(); - assert_eq!( - broker.request_consumer.direction(), - ControlRingDirection::Requests - ); - assert_eq!( - broker.response_producer.direction(), - ControlRingDirection::Responses - ); - assert_eq!( - broker.notification_producer.direction(), - ControlRingDirection::Notifications - ); - assert!(Arc::ptr_eq( - &broker.request_consumer.ring, - &broker.response_producer.ring + // A hostile peer can lower or overstate demand; the broker never + // takes back a lane and never exceeds the layout. + local.request(1).unwrap(); + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); - assert!(Arc::ptr_eq( - &broker.request_consumer.ring, - &broker.notification_producer.ring + ring.memory + .store_u32_release( + ring.layout.demand_offset() + REQUESTED_LANES_OFFSET, + u32::MAX.to_le(), + ) + .unwrap(); + for lane in 3..ControlRingLayout::MULTI_LANE.lane_count() { + assert_eq!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Lane(lane)) + ); + } + assert!(matches!( + broker.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) )); } #[test] fn request_response_and_notification_rings_publish_independently() { let ring = Arc::new(test_ring()); - let mut request_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Requests); - let mut request_consumer = - ControlRingConsumer::new(Arc::clone(&ring), ControlRingDirection::Requests); - let mut response_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Responses); - let mut response_consumer = - ControlRingConsumer::new(Arc::clone(&ring), ControlRingDirection::Responses); - let mut notification_producer = - ControlRingProducer::new(Arc::clone(&ring), ControlRingDirection::Notifications); - let mut notification_consumer = - ControlRingConsumer::new(ring, ControlRingDirection::Notifications); + let mut request_producer = ControlRingProducer::new(Arc::clone(&ring), REQUESTS); + let mut request_consumer = ControlRingConsumer::new(Arc::clone(&ring), REQUESTS); + let mut response_producer = ControlRingProducer::new(Arc::clone(&ring), RESPONSES); + let mut response_consumer = ControlRingConsumer::new(Arc::clone(&ring), RESPONSES); + let mut notification_producer = ControlRingProducer::new(Arc::clone(&ring), NOTIFICATIONS); + let mut notification_consumer = ControlRingConsumer::new(ring, NOTIFICATIONS); assert_eq!( request_producer.try_write(&[1, 2, 3]), @@ -1181,11 +1648,7 @@ mod tests { #[test] fn each_direction_enforces_its_own_capacity() { - for direction in [ - ControlRingDirection::Requests, - ControlRingDirection::Responses, - ControlRingDirection::Notifications, - ] { + for direction in [REQUESTS, RESPONSES, NOTIFICATIONS] { let (mut producer, mut consumer) = test_ring().into_endpoints(direction, direction); for value in 0..direction.slot_count() { assert_eq!( @@ -1232,10 +1695,7 @@ mod tests { CONTROL_RING_SLOT_HEADER_SIZE - size_of::() ), (0, size_of::()), - ( - ControlRingDirection::Requests.producer_epoch_offset(), - size_of::() - ), + (REQUESTS.producer_epoch_offset(), size_of::()), ] ); } @@ -1243,11 +1703,8 @@ mod tests { #[test] fn failed_payload_metadata_or_word_publication_does_not_publish_progress() { for failed_write in [1, 2, 3] { - let ring = ControlRing::new(FailingWriteMemory::new()).unwrap(); - let (mut producer, mut consumer) = ring.into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(FailingWriteMemory::new(), LAYOUT).unwrap(); + let (mut producer, mut consumer) = ring.into_endpoints(REQUESTS, REQUESTS); producer.try_write(&[7]).unwrap(); producer.memory().fail_after(failed_write); assert_eq!( @@ -1282,14 +1739,14 @@ mod tests { let (mut producer, mut consumer) = test_endpoints(); producer.try_write(&[7; 100]).unwrap(); - for _ in 1..CONTROL_RING_SLOT_COUNT { + for _ in 1..SLOT_COUNT { producer.try_write(&[8]).unwrap(); } assert_eq!( consumer.try_read(|payload| Ok::<_, ()>(payload.len())), Ok(ControlRingReadStatus::Message(100)) ); - for _ in 1..CONTROL_RING_SLOT_COUNT { + for _ in 1..SLOT_COUNT { consumer.try_read(owned_bytes).unwrap(); } consumer.publish_head().unwrap(); @@ -1310,7 +1767,7 @@ mod tests { fn shared_epochs_publish_and_acknowledge_a_full_batch() { let (mut producer, mut consumer) = test_endpoints(); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..SLOT_COUNT { let value = u8::try_from(value).unwrap(); assert_eq!( producer.try_write(&[value]), @@ -1322,7 +1779,7 @@ mod tests { Ok(ControlRingWriteStatus::Full { wait_epoch: 0 }) ); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..SLOT_COUNT { let value = u8::try_from(value).unwrap(); assert_eq!( consumer.try_read(owned_bytes), @@ -1332,7 +1789,7 @@ mod tests { assert_eq!( consumer.try_read(owned_bytes), Ok(ControlRingReadStatus::Empty { - wait_epoch: u32::try_from(CONTROL_RING_SLOT_COUNT).unwrap() + wait_epoch: u32::try_from(SLOT_COUNT).unwrap() }) ); assert_eq!( @@ -1355,10 +1812,9 @@ mod tests { fn wakeup_epochs_wrap_without_controlling_ring_progress() { let (mut producer, mut consumer) = test_endpoints(); - producer.memory().store_u32_for_test( - ControlRingDirection::Requests.producer_epoch_offset(), - u32::MAX, - ); + producer + .memory() + .store_u32_for_test(REQUESTS.producer_epoch_offset(), u32::MAX); producer.try_write(&[7]).unwrap(); assert_eq!( consumer.try_read(owned_bytes), @@ -1369,15 +1825,14 @@ mod tests { Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) ); - consumer.memory().store_u32_for_test( - ControlRingDirection::Requests.consumer_epoch_offset(), - u32::MAX, - ); + consumer + .memory() + .store_u32_for_test(REQUESTS.consumer_epoch_offset(), u32::MAX); consumer.publish_head().unwrap(); assert_eq!( producer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(0) ); assert_eq!(producer.refresh_head(), Ok(())); @@ -1390,7 +1845,7 @@ mod tests { assert_eq!( consumer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(0) ); @@ -1401,7 +1856,7 @@ mod tests { assert_eq!( consumer .memory() - .load_u32_acquire(ControlRingDirection::Requests.consumer_epoch_offset()), + .load_u32_acquire(REQUESTS.consumer_epoch_offset()), Ok(1) ); } @@ -1438,10 +1893,7 @@ mod tests { fn assert_rejected(image: &[u8; CONTROL_RING_SLOT_SIZE], expected: ControlRingError) { let ring = test_ring(); install_slot(ring.memory(), image); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), Err(ControlRingReadError::Ring(expected)) @@ -1480,7 +1932,7 @@ mod tests { Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) ); install_slot(consumer.memory(), &raw_slot(1, 1, 0, &[7])); - consumer.head = CONTROL_RING_SLOT_COUNT; + consumer.head = SLOT_COUNT; assert_eq!( consumer.try_read(owned_bytes), Ok(ControlRingReadStatus::Empty { wait_epoch: 0 }) @@ -1557,11 +2009,8 @@ mod tests { #[test] fn torn_length_snapshot_is_rejected_before_payload_slicing() { let memory = TearingMemory::with_torn_length(&raw_slot(1, 1, 0, &[7])); - let ring = ControlRing::new(memory).unwrap(); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), @@ -1577,11 +2026,8 @@ mod tests { #[test] fn sequence_change_during_body_copy_is_rejected() { let memory = TearingMemory::with_changed_sequence(&raw_slot(1, 1, 0, &[7])); - let ring = ControlRing::new(memory).unwrap(); - let (_, mut consumer) = ring.into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); + let (_, mut consumer) = ring.into_endpoints(RESPONSES, REQUESTS); assert_eq!( consumer.try_read(owned_bytes), @@ -1630,17 +2076,18 @@ mod tests { } fn test_ring() -> ControlRing { - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)).unwrap() + test_ring_with(LAYOUT) + } + + fn test_ring_with(layout: ControlRingLayout) -> ControlRing { + ControlRing::new(TestMemory::new(layout.memory_size()), layout).unwrap() } fn test_endpoints() -> ( ControlRingProducer, ControlRingConsumer, ) { - test_ring().into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ) + test_ring().into_endpoints(REQUESTS, REQUESTS) } fn owned_bytes(payload: &[u8]) -> Result, ()> { @@ -1653,14 +2100,10 @@ mod tests { #[test] fn wakes_skip_peers_that_announce_no_wait() { - let (mut producer, mut consumer) = - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)) - .unwrap() - .into_endpoints( - ControlRingDirection::Requests, - ControlRingDirection::Requests, - ); - let direction = ControlRingDirection::Requests; + let (mut producer, mut consumer) = ControlRing::new(TestMemory::new(MEMORY_SIZE), LAYOUT) + .unwrap() + .into_endpoints(REQUESTS, REQUESTS); + let direction = REQUESTS; assert_eq!( producer.try_write(b"x"), Ok(ControlRingWriteStatus::Written) @@ -1696,15 +2139,11 @@ mod tests { #[test] fn waits_announce_themselves_only_while_blocking() { - let (mut producer, mut consumer) = - ControlRing::new(TestMemory::new(CONTROL_RING_MEMORY_SIZE)) - .unwrap() - .into_endpoints( - ControlRingDirection::Responses, - ControlRingDirection::Requests, - ); - let requests = ControlRingDirection::Requests; - let responses = ControlRingDirection::Responses; + let (mut producer, mut consumer) = ControlRing::new(TestMemory::new(MEMORY_SIZE), LAYOUT) + .unwrap() + .into_endpoints(RESPONSES, REQUESTS); + let requests = REQUESTS; + let responses = RESPONSES; let Ok(ControlRingReadStatus::Empty { wait_epoch }) = consumer.try_read(|frame| Ok::<_, ()>(frame.to_vec())) else { @@ -1763,10 +2202,7 @@ mod tests { fn store_test_consumer_head(memory: &TestMemory, head: u64) { memory - .store_u64_release( - ControlRingDirection::Requests.consumer_head_offset(), - head.to_le(), - ) + .store_u64_release(REQUESTS.consumer_head_offset(), head.to_le()) .unwrap(); } @@ -1937,7 +2373,7 @@ mod tests { impl FailingWriteMemory { fn new() -> Self { Self { - bytes: Mutex::new(vec![0; CONTROL_RING_MEMORY_SIZE]), + bytes: Mutex::new(vec![0; MEMORY_SIZE]), write_count: AtomicUsize::new(0), fail_on_write: AtomicUsize::new(usize::MAX), } @@ -2054,7 +2490,7 @@ mod tests { } fn new(first_slot: &[u8; CONTROL_RING_SLOT_SIZE], tear: Tear) -> Self { - let mut bytes = vec![0; CONTROL_RING_MEMORY_SIZE]; + let mut bytes = vec![0; MEMORY_SIZE]; bytes[..CONTROL_RING_SLOT_SIZE].copy_from_slice(first_slot); Self { bytes: Mutex::new(bytes), diff --git a/litebox_broker_transport/src/lib.rs b/litebox_broker_transport/src/lib.rs index a103f59a2d..2d0259470f 100644 --- a/litebox_broker_transport/src/lib.rs +++ b/litebox_broker_transport/src/lib.rs @@ -28,8 +28,8 @@ extern crate alloc; #[cfg(test)] extern crate std; +pub mod call_lane_pool; pub mod channel; pub mod control_ring; -pub mod pending_calls; pub mod setup_frame; pub mod shared_memory; diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs deleted file mode 100644 index e1ea50b1d6..0000000000 --- a/litebox_broker_transport/src/pending_calls.rs +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -//! Portable pending-call coordination over caller-supplied synchronization. - -use alloc::collections::BTreeMap; -use alloc::collections::btree_map::Entry; -use alloc::sync::Arc; -use core::ops::{Deref, DerefMut}; - -use litebox_broker_protocol::RequestId; -use litebox_broker_protocol::message::BrokerResponse; - -/// Maximum number of active calls waiting for broker responses. -pub const MAX_PENDING_CALLS: usize = 64; - -/// A mutex usable by [`PendingCalls`]. -pub trait PendingCallsMutex { - /// Guard granting access to the protected value. - type Guard<'a>: Deref + DerefMut - where - Self: 'a, - T: 'a; - - /// Locks the mutex. - fn lock(&self) -> Self::Guard<'_>; -} - -/// A condition variable paired with a [`PendingCallsMutex`]. -pub trait PendingCallsCondvar { - /// Mutex type whose guard this condition variable waits on. - type Mutex: PendingCallsMutex; - - /// Atomically releases the guard and waits for a notification. - fn wait<'a>( - &self, - guard: >::Guard<'a>, - ) -> >::Guard<'a> - where - Self::Mutex: 'a, - T: 'a; - - /// Wakes one waiter. - fn notify_one(&self); - - /// Wakes all waiters. - fn notify_all(&self); -} - -/// Supplies synchronization primitives for [`PendingCalls`]. -pub trait PendingCallsSync { - /// Mutex protecting a value of type `T`. - type Mutex: PendingCallsMutex; - - /// Condition variable paired with [`Self::Mutex`]. - type Condvar: PendingCallsCondvar>; - - /// Creates a mutex protecting `value`. - fn mutex(value: T) -> Self::Mutex; - - /// Creates a condition variable. - fn condvar() -> Self::Condvar; -} - -/// Failure from pending-call registry coordination. -#[derive(Debug)] -pub enum PendingCallsError { - /// The association had already failed. - AssociationFailed(Arc), - /// The guarded operation failed. - Operation(Error), - /// A request reused an active request identifier. - DuplicateRequestId, - /// A response did not identify an active request. - UnknownResponseId, -} - -/// Concurrent registry of requests awaiting broker responses. -/// -/// Callers waiting for responses also read them: while any call is pending, -/// one waiting caller holds the reader role and completes responses for every -/// caller until its own arrives, then hands the role to another pending call. -/// A response therefore reaches a lone caller without a thread handoff. -pub struct PendingCalls { - state: Sync::Mutex>, - capacity_available: Sync::Condvar>, -} - -struct PendingCallsInner { - calls: BTreeMap>>, - failure: Option>, - /// The call whose caller holds the reader role, if any call is pending. - reader: Option>>, -} - -/// Completion state shared between a pending call and its registry. -struct CallSlot { - state: Sync::Mutex>, - changed: Sync::Condvar>, -} - -struct CallState { - result: Option>>, - /// Whether this call's caller holds the reader role. - reads_responses: bool, -} - -impl CallSlot { - fn new(reads_responses: bool) -> Self { - Self { - state: Sync::mutex(CallState { - result: None, - reads_responses, - }), - changed: Sync::condvar(), - } - } - - fn resolve(&self, result: Result>, notify: bool) { - let mut state = self.state.lock(); - assert!( - state.result.is_none(), - "broker pending call already resolved" - ); - state.result = Some(result); - if notify { - self.changed.notify_one(); - } - } - - fn grant_reader_role(&self) { - self.state.lock().reads_responses = true; - self.changed.notify_one(); - } -} - -/// One registered request awaiting its broker response. -/// -/// Dropping the call unregisters it and hands off the reader role if it holds -/// it, so an abandoned call never strands other callers. -pub struct PendingCall<'calls, Sync: PendingCallsSync, Error> { - pending_calls: &'calls PendingCalls, - request_id: RequestId, - slot: Arc>, -} - -impl PendingCall<'_, Sync, Error> { - /// Blocks until the broker responds or the association fails. - /// - /// While this caller holds the reader role, it calls `read_responses` - /// repeatedly. Each call must block until it reads at least one response - /// and passes it to [`PendingCalls::complete`], or until the association - /// fails and the failure is recorded with [`PendingCalls::record_failure`]. - /// Only the caller holding the reader role calls `read_responses`. - pub fn wait(self, mut read_responses: impl FnMut()) -> Result> { - loop { - { - let mut state = self.slot.state.lock(); - loop { - if let Some(result) = state.result.take() { - return result; - } - if state.reads_responses { - break; - } - state = self.slot.changed.wait(state); - } - } - read_responses(); - } - } -} - -impl Drop for PendingCall<'_, Sync, Error> { - fn drop(&mut self) { - self.pending_calls.unregister(self.request_id, &self.slot); - } -} - -impl PendingCalls { - /// Creates an empty live pending-call registry. - pub fn new() -> Self { - Self { - state: Sync::mutex(PendingCallsInner { - calls: BTreeMap::new(), - failure: None, - reader: None, - }), - capacity_available: Sync::condvar(), - } - } - - /// Registers a request, blocking while the pending-call limit is full. - /// - /// The first call registered while no call is pending takes the reader - /// role. - pub fn register( - &self, - request_id: RequestId, - ) -> Result, PendingCallsError> { - let mut state = self.state.lock(); - while state.calls.len() >= MAX_PENDING_CALLS && state.failure.is_none() { - state = self.capacity_available.wait(state); - } - let state = &mut *state; - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - let Entry::Vacant(entry) = state.calls.entry(request_id) else { - return Err(PendingCallsError::DuplicateRequestId); - }; - let slot = Arc::new(CallSlot::new(state.reader.is_none())); - entry.insert(Arc::clone(&slot)); - if state.reader.is_none() { - state.reader = Some(Arc::clone(&slot)); - } - Ok(PendingCall { - pending_calls: self, - request_id, - slot, - }) - } - - /// Completes the pending call identified by `response`. - pub fn complete(&self, response: BrokerResponse) -> Result<(), PendingCallsError> { - let (slot, notify) = { - let mut state = self.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - let Some(slot) = state.calls.remove(&response.request_id) else { - return Err(PendingCallsError::UnknownResponseId); - }; - // Registrations wait only while the registry is full. - if state.calls.len() + 1 == MAX_PENDING_CALLS { - self.capacity_available.notify_all(); - } - // The reader completes its own call while reading, not waiting. - let notify = !state - .reader - .as_ref() - .is_some_and(|reader| Arc::ptr_eq(reader, &slot)); - (slot, notify) - }; - slot.resolve(Ok(response), notify); - Ok(()) - } - - /// Removes `slot` if it is still registered and passes on its reader role. - fn unregister(&self, request_id: RequestId, slot: &Arc>) { - let mut state = self.state.lock(); - if state - .calls - .get(&request_id) - .is_some_and(|registered| Arc::ptr_eq(registered, slot)) - { - state.calls.remove(&request_id); - if state.calls.len() + 1 == MAX_PENDING_CALLS { - self.capacity_available.notify_all(); - } - } - let holds_reader_role = state - .reader - .as_ref() - .is_some_and(|reader| Arc::ptr_eq(reader, slot)); - if holds_reader_role && state.failure.is_none() { - state.reader = state.calls.values().next().map(|next| { - next.grant_reader_role(); - Arc::clone(next) - }); - } - } - - /// Records the first terminal failure and resolves every pending call. - /// - /// Returns whether this call recorded the first failure. - pub fn record_failure(&self, error: Arc) -> bool { - let pending_calls = { - let mut state = self.state.lock(); - if state.failure.is_some() { - return false; - } - state.failure = Some(Arc::clone(&error)); - let pending_calls = core::mem::take(&mut state.calls); - self.capacity_available.notify_all(); - pending_calls - }; - for pending_call in pending_calls.into_values() { - pending_call.resolve(Err(Arc::clone(&error)), true); - } - true - } - - /// Returns the association's terminal failure, if one was recorded. - pub fn current_failure(&self) -> Option> { - self.state.lock().failure.as_ref().map(Arc::clone) - } - - /// Runs an operation while excluding failure recording. - pub fn run_if_live( - &self, - operation: impl FnOnce() -> Result, - ) -> Result> { - let state = self.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); - } - operation().map_err(PendingCallsError::Operation) - } -} - -impl Default for PendingCalls { - fn default() -> Self { - Self::new() - } -} diff --git a/litebox_broker_transport_linux_userland/src/call_lane_pool.rs b/litebox_broker_transport_linux_userland/src/call_lane_pool.rs new file mode 100644 index 0000000000..f10d1108fe --- /dev/null +++ b/litebox_broker_transport_linux_userland/src/call_lane_pool.rs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +use std::io::Error; +use std::marker::PhantomData; +use std::sync::{Condvar, Mutex, MutexGuard}; + +use litebox_broker_transport::call_lane_pool::{ + CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, + CallLanePoolSync, +}; + +use crate::setup::copy_io_error; + +pub(crate) type CallLanePool = GenericCallLanePool; + +pub(crate) struct StdCallLanePoolSync; + +pub(crate) struct StdCallLanePoolMutex(Mutex); + +pub(crate) struct StdCallLanePoolCondvar(Condvar, PhantomData); + +impl CallLanePoolMutex for StdCallLanePoolMutex { + type Guard<'a> + = MutexGuard<'a, T> + where + T: 'a; + + fn lock(&self) -> Self::Guard<'_> { + self.0.lock().expect("broker call-lanes mutex poisoned") + } +} + +impl CallLanePoolCondvar for StdCallLanePoolCondvar { + type Mutex = StdCallLanePoolMutex; + + fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> + where + Self::Mutex: 'a, + T: 'a, + { + self.0 + .wait(guard) + .expect("broker call-lanes mutex poisoned") + } + + fn notify_one(&self) { + self.0.notify_one(); + } + + fn notify_all(&self) { + self.0.notify_all(); + } +} + +impl CallLanePoolSync for StdCallLanePoolSync { + type Mutex = StdCallLanePoolMutex; + type Condvar = StdCallLanePoolCondvar; + + fn mutex(value: T) -> Self::Mutex { + StdCallLanePoolMutex(Mutex::new(value)) + } + + fn condvar() -> Self::Condvar { + StdCallLanePoolCondvar(Condvar::new(), PhantomData) + } +} + +pub(crate) fn call_lane_pool_error(error: CallLanePoolError) -> Error { + match error { + CallLanePoolError::AssociationFailed(error) => copy_io_error(&error), + CallLanePoolError::Operation(error) => error, + } +} + +#[cfg(test)] +mod tests { + use std::io::{Error, ErrorKind}; + use std::sync::Arc; + use std::sync::mpsc; + use std::time::Duration; + + use litebox_broker_transport::call_lane_pool::CallLanePoolError; + + use super::CallLanePool; + + fn no_demand(_: usize) -> Result<(), Error> { + unreachable!("lane 0 needs no demand") + } + + #[test] + fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { + let lanes = CallLanePool::new(vec![0, 1, 2]); + let mut demands = Vec::new(); + let first = lanes.claim(no_demand).unwrap(); + let second = lanes + .claim(|count| { + demands.push(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*first, *second), (0, 1)); + drop(first); + let reused = lanes.claim(no_demand).unwrap(); + assert_eq!(*reused, 0); + drop((reused, second)); + let again = lanes.claim(no_demand).unwrap(); + let next = lanes + .claim(|_| panic!("lane 1 is already demanded")) + .unwrap(); + assert_eq!((*again, *next), (0, 1)); + assert_eq!(demands, [2]); + } + + #[test] + fn a_failed_demand_leaves_the_lane_free_and_undemanded() { + let lanes = CallLanePool::new(vec![0, 1]); + let _first = lanes.claim(no_demand).unwrap(); + assert!(matches!( + lanes.claim(|_| Err(Error::other("demand failed"))), + Err(CallLanePoolError::Operation(_)) + )); + let mut demanded = None; + let second = lanes + .claim(|count| { + demanded = Some(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*second, demanded), (1, Some(2))); + } + + #[test] + fn claims_wait_while_every_lane_is_held() { + let lanes = CallLanePool::new(vec![0]); + let held = lanes.claim(no_demand).unwrap(); + let (claimed, claim) = mpsc::channel(); + std::thread::scope(|scope| { + scope.spawn(|| { + let lane = lanes.claim(no_demand).map(|lane| *lane); + claimed.send(lane.is_ok()).unwrap(); + }); + assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); + drop(held); + assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); + }); + } + + #[test] + fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { + let lanes = CallLanePool::new(vec![0]); + let _held = lanes.claim(no_demand).unwrap(); + std::thread::scope(|scope| { + let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); + std::thread::sleep(Duration::from_millis(10)); + assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); + assert!(matches!( + waiter.join().unwrap(), + Err(CallLanePoolError::AssociationFailed(_)) + )); + }); + assert!(!lanes.record_failure(Arc::new(Error::from(ErrorKind::Other)))); + assert_eq!( + lanes.current_failure().unwrap().kind(), + ErrorKind::BrokenPipe + ); + assert!(matches!( + lanes.run_if_live(|| Ok(())), + Err(CallLanePoolError::AssociationFailed(_)) + )); + } +} diff --git a/litebox_broker_transport_linux_userland/src/lib.rs b/litebox_broker_transport_linux_userland/src/lib.rs index abe11c89e0..e8a9a36eae 100644 --- a/litebox_broker_transport_linux_userland/src/lib.rs +++ b/litebox_broker_transport_linux_userland/src/lib.rs @@ -19,7 +19,7 @@ #![cfg(target_os = "linux")] -mod pending_calls; +mod call_lane_pool; mod setup; pub mod existing_runner; diff --git a/litebox_broker_transport_linux_userland/src/memfd.rs b/litebox_broker_transport_linux_userland/src/memfd.rs index 43816d9094..7b37bc3334 100644 --- a/litebox_broker_transport_linux_userland/src/memfd.rs +++ b/litebox_broker_transport_linux_userland/src/memfd.rs @@ -41,12 +41,7 @@ use rustix::net::{ }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_PARALLEL_SPIN_LIMIT, MemoryAccessPolicy, - WaitableSharedMemory, -}; -#[cfg(test)] -use litebox_broker_transport::control_ring::{ - memory_permits_byte_range, memory_permits_u32, memory_permits_u64, + CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingLayout, MemoryAccessPolicy, WaitableSharedMemory, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_platform::buffer::RawBytes; @@ -207,9 +202,12 @@ impl MemfdSharedMemory { Self::create_with_policy(length, MemoryAccessPolicy::Bytes) } - /// Creates and maps a sealed memfd for one control ring. - pub fn create_control_ring() -> IoResult { - Self::create_with_policy(CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy::ControlRing) + /// Creates and maps a sealed memfd for one control ring with `layout`. + pub fn create_control_ring(layout: ControlRingLayout) -> IoResult { + Self::create_with_policy( + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), + ) } fn create_with_policy(length: usize, policy: MemoryAccessPolicy) -> IoResult { @@ -238,12 +236,12 @@ impl MemfdSharedMemory { Self::from_received_fd_with_policy(fd, expected_length, MemoryAccessPolicy::Bytes) } - /// Maps a control-ring memfd received from the broker. - pub fn control_ring_from_received_fd(fd: OwnedFd) -> IoResult { + /// Maps a control-ring memfd with `layout` received from the broker. + pub fn control_ring_from_received_fd(fd: OwnedFd, layout: ControlRingLayout) -> IoResult { Self::from_received_fd_with_policy( fd, - CONTROL_RING_MEMORY_SIZE, - MemoryAccessPolicy::ControlRing, + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), ) } @@ -288,7 +286,7 @@ impl MemfdSharedMemory { unsafe { madvise(address.as_ptr().cast(), length, Advice::LinuxDontFork) }?; let spin_limit = match policy { MemoryAccessPolicy::Bytes => 0, - MemoryAccessPolicy::ControlRing => parallel_spin_limit(), + MemoryAccessPolicy::ControlRing(_) => parallel_spin_limit(), }; Ok(Self { fd, @@ -470,13 +468,14 @@ pub fn receive_memfd( MemfdSharedMemory::from_received_fd(fd, expected_length) } -/// Receives and maps one control-ring memfd from the broker. +/// Receives and maps one control-ring memfd with `layout` from the broker. pub fn receive_control_ring_memfd( stream: &mut UnixStream, + layout: ControlRingLayout, deadline: Option, ) -> IoResult { let fd = with_read_deadline(stream, deadline, receive_fd)?; - MemfdSharedMemory::control_ring_from_received_fd(fd) + MemfdSharedMemory::control_ring_from_received_fd(fd, layout) } /// Sparse memfd holding a process memory image for a runner being started. @@ -734,8 +733,7 @@ mod tests { SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE, SharedBufferSlotIndex, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_SLOT_COUNT, ControlRing, ControlRingReadStatus, - ControlRingWriteStatus, + ControlRing, ControlRingReadStatus, ControlRingWriteStatus, }; use litebox_broker_transport::shared_memory::SharedBufferPool; use rustix::io::FdFlags; @@ -744,6 +742,9 @@ mod tests { use std::thread; use std::time::Duration; + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + const MEMORY_SIZE: usize = LAYOUT.memory_size(); + #[test] fn mappings_share_bytes_and_validate_ranges() { let first = MemfdSharedMemory::create(64).unwrap(); @@ -796,19 +797,20 @@ mod tests { #[test] fn mappings_enforce_control_ring_typed_access() { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let sequence_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u64(*offset)) + let sequence_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u64(*offset)) .unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); - let body_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_byte_range(*offset, 1)) + let body_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_byte_range(*offset, 1)) .unwrap(); first @@ -866,7 +868,7 @@ mod tests { Err(SharedMemoryError::InvalidRange) ); assert_eq!( - second.load_u64_acquire(CONTROL_RING_MEMORY_SIZE), + second.load_u64_acquire(MEMORY_SIZE), Err(SharedMemoryError::InvalidRange) ); assert_eq!( @@ -884,9 +886,9 @@ mod tests { #[test] fn futex_wait_returns_without_peer_cooperation() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); let start = Instant::now(); @@ -897,13 +899,13 @@ mod tests { #[test] fn peer_descriptor_writes_are_read_as_untrusted_snapshots() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let peer_fd = memory.as_fd().try_clone_to_owned().unwrap(); - let sequence_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u64(*offset)) + let sequence_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u64(*offset)) .unwrap(); - let epoch_offset = (0..CONTROL_RING_MEMORY_SIZE) - .find(|offset| memory_permits_u32(*offset)) + let epoch_offset = (0..MEMORY_SIZE) + .find(|offset| LAYOUT.permits_u32(*offset)) .unwrap(); let mut expected_sequence = [0; size_of::()]; @@ -983,9 +985,10 @@ mod tests { let received_pool = MemfdSharedMemory::from_received_fd(pool.fd.as_fd().try_clone_to_owned().unwrap(), 64) .unwrap(); - let ring = MemfdSharedMemory::create_control_ring().unwrap(); + let ring = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let received_ring = MemfdSharedMemory::control_ring_from_received_fd( ring.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); for memory in [&pool, &received_pool, &ring, &received_ring] { @@ -1052,13 +1055,13 @@ mod tests { #[test] fn transfers_exact_sealed_control_ring_mapping() { - let memory = MemfdSharedMemory::create_control_ring().unwrap(); - let ring = ControlRing::new(memory).unwrap(); + let memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let ring = ControlRing::new(memory, LAYOUT).unwrap(); let (mut receiver, mut sender) = UnixStream::pair().unwrap(); send_memfd(&mut sender, ring.memory(), None).unwrap(); - let mapped = receive_control_ring_memfd(&mut receiver, None).unwrap(); - let mapped_ring = ControlRing::new(mapped).unwrap(); + let mapped = receive_control_ring_memfd(&mut receiver, LAYOUT, None).unwrap(); + let mapped_ring = ControlRing::new(mapped, LAYOUT).unwrap(); ring.memory().write(13, &[1, 2, 3]).unwrap(); let mut bytes = [0; 3]; mapped_ring.memory().read(13, &mut bytes).unwrap(); @@ -1073,18 +1076,23 @@ mod tests { #[test] fn shared_futex_wakeup_prevents_missed_cross_mapping_work() { - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let local_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.fd.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let mut producer = ControlRing::new(local_memory) + let mut producer = ControlRing::new(local_memory, LAYOUT) .unwrap() .into_local() + .lanes + .swap_remove(0) .request_producer; - let mut consumer = ControlRing::new(broker_memory) + let mut consumer = ControlRing::new(broker_memory, LAYOUT) .unwrap() .into_broker() + .lanes + .swap_remove(0) .request_consumer; let empty_checked = Arc::new(Barrier::new(2)); let broker_empty_checked = Arc::clone(&empty_checked); @@ -1100,7 +1108,7 @@ mod tests { }; broker_empty_checked.wait(); consumer.wait_for_message(producer_epoch).unwrap(); - for expected in 0..CONTROL_RING_SLOT_COUNT { + for expected in 0..LAYOUT.requests(0).slot_count() { let expected = u8::try_from(expected).unwrap(); loop { match consumer @@ -1136,7 +1144,7 @@ mod tests { }); empty_checked.wait(); - for value in 0..CONTROL_RING_SLOT_COUNT { + for value in 0..LAYOUT.requests(0).slot_count() { let value = u8::try_from(value).unwrap(); assert_eq!( producer.try_write(&[value]), @@ -1250,7 +1258,7 @@ mod tests { .kind(), ErrorKind::InvalidInput ); - let control = MemfdSharedMemory::create_control_ring().unwrap(); + let control = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); assert_eq!( image .write_from_shared(0, &control, 0..1) diff --git a/litebox_broker_transport_linux_userland/src/pending_calls.rs b/litebox_broker_transport_linux_userland/src/pending_calls.rs deleted file mode 100644 index 4d092f09c1..0000000000 --- a/litebox_broker_transport_linux_userland/src/pending_calls.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::pending_calls::{ - PendingCalls as GenericPendingCalls, PendingCallsCondvar, PendingCallsError, PendingCallsMutex, - PendingCallsSync, -}; - -use crate::setup::{copy_io_error, invalid_data}; - -pub(crate) type PendingCalls = GenericPendingCalls; - -pub(crate) struct StdPendingCallsSync; - -pub(crate) struct StdPendingCallsMutex(Mutex); - -pub(crate) struct StdPendingCallsCondvar(Condvar, PhantomData); - -impl PendingCallsMutex for StdPendingCallsMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker pending mutex poisoned") - } -} - -impl PendingCallsCondvar for StdPendingCallsCondvar { - type Mutex = StdPendingCallsMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0.wait(guard).expect("broker pending mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl PendingCallsSync for StdPendingCallsSync { - type Mutex = StdPendingCallsMutex; - type Condvar = StdPendingCallsCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdPendingCallsMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdPendingCallsCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn pending_calls_error(error: PendingCallsError) -> Error { - match error { - PendingCallsError::AssociationFailed(error) => copy_io_error(&error), - PendingCallsError::Operation(error) => error, - PendingCallsError::DuplicateRequestId => invalid_data("duplicate broker request ID"), - PendingCallsError::UnknownResponseId => { - invalid_data("broker returned an unknown response ID") - } - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc; - use std::time::Duration; - - use litebox_broker_protocol::RequestId; - use litebox_broker_protocol::message::{BrokerResponse, BrokerResult}; - use litebox_broker_transport::pending_calls::{MAX_PENDING_CALLS, PendingCallsError}; - - use super::PendingCalls; - - fn response(id: u64) -> BrokerResponse { - BrokerResponse { - request_id: RequestId(id), - result: BrokerResult::ObjectClosed, - } - } - - #[test] - fn a_full_registry_wakes_registration_when_a_call_completes() { - let pending_calls = PendingCalls::new(); - let mut calls: Vec<_> = (0..MAX_PENDING_CALLS as u64) - .map(|id| pending_calls.register(RequestId(id)).unwrap()) - .collect(); - let (registered, registration) = mpsc::channel(); - std::thread::scope(|scope| { - scope.spawn(|| { - let call = pending_calls.register(RequestId(MAX_PENDING_CALLS as u64)); - registered.send(call.is_ok()).unwrap(); - }); - assert!( - registration - .recv_timeout(Duration::from_millis(50)) - .is_err() - ); - pending_calls.complete(response(0)).unwrap(); - assert!(registration.recv_timeout(Duration::from_secs(10)).unwrap()); - }); - assert!(calls.remove(0).wait(|| unreachable!()).is_ok()); - } - - #[test] - fn a_finished_reader_hands_its_role_to_a_pending_call() { - let pending_calls = PendingCalls::new(); - let reader = pending_calls.register(RequestId(1)).unwrap(); - let follower = pending_calls.register(RequestId(2)).unwrap(); - let follower_read = AtomicBool::new(false); - std::thread::scope(|scope| { - let follower = scope.spawn(|| { - follower.wait(|| { - follower_read.store(true, Ordering::Relaxed); - pending_calls.complete(response(2)).unwrap(); - }) - }); - let result = reader.wait(|| pending_calls.complete(response(1)).unwrap()); - assert_eq!(result.unwrap().request_id, RequestId(1)); - assert_eq!(follower.join().unwrap().unwrap().request_id, RequestId(2)); - }); - assert!(follower_read.load(Ordering::Relaxed)); - } - - #[test] - fn an_abandoned_reader_hands_off_its_role() { - let pending_calls = PendingCalls::new(); - let reader = pending_calls.register(RequestId(1)).unwrap(); - let follower = pending_calls.register(RequestId(2)).unwrap(); - drop(reader); - let result = follower.wait(|| pending_calls.complete(response(2)).unwrap()); - assert_eq!(result.unwrap().request_id, RequestId(2)); - assert!(matches!( - pending_calls.complete(response(1)), - Err(PendingCallsError::UnknownResponseId) - )); - } -} diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 20c6764883..cc1dde42cd 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -24,12 +24,13 @@ use litebox_broker_protocol::wire::{ encode_notification, encode_response, }; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, PeerCredential, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + BrokerControlRingEndpoints, BrokerControlRingLane, BrokerLaneDemand, ControlRing, + ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, }; use crate::memfd::{MemfdProcessImage, MemfdSharedMemory}; @@ -99,16 +100,20 @@ pub struct UnixStreamHostSetupChannel { negotiated: bool, } -/// Request-reading endpoint of an active host control-ring association. -pub struct UnixControlRingHostRequestSource { - consumer: ControlRingConsumer, +type BrokerLane = BrokerControlRingLane; + +/// Source of the call lanes of an active host control-ring association, which +/// hands out each lane once the runner starts using it. +pub struct UnixControlRingHostCallLaneSource { + /// Each lane, or `None` once handed out. + lanes: Vec>, + demand: BrokerLaneDemand, association: Arc, } -/// Shared response-writing endpoint of an active host control-ring association. -#[derive(Clone)] -pub struct UnixControlRingHostResponseSink { - producer: Arc>>, +/// One call lane of an active host control-ring association. +pub struct UnixControlRingHostCallLane { + lane: BrokerLane, association: Arc, } @@ -119,13 +124,11 @@ pub struct UnixControlRingHostShutdown { /// State shared by every activated host endpoint of one association: the setup /// socket used for liveness and teardown, terminal status, and the wake handles -/// of all three ring directions. +/// of every ring wait. struct HostRingAssociation { control_stream: UnixStream, status: Mutex, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } enum HostAssociationStatus { @@ -182,13 +185,14 @@ impl UnixStreamHostSetupChannel { } /// Consumes a negotiated setup channel into independently usable active - /// request, response, notification, and shutdown handles. + /// call-lane, notification, and shutdown handles. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( mut self, ring: ControlRing, ) -> IoResult<( - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, )> { @@ -197,43 +201,47 @@ impl UnixStreamHostSetupChannel { "broker host setup channel activated before negotiation completed", )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_broker(); + let wakes = endpoints.wake_handles(); + let BrokerControlRingEndpoints { + lanes, + lane_demand: Some(demand), + notification_producer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let Some(ready) = read_setup_frame(&mut self.stream, self.setup_deadline)? else { return Err(Error::new( ErrorKind::UnexpectedEof, "runner closed before control-ring setup acknowledgement", )); }; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "runner sent an invalid control-ring setup acknowledgement", )); } - write_setup_frame(&mut self.stream, CONTROL_RING_READY, self.setup_deadline)?; + write_setup_frame(&mut self.stream, ready_token, self.setup_deadline)?; let shutdown_stream = self.stream.try_clone()?; - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer, - } = ring.into_broker(); let association = Arc::new(HostRingAssociation { control_stream: shutdown_stream, status: Mutex::new(HostAssociationStatus::Live), - request_wake: request_consumer.wake_handle(), - response_wake: response_producer.wake_handle(), - notification_wake: notification_producer.wake_handle(), + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() .name("litebox-runner-liveness".to_owned()) .spawn(move || monitor_host_socket(&mut self.stream, &monitor_association))?; Ok(( - UnixControlRingHostRequestSource { - consumer: request_consumer, - association: Arc::clone(&association), - }, - UnixControlRingHostResponseSink { - producer: Arc::new(Mutex::new(response_producer)), + UnixControlRingHostCallLaneSource { + lanes: lanes.into_iter().map(Some).collect(), + demand, association: Arc::clone(&association), }, UnixControlRingHostNotificationChannel { @@ -301,23 +309,78 @@ impl HostSetupChannel for UnixStreamHostSetupChannel { } } -impl UnixControlRingHostRequestSource { - /// Receives one active broker request. +impl UnixControlRingHostCallLaneSource { + /// Blocks until the runner starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` once the runner closed the association. + /// + /// # Panics + /// + /// Panics if the lane demand hands out a lane more than once. + pub fn next_lane(&mut self) -> IoResult> { + loop { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + match self.demand.try_next_lane() { + Ok(ControlRingDemandStatus::Lane(index)) => { + let lane = self.lanes[index] + .take() + .expect("lane demand hands out each lane once"); + return Ok(Some(UnixControlRingHostCallLane { + lane, + association: Arc::clone(&self.association), + })); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) => { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + if let Err(error) = self.demand.wait_for_demand(wait_epoch) { + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + Err(error) => { + let error = ring_error(error); + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + } + } +} + +impl HostCallLaneSource for UnixControlRingHostCallLaneSource { + type Error = Error; + type Lane = UnixControlRingHostCallLane; + + fn next_lane(&mut self) -> IoResult> { + Self::next_lane(self) + } +} + +impl UnixControlRingHostCallLane { + /// Receives this lane's next broker request. pub fn recv_request(&mut self) -> IoResult> { + let consumer = &mut self.lane.request_consumer; loop { if let Some(error) = self.association.current_failure() { return Err(error); } - match self.consumer.try_read(decode_request) { + match consumer.try_read(decode_request) { Ok(ControlRingReadStatus::Message(request)) => { - self.association.acknowledge_request(&mut self.consumer)?; + self.association.acknowledge_request(consumer)?; return Ok(HostReceive::Message(request)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(terminal) = self.association.request_terminal_result() { - return terminal; + if let Some(ended) = self.association.ended() { + return ended.map(|()| HostReceive::PeerClosed); } - if let Err(error) = self.consumer.wait_for_message(wait_epoch) { + if let Err(error) = consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); let _ = self.association.fail(error); return result; @@ -341,26 +404,13 @@ impl UnixControlRingHostRequestSource { } } } -} - -impl HostRequestSource for UnixControlRingHostRequestSource { - type Error = Error; - fn recv_request(&mut self) -> IoResult> { - Self::recv_request(self) - } -} - -impl UnixControlRingHostResponseSink { - /// Serializes and sends one complete active broker response. - pub fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + /// Serializes and sends the response to this lane's last request. + pub fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { let frame = encode_response(response.clone()); - let mut producer = self - .producer - .lock() - .map_err(|_| Error::other("broker response writer mutex poisoned"))?; + let producer = &mut self.lane.response_producer; loop { - match self.association.try_publish(&mut producer, &frame)? { + match self.association.try_publish(producer, &frame)? { ControlRingWriteStatus::Written => return Ok(()), ControlRingWriteStatus::Full { wait_epoch } => { if let Err(error) = producer.wait_for_capacity(wait_epoch) { @@ -374,10 +424,14 @@ impl UnixControlRingHostResponseSink { } } -impl HostResponseSink for UnixControlRingHostResponseSink { +impl HostCallLane for UnixControlRingHostCallLane { type Error = Error; - fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + fn recv_request(&mut self) -> IoResult> { + Self::recv_request(self) + } + + fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { Self::send_response(self, response) } } @@ -438,12 +492,10 @@ impl HostRingAssociation { *status = HostAssociationStatus::Failed(Arc::new(error)); } } - let request_wake = self.request_wake.interrupt_wait(); - let response_wake = self.response_wake.interrupt_wait(); - let notification_wake = self.notification_wake.interrupt_wait(); - request_wake - .and(response_wake) - .and(notification_wake) + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) .and(shutdown_socket(&self.control_stream)) } @@ -457,19 +509,21 @@ impl HostRingAssociation { *status = HostAssociationStatus::PeerClosed; } } - let _ = self.request_wake.interrupt_wait(); - let _ = self.response_wake.interrupt_wait(); - let _ = self.notification_wake.interrupt_wait(); + for wake in &self.wakes { + let _ = wake.interrupt_wait(); + } } - fn request_terminal_result(&self) -> Option>> { + /// Returns how the association ended, if it has: `Ok` if the runner + /// closed it. + fn ended(&self) -> Option> { match &*self .status .lock() .expect("broker host association mutex poisoned") { HostAssociationStatus::Live => None, - HostAssociationStatus::PeerClosed => Some(Ok(HostReceive::PeerClosed)), + HostAssociationStatus::PeerClosed => Some(Ok(())), HostAssociationStatus::Failed(error) => Some(Err(copy_io_error(error))), } } @@ -562,7 +616,7 @@ mod tests { LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_NOTIFICATION_SLOT_COUNT, CONTROL_RING_SLOT_COUNT, LocalControlRingEndpoints, + CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRingLayout, LocalControlRingEndpoints, }; use crate::unix_socket::local::{ @@ -575,19 +629,23 @@ mod tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; + type Local = LocalControlRingEndpoints; + + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; fn ring_pair() -> ( ControlRing, ControlRing, ) { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); ( - ControlRing::new(first).unwrap(), - ControlRing::new(second).unwrap(), + ControlRing::new(first, LAYOUT).unwrap(), + ControlRing::new(second, LAYOUT).unwrap(), ) } @@ -635,39 +693,43 @@ mod tests { } fn activate_host() -> ( - UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, + UnixControlRingHostCallLaneSource, UnixControlRingHostShutdown, - Producer, - Consumer, + Local, UnixStream, ) { let (peer_stream, host_stream) = UnixStream::pair().unwrap(); let mut ack_stream = peer_stream.try_clone().unwrap(); let acknowledgement = thread::spawn(move || { - write_setup_frame(&mut ack_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut ack_stream, LAYOUT.ready_token(), None).unwrap(); assert_eq!( read_setup_frame(&mut ack_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); }); let (local_ring, host_ring) = ring_pair(); let channel = negotiated_host(host_stream); - let (source, sink, _notifications, shutdown) = channel.into_active(host_ring).unwrap(); + let (lanes, _notifications, shutdown) = channel.into_active(host_ring).unwrap(); acknowledgement.join().unwrap(); - let LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer: _, - } = local_ring.into_local(); - ( - source, - sink, - shutdown, - request_producer, - response_consumer, - peer_stream, - ) + (lanes, shutdown, local_ring.into_local(), peer_stream) + } + + /// Activates a host association and takes its always-demanded first lane. + fn activate_host_lane() -> ( + UnixControlRingHostCallLane, + UnixControlRingHostShutdown, + Local, + UnixStream, + ) { + let (mut lanes, shutdown, local, peer) = activate_host(); + let lane = lanes.next_lane().unwrap().unwrap(); + (lane, shutdown, local, peer) + } + + fn demand_lanes(local: &Local, lanes: usize) { + let demand = local.lane_demand.as_ref().unwrap(); + demand.request(lanes).unwrap(); + demand.wake_broker().unwrap(); } fn notification_channel_pair() -> ( @@ -681,7 +743,7 @@ mod tests { let host_active = thread::spawn(move || host_control.into_active(host_ring).unwrap()); let (local_call, local_notifications, _local_shutdown) = local_setup.into_active(local_ring, || {}).unwrap(); - let (_source, _sink, host_notifications, shutdown) = host_active.join().unwrap(); + let (_lanes, host_notifications, shutdown) = host_active.join().unwrap(); ( local_call, local_notifications, @@ -765,119 +827,163 @@ mod tests { let host_active = thread::spawn(move || host.into_active(host_ring).unwrap()); let (local, _local_notifications, _local_shutdown) = local_setup.into_active(local_ring, || {}).unwrap(); - let (mut source, sink, _host_notifications, _shutdown) = host_active.join().unwrap(); + let (mut lanes, _host_notifications, _shutdown) = host_active.join().unwrap(); let caller = thread::spawn(move || local.call(request(7))); - let HostReceive::Message(received) = source.recv_request().unwrap() else { + let mut lane = lanes.next_lane().unwrap().unwrap(); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { panic!("expected ring request"); }; - sink.send_response(&response(received.request_id)).unwrap(); + lane.send_response(&response(received.request_id)).unwrap(); assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(7)); } #[test] - fn host_activation_decodes_requests_and_cloned_sinks_publish_complete_responses() { - let (mut source, sink, _shutdown, mut requests, mut responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(1))); - assert!(matches!( - source.recv_request().unwrap(), - HostReceive::Message(BrokerRequest { - request_id: RequestId(1), - .. - }) - )); + fn host_rejects_single_lane_layouts() { + let layout = ControlRingLayout::SINGLE_LANE; + let memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); + let (_peer, host_stream) = UnixStream::pair().unwrap(); + let Err(error) = + negotiated_host(host_stream).into_active(ControlRing::new(memory, layout).unwrap()) + else { + panic!("a single-lane layout was activated"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn host_hands_out_lanes_on_demand_and_each_lane_serves_its_own_calls() { + let (mut lanes, _shutdown, mut local, peer) = activate_host(); + let mut first = lanes.next_lane().unwrap().unwrap(); + let next = thread::spawn(move || lanes.next_lane().map(|lane| (lane, lanes))); + thread::sleep(Duration::from_millis(20)); + assert!(!next.is_finished()); + demand_lanes(&local, 2); + let (second, mut lanes) = next.join().unwrap().unwrap(); + let mut second = second.unwrap(); + + for (index, lane) in [&mut first, &mut second].into_iter().enumerate() { + let id = u64::try_from(index).unwrap(); + write_payload( + &mut local.lanes[index].request_producer, + &encode_request(request(id)), + ); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { + panic!("expected ring request"); + }; + assert_eq!(received.request_id, RequestId(id)); + lane.send_response(&response(received.request_id)).unwrap(); + } + for (index, lane) in local.lanes.iter_mut().take(2).enumerate() { + assert_eq!( + read_response(&mut lane.response_consumer).request_id, + RequestId(u64::try_from(index).unwrap()) + ); + } - let first = sink.clone(); - let writer = thread::spawn(move || first.send_response(&response(RequestId(3)))); - sink.send_response(&response(RequestId(7))).unwrap(); - writer.join().unwrap().unwrap(); - let mut ids = [ - read_response(&mut responses).request_id, - read_response(&mut responses).request_id, - ]; - ids.sort(); - assert_eq!(ids, [RequestId(3), RequestId(7)]); + // Each remaining lane is handed out once; then the source waits until + // the peer closes. + demand_lanes(&local, LAYOUT.lane_count()); + for _ in 2..LAYOUT.lane_count() { + assert!(lanes.next_lane().unwrap().is_some()); + } + let last = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + assert!(!last.is_finished()); + drop(peer); + assert!(last.join().unwrap().unwrap().is_none()); } #[test] - fn host_clean_close_wakes_request_wait_as_peer_closed() { - let (mut source, _sink, _shutdown, _requests, _responses, peer) = activate_host(); - let receiver = thread::spawn(move || source.recv_request()); + fn host_clean_close_wakes_request_and_lane_waits_as_peer_closed() { + let (mut lanes, _shutdown, _local, peer) = activate_host(); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); drop(peer); assert_eq!(receiver.join().unwrap().unwrap(), HostReceive::PeerClosed); + assert!(next.join().unwrap().unwrap().is_none()); } #[test] fn host_failure_preempts_queued_and_decoded_requests_but_peer_close_drains() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(1))); - source - .association + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(1)), + ); + lane.association .fail(Error::new(ErrorKind::TimedOut, "test failure")) .unwrap(); - assert_eq!( - source.recv_request().unwrap_err().kind(), - ErrorKind::TimedOut - ); + assert_eq!(lane.recv_request().unwrap_err().kind(), ErrorKind::TimedOut); - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(2))); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(2)), + ); assert!(matches!( - source.consumer.try_read(decode_request).unwrap(), + lane.lane.request_consumer.try_read(decode_request).unwrap(), ControlRingReadStatus::Message(_) )); - source - .association + lane.association .fail(Error::new(ErrorKind::TimedOut, "test failure")) .unwrap(); assert_eq!( - source - .association - .acknowledge_request(&mut source.consumer) + lane.association + .acknowledge_request(&mut lane.lane.request_consumer) .unwrap_err() .kind(), ErrorKind::TimedOut ); - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &encode_request(request(3))); - source.association.peer_closed(); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload( + &mut local.lanes[0].request_producer, + &encode_request(request(3)), + ); + lane.association.peer_closed(); assert!(matches!( - source.recv_request().unwrap(), + lane.recv_request().unwrap(), HostReceive::Message(BrokerRequest { request_id: RequestId(3), .. }) )); - assert_eq!(source.recv_request().unwrap(), HostReceive::PeerClosed); + assert_eq!(lane.recv_request().unwrap(), HostReceive::PeerClosed); } #[test] - fn dropping_host_shutdown_guard_wakes_request_wait_and_closes_socket() { - let (mut source, sink, shutdown, _requests, _responses, mut peer) = activate_host(); + fn dropping_host_shutdown_guard_wakes_waits_and_closes_socket() { + let (mut lanes, shutdown, _local, mut peer) = activate_host(); peer.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); - let receiver = thread::spawn(move || source.recv_request()); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); - drop(sink); drop(shutdown); assert_eq!( receiver.join().unwrap().unwrap_err().kind(), ErrorKind::ConnectionAborted ); + assert_eq!( + next.join().unwrap().err().map(|error| error.kind()), + Some(ErrorKind::ConnectionAborted) + ); let mut byte = [0]; assert_eq!(peer.read(&mut byte).unwrap(), 0); } #[test] - fn host_close_wakes_response_producer_blocked_on_full_ring() { - let (_source, sink, _shutdown, _requests, _responses, peer) = activate_host(); - for id in 0..CONTROL_RING_SLOT_COUNT { - sink.send_response(&response(RequestId(id))).unwrap(); - } - let blocked_sink = sink.clone(); - let blocked = thread::spawn(move || blocked_sink.send_response(&response(RequestId(99)))); + fn host_close_wakes_response_producer_blocked_on_full_lane() { + let (mut lane, _shutdown, _local, peer) = activate_host_lane(); + lane.send_response(&response(RequestId(1))).unwrap(); + let blocked = thread::spawn(move || lane.send_response(&response(RequestId(2)))); thread::sleep(Duration::from_millis(20)); + assert!(!blocked.is_finished()); drop(peer); assert_eq!( blocked.join().unwrap().unwrap_err().kind(), @@ -887,25 +993,22 @@ mod tests { #[test] fn host_reports_wrong_phase_ring_message_as_protocol_violation() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); write_payload( - &mut requests, + &mut local.lanes[0].request_producer, &encode_handshake_request(BrokerHandshakeRequest { protocol_version: BROKER_PROTOCOL_VERSION, }), ); - assert_eq!( - source.recv_request().unwrap(), - HostReceive::ProtocolViolation - ); + assert_eq!(lane.recv_request().unwrap(), HostReceive::ProtocolViolation); } #[test] fn malformed_host_ring_request_is_fatal_invalid_data() { - let (mut source, _sink, _shutdown, mut requests, _responses, _peer) = activate_host(); - write_payload(&mut requests, &[u8::MAX]); + let (mut lane, _shutdown, mut local, _peer) = activate_host_lane(); + write_payload(&mut local.lanes[0].request_producer, &[u8::MAX]); assert_eq!( - source.recv_request().unwrap_err().kind(), + lane.recv_request().unwrap_err().kind(), ErrorKind::InvalidData ); } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index d8009c6480..e5e298537f 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -11,8 +11,8 @@ use std::io::{Error, ErrorKind, Read, Result as IoResult}; use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::path::Path; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -30,17 +30,18 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; +use litebox_broker_transport::call_lane_pool::CallLanePoolError; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; -pub use litebox_broker_transport::pending_calls::MAX_PENDING_CALLS; +use crate::call_lane_pool::{CallLanePool, call_lane_pool_error}; use crate::memfd::MemfdSharedMemory; -use crate::pending_calls::{PendingCalls, pending_calls_error}; use crate::setup::{ copy_io_error, invalid_data, read_setup_frame, ring_error, shutdown_socket, wire_error, write_setup_frame, @@ -66,22 +67,20 @@ pub struct UnixControlRingLocalShutdown { association: Arc, } +type LocalLane = LocalControlRingLane; + /// State shared by every activated local endpoint of one association: the -/// request producer, the response consumer read by whichever caller holds the -/// pending-call reader role, the setup socket used for liveness and teardown, -/// pending call tracking, and the wake handles of all three ring directions. +/// call lanes and their demand, the setup socket used for liveness and +/// teardown, and the wake handles of every ring wait. struct LocalRingAssociation { - request_producer: Mutex>, - response_consumer: Mutex>, + lanes: CallLanePool, + lane_demand: LocalLaneDemand, control_stream: UnixStream, - pending_calls: Arc, /// Whether the call channel was dropped, which ends the association as a /// local close rather than a failure. closed: AtomicBool, on_failure: Arc, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } /// Local notification receiver for a shared-ring Unix broker association. @@ -137,12 +136,14 @@ impl UnixStreamLocalSetupChannel { crate::memfd::receive_memfd(&mut self.stream, expected_len, deadline) } - /// Receives the control-ring memfd offered by the broker during setup. + /// Receives the control-ring memfd with `layout` offered by the broker + /// during setup. pub fn receive_control_ring( &mut self, + layout: ControlRingLayout, deadline: Option, ) -> IoResult { - crate::memfd::receive_control_ring_memfd(&mut self.stream, deadline) + crate::memfd::receive_control_ring_memfd(&mut self.stream, layout, deadline) } /// Receives the optional process image offered by the broker during @@ -159,7 +160,7 @@ impl UnixStreamLocalSetupChannel { /// monitor. /// /// The ring must be the control-ring memfd received during this - /// setup exchange. + /// setup exchange, with a layout of more than one lane. pub fn into_active( self, ring: ControlRing, @@ -175,38 +176,42 @@ impl UnixStreamLocalSetupChannel { )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_local(); + let wakes = endpoints.wake_handles(); + let LocalControlRingEndpoints { + lanes, + lane_demand: Some(lane_demand), + notification_consumer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let mut setup_stream = self.stream; - write_setup_frame(&mut setup_stream, CONTROL_RING_READY, self.setup_deadline)?; + write_setup_frame(&mut setup_stream, ready_token, self.setup_deadline)?; let Some(ready) = read_setup_frame(&mut setup_stream, self.setup_deadline)? else { return Err(Error::new( ErrorKind::UnexpectedEof, "broker closed before control-ring setup acknowledgement", )); }; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "broker sent an invalid control-ring setup acknowledgement", )); } let shutdown_stream = setup_stream.try_clone()?; - let litebox_broker_transport::control_ring::LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer, - } = ring.into_local(); - let pending_calls = Arc::new(PendingCalls::new()); - let on_failure: Arc = Arc::new(on_failure); let association = Arc::new(LocalRingAssociation { - request_wake: request_producer.wake_handle(), - request_producer: Mutex::new(request_producer), + lanes: CallLanePool::new(lanes), + lane_demand, control_stream: shutdown_stream, - pending_calls: Arc::clone(&pending_calls), closed: AtomicBool::new(false), - on_failure, - response_wake: response_consumer.wake_handle(), - response_consumer: Mutex::new(response_consumer), - notification_wake: notification_consumer.wake_handle(), + on_failure: Arc::new(on_failure), + wakes, }); let monitor_association = Arc::clone(&association); if let Err(error) = thread::Builder::new() @@ -338,46 +343,26 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; - let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; - let request_frame = encode_request(request); - - let write_result = { - let mut producer = association - .request_producer - .lock() - .expect("broker request writer mutex poisoned"); - loop { - let write_status = association - .pending_calls - .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) - .map_err(pending_calls_error); - match write_status { - Ok(ControlRingWriteStatus::Written) => { - if let Err(error) = producer.wake_consumer() { - break Err(error); - } - break Ok(()); - } - Ok(ControlRingWriteStatus::Full { wait_epoch }) => { - if let Err(error) = producer.wait_for_capacity(wait_epoch) { - break Err(error); - } - } - Err(error) => break Err(error), - } + let mut lane = match association + .lanes + .claim(|lanes| association.demand_lanes(lanes)) + { + Ok(lane) => lane, + Err(CallLanePoolError::AssociationFailed(error)) => return Err(copy_io_error(&error)), + Err(CallLanePoolError::Operation(error)) => { + let result = Err(copy_io_error(&error)); + let _ = association.fail(error); + return result; } }; - if let Err(error) = write_result { - let _ = association.fail(error); + let result = association.call_on_lane(&mut lane, request); + // Fail before freeing the lane, so no other call can find it midway + // through this call. + if let Err(error) = &result { + let _ = association.fail(copy_io_error(error)); } - - pending_call - .wait(|| association.read_response()) - .map_err(|error| copy_io_error(&error)) + drop(lane); + result } } @@ -386,7 +371,7 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { - if let Some(error) = self.association.pending_calls.current_failure() { + if let Some(error) = self.association.lanes.current_failure() { return self.association.end_notifications(&error); } match self.consumer.try_read(decode_notification) { @@ -400,7 +385,7 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(error) = self.association.pending_calls.current_failure() { + if let Some(error) = self.association.lanes.current_failure() { return self.association.end_notifications(&error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { @@ -442,14 +427,14 @@ impl LocalRingAssociation { consumer: &mut ControlRingConsumer, ) -> IoResult<()> { let result = self - .pending_calls + .lanes .run_if_live(|| { consumer .publish_head() .map_err(ring_error) .and_then(|()| consumer.wake_producer()) }) - .map_err(pending_calls_error); + .map_err(call_lane_pool_error); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -458,64 +443,69 @@ impl LocalRingAssociation { Ok(()) } - /// Reads one response for the caller holding the pending-call reader - /// role, blocking until one arrives or the association fails. - fn read_response(&self) { - let mut consumer = self - .response_consumer - .lock() - .expect("broker response reader mutex poisoned"); + /// Raises the lane demand to `lanes` and wakes the broker to serve them. + fn demand_lanes(&self, lanes: usize) -> IoResult<()> { + self.lane_demand.request(lanes).map_err(ring_error)?; + self.lane_demand.wake_broker() + } + + /// Writes `request` to `lane` and reads the broker's response from it. + fn call_on_lane( + &self, + lane: &mut LocalLane, + request: BrokerRequest, + ) -> IoResult { + let request_id = request.request_id; + let request_frame = encode_request(request); + let producer = &mut lane.request_producer; loop { - match consumer.try_read(decode_response) { - Ok(ControlRingReadStatus::Message(response)) => { - if let Err(error) = consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - .and_then(|()| { - self.pending_calls - .complete(response) - .map_err(pending_calls_error) - }) - { - let _ = self.fail(error); - } - return; + let write_status = self + .lanes + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) + .map_err(call_lane_pool_error)?; + match write_status { + ControlRingWriteStatus::Written => break, + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch)?; } + } + } + producer.wake_consumer()?; + + let consumer = &mut lane.response_consumer; + let response = loop { + match consumer.try_read(decode_response) { + Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if self.pending_calls.current_failure().is_some() { - return; - } - if let Err(error) = consumer.wait_for_message(wait_epoch) { - let _ = self.fail(error); - return; + if let Some(error) = self.lanes.current_failure() { + return Err(copy_io_error(&error)); } + consumer.wait_for_message(wait_epoch)?; } - Err(ControlRingReadError::Ring(error)) => { - let _ = self.fail(ring_error(error)); - return; - } - Err(ControlRingReadError::Decode(error)) => { - let _ = self.fail(wire_error(error)); - return; - } + Err(ControlRingReadError::Ring(error)) => return Err(ring_error(error)), + Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } + }; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer()?; + if response.request_id != request_id { + return Err(invalid_data("broker answered a different request")); } + Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - let first_failure = self.pending_calls.record_failure(Arc::new(error)); - let request_wake = self.request_wake.interrupt_wait(); - let response_wake = self.response_wake.interrupt_wait(); - let notification_wake = self.notification_wake.interrupt_wait(); + let first_failure = self.lanes.record_failure(Arc::new(error)); + let wakes = self + .wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and); let shutdown_result = shutdown_socket(&self.control_stream); if first_failure { (self.on_failure)(); } - request_wake - .and(response_wake) - .and(notification_wake) - .and(shutdown_result) + wakes.and(shutdown_result) } } @@ -554,6 +544,9 @@ mod control_ring_tests { }; use litebox_broker_protocol::{ObjectHandle, RequestId}; use litebox_broker_transport::channel::{LocalCallChannel, LocalSetupChannel}; + use litebox_broker_transport::control_ring::{ + BrokerControlRingEndpoints, ControlRingDemandStatus, ControlRingProducer, + }; use rustix::fs::{OFlags, fcntl_getfl}; use std::io::{Read, Write}; use std::os::fd::AsFd; @@ -564,6 +557,9 @@ mod control_ring_tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; + type Broker = BrokerControlRingEndpoints; + + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; struct TestSocketPath(PathBuf); @@ -615,14 +611,15 @@ mod control_ring_tests { ControlRing, ControlRing, ) { - let first = MemfdSharedMemory::create_control_ring().unwrap(); + let first = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let second = MemfdSharedMemory::control_ring_from_received_fd( first.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); ( - ControlRing::new(first).unwrap(), - ControlRing::new(second).unwrap(), + ControlRing::new(first, LAYOUT).unwrap(), + ControlRing::new(second, LAYOUT).unwrap(), ) } @@ -640,8 +637,7 @@ mod control_ring_tests { UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, - Producer, - Consumer, + Broker, UnixStream, ) { let (local_stream, peer_stream) = UnixStream::pair().unwrap(); @@ -649,25 +645,19 @@ mod control_ring_tests { let acknowledgement = thread::spawn(move || { assert_eq!( read_setup_frame(&mut ack_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); - write_setup_frame(&mut ack_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut ack_stream, LAYOUT.ready_token(), None).unwrap(); }); let (local_ring, broker_ring) = ring_pair(); let setup = negotiated_local(local_stream); let (channel, notifications, shutdown) = setup.into_active(local_ring, on_failure).unwrap(); acknowledgement.join().unwrap(); - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer: _, - } = broker_ring.into_broker(); ( channel, notifications, shutdown, - response_producer, - request_consumer, + broker_ring.into_broker(), peer_stream, ) } @@ -759,9 +749,9 @@ mod control_ring_tests { let acknowledgement = thread::spawn(move || { assert_eq!( read_setup_frame(&mut host_stream, None).unwrap().unwrap(), - CONTROL_RING_READY + LAYOUT.ready_token() ); - write_setup_frame(&mut host_stream, CONTROL_RING_READY, None).unwrap(); + write_setup_frame(&mut host_stream, LAYOUT.ready_token(), None).unwrap(); host_stream }); let (ring, _) = ring_pair(); @@ -777,9 +767,21 @@ mod control_ring_tests { } #[test] - fn local_matches_out_of_order_ring_responses_without_socket_frames() { - let (channel, _notifications, _shutdown, mut responses, mut requests, mut peer) = - activate_local(|| {}); + fn single_lane_layouts_are_rejected() { + let layout = ControlRingLayout::SINGLE_LANE; + let memory = MemfdSharedMemory::create_control_ring(layout).unwrap(); + let (local_stream, _peer) = UnixStream::pair().unwrap(); + let Err(error) = negotiated_local(local_stream) + .into_active(ControlRing::new(memory, layout).unwrap(), || {}) + else { + panic!("a single-lane layout was activated"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn concurrent_calls_use_separate_lanes_without_socket_frames() { + let (channel, _notifications, _shutdown, mut broker, mut peer) = activate_local(|| {}); peer.set_read_timeout(Some(Duration::from_millis(100))) .unwrap(); let channel = Arc::new(channel); @@ -787,17 +789,38 @@ mod control_ring_tests { let channel = Arc::clone(&channel); thread::spawn(move || channel.call(request(id))) }); - let first = read_request(&mut requests); - let second = read_request(&mut requests); + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + loop { + match demand.try_next_lane().unwrap() { + ControlRingDemandStatus::Lane(lane) => { + assert_eq!(lane, 1); + break; + } + ControlRingDemandStatus::Satisfied { wait_epoch } => { + demand.wait_for_demand(wait_epoch).unwrap(); + } + } + } + let first = read_request(&mut broker.lanes[0].request_consumer); + let second = read_request(&mut broker.lanes[1].request_consumer); + // The later lane answers first; neither call waits for the other. write_payload( - &mut responses, + &mut broker.lanes[1].response_producer, &encode_response(response(second.request_id)), ); - write_payload(&mut responses, &encode_response(response(first.request_id))); + write_payload( + &mut broker.lanes[0].response_producer, + &encode_response(response(first.request_id)), + ); for call in calls { assert!(call.join().unwrap().is_ok()); } + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); let mut byte = [0]; assert!(matches!( peer.read(&mut byte).unwrap_err().kind(), @@ -806,12 +829,38 @@ mod control_ring_tests { } #[test] - fn unknown_duplicate_and_malformed_responses_fail_closed() { + fn sequential_calls_reuse_the_first_lane() { + let (channel, _notifications, _shutdown, mut broker, _peer) = activate_local(|| {}); + let channel = Arc::new(channel); + for id in 1..=3 { + let caller = { + let channel = Arc::clone(&channel); + thread::spawn(move || channel.call(request(id))) + }; + let lane = &mut broker.lanes[0]; + let request = read_request(&mut lane.request_consumer); + assert_eq!(request.request_id, RequestId(id)); + write_payload( + &mut lane.response_producer, + &encode_response(response(request.request_id)), + ); + assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(id)); + } + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); + } + + #[test] + fn mismatched_and_malformed_responses_fail_closed() { for payload_kind in 0..3 { let failures = Arc::new(AtomicUsize::new(0)); let callback_failures = Arc::clone(&failures); let (failure_reported, wait_for_failure) = mpsc::channel(); - let (channel, _notifications, _shutdown, mut responses, mut requests, _peer) = + let (channel, _notifications, _shutdown, mut broker, _peer) = activate_local(move || { callback_failures.fetch_add(1, Ordering::SeqCst); failure_reported.send(()).unwrap(); @@ -821,22 +870,19 @@ mod control_ring_tests { let channel = Arc::clone(&channel); thread::spawn(move || channel.call(request(id))) }); - read_request(&mut requests); - read_request(&mut requests); + read_request(&mut broker.lanes[0].request_consumer); + let other = read_request(&mut broker.lanes[1].request_consumer); + let producer = &mut broker.lanes[0].response_producer; match payload_kind { - 0 => write_payload(&mut responses, &encode_response(response(RequestId(99)))), - 1 => { - let duplicate = encode_response(response(RequestId(1))); - write_payload(&mut responses, &duplicate); - write_payload(&mut responses, &duplicate); - } - _ => write_payload(&mut responses, &[u8::MAX]), + 0 => write_payload(producer, &encode_response(response(RequestId(99)))), + 1 => write_payload(producer, &encode_response(response(other.request_id))), + _ => write_payload(producer, &[u8::MAX]), } - let results = calls.map(|call| call.join().unwrap()); - let error_count = results.iter().filter(|result| result.is_err()).count(); - assert_eq!(error_count, if payload_kind == 1 { 1 } else { 2 }); + for call in calls { + assert!(call.join().unwrap().is_err()); + } wait_for_failure .recv_timeout(Duration::from_secs(1)) .expect("failure callback was not invoked"); @@ -845,25 +891,36 @@ mod control_ring_tests { } #[test] - fn local_socket_eof_and_shutdown_wake_pending_calls() { + fn local_socket_eof_and_shutdown_wake_waiting_calls() { for close_peer in [false, true] { - let (channel, _notifications, shutdown, _responses, mut requests, peer) = - activate_local(|| {}); - let caller = thread::spawn(move || channel.call(request(1))); - read_request(&mut requests); + let (channel, _notifications, shutdown, mut broker, peer) = activate_local(|| {}); + let channel = Arc::new(channel); + let call = |id| { + let channel = Arc::clone(&channel); + thread::spawn(move || channel.call(request(id))) + }; + let callers: Vec<_> = (0..LAYOUT.lane_count() as u64).map(call).collect(); + for lane in &mut broker.lanes { + read_request(&mut lane.request_consumer); + } + // Every lane is held, so this call waits to claim one. + let waiting = call(u64::MAX); + thread::sleep(Duration::from_millis(10)); if close_peer { drop(peer); } else { shutdown.shutdown().unwrap(); } - assert!(caller.join().unwrap().is_err()); + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + assert!(waiting.join().unwrap().is_err()); } } #[test] fn dropping_the_call_channel_ends_notifications_cleanly() { - let (channel, mut notifications, _shutdown, _responses, _requests, _peer) = - activate_local(|| {}); + let (channel, mut notifications, _shutdown, _broker, _peer) = activate_local(|| {}); let receiver = thread::spawn(move || notifications.recv_notification()); drop(channel); assert!(receiver.join().unwrap().unwrap().is_none()); @@ -973,37 +1030,9 @@ mod control_ring_tests { ); } - #[test] - fn completed_call_wins_over_later_failure_and_failure_wins_before_completion() { - let pending = PendingCalls::new(); - let completed = pending.register(RequestId(1)).unwrap(); - pending.complete(response(RequestId(1))).unwrap(); - pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); - assert_eq!( - completed.wait(|| unreachable!()).unwrap().request_id, - RequestId(1) - ); - - let pending = PendingCalls::new(); - let failed = pending.register(RequestId(2)).unwrap(); - pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); - assert!(pending.complete(response(RequestId(2))).is_err()); - assert_eq!( - failed.wait(|| unreachable!()).unwrap_err().kind(), - ErrorKind::ConnectionAborted - ); - } - #[test] fn failure_recording_waits_for_in_progress_publication() { - let pending = Arc::new(PendingCalls::new()); - let pending_call = pending.register(RequestId(1)).unwrap(); + let pending = Arc::new(CallLanePool::new(vec![()])); let publication_state = Arc::new(AtomicUsize::new(0)); let (publication_started, wait_for_publication) = std::sync::mpsc::sync_channel(0); let (release_publication, publication_released) = std::sync::mpsc::sync_channel(0); @@ -1047,24 +1076,10 @@ mod control_ring_tests { .recv_timeout(Duration::from_secs(1)) .unwrap(); failure.join().unwrap(); - assert_eq!( - pending_call.wait(|| unreachable!()).unwrap_err().kind(), - ErrorKind::ConnectionAborted - ); - } - - #[test] - fn duplicate_pending_registration_preserves_original() { - let pending = PendingCalls::new(); - let original = pending.register(RequestId(1)).unwrap(); - let Err(error) = pending.register(RequestId(1)).map_err(pending_calls_error) else { - panic!("duplicate registration unexpectedly succeeded"); - }; - assert_eq!(error.kind(), ErrorKind::InvalidData); - pending.complete(response(RequestId(1))).unwrap(); - assert_eq!( - original.wait(|| unreachable!()).unwrap().request_id, - RequestId(1) - ); + assert!(matches!( + pending.claim(|_| Ok(())), + Err(CallLanePoolError::AssociationFailed(error)) + if error.kind() == ErrorKind::ConnectionAborted + )); } } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs b/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs index 43407e1409..77d7f65e98 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/mod.rs @@ -17,11 +17,11 @@ mod host; mod local; pub use host::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, - validate_peer_process, + UnixControlRingHostCallLane, UnixControlRingHostCallLaneSource, + UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, + UnixStreamHostSetupChannel, validate_peer_process, }; pub use local::{ - MAX_PENDING_CALLS, UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, + UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, UnixStreamLocalSetupChannel, }; diff --git a/litebox_broker_transport_windows_userland/src/call_lane_pool.rs b/litebox_broker_transport_windows_userland/src/call_lane_pool.rs new file mode 100644 index 0000000000..f10d1108fe --- /dev/null +++ b/litebox_broker_transport_windows_userland/src/call_lane_pool.rs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +use std::io::Error; +use std::marker::PhantomData; +use std::sync::{Condvar, Mutex, MutexGuard}; + +use litebox_broker_transport::call_lane_pool::{ + CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, + CallLanePoolSync, +}; + +use crate::setup::copy_io_error; + +pub(crate) type CallLanePool = GenericCallLanePool; + +pub(crate) struct StdCallLanePoolSync; + +pub(crate) struct StdCallLanePoolMutex(Mutex); + +pub(crate) struct StdCallLanePoolCondvar(Condvar, PhantomData); + +impl CallLanePoolMutex for StdCallLanePoolMutex { + type Guard<'a> + = MutexGuard<'a, T> + where + T: 'a; + + fn lock(&self) -> Self::Guard<'_> { + self.0.lock().expect("broker call-lanes mutex poisoned") + } +} + +impl CallLanePoolCondvar for StdCallLanePoolCondvar { + type Mutex = StdCallLanePoolMutex; + + fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> + where + Self::Mutex: 'a, + T: 'a, + { + self.0 + .wait(guard) + .expect("broker call-lanes mutex poisoned") + } + + fn notify_one(&self) { + self.0.notify_one(); + } + + fn notify_all(&self) { + self.0.notify_all(); + } +} + +impl CallLanePoolSync for StdCallLanePoolSync { + type Mutex = StdCallLanePoolMutex; + type Condvar = StdCallLanePoolCondvar; + + fn mutex(value: T) -> Self::Mutex { + StdCallLanePoolMutex(Mutex::new(value)) + } + + fn condvar() -> Self::Condvar { + StdCallLanePoolCondvar(Condvar::new(), PhantomData) + } +} + +pub(crate) fn call_lane_pool_error(error: CallLanePoolError) -> Error { + match error { + CallLanePoolError::AssociationFailed(error) => copy_io_error(&error), + CallLanePoolError::Operation(error) => error, + } +} + +#[cfg(test)] +mod tests { + use std::io::{Error, ErrorKind}; + use std::sync::Arc; + use std::sync::mpsc; + use std::time::Duration; + + use litebox_broker_transport::call_lane_pool::CallLanePoolError; + + use super::CallLanePool; + + fn no_demand(_: usize) -> Result<(), Error> { + unreachable!("lane 0 needs no demand") + } + + #[test] + fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { + let lanes = CallLanePool::new(vec![0, 1, 2]); + let mut demands = Vec::new(); + let first = lanes.claim(no_demand).unwrap(); + let second = lanes + .claim(|count| { + demands.push(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*first, *second), (0, 1)); + drop(first); + let reused = lanes.claim(no_demand).unwrap(); + assert_eq!(*reused, 0); + drop((reused, second)); + let again = lanes.claim(no_demand).unwrap(); + let next = lanes + .claim(|_| panic!("lane 1 is already demanded")) + .unwrap(); + assert_eq!((*again, *next), (0, 1)); + assert_eq!(demands, [2]); + } + + #[test] + fn a_failed_demand_leaves_the_lane_free_and_undemanded() { + let lanes = CallLanePool::new(vec![0, 1]); + let _first = lanes.claim(no_demand).unwrap(); + assert!(matches!( + lanes.claim(|_| Err(Error::other("demand failed"))), + Err(CallLanePoolError::Operation(_)) + )); + let mut demanded = None; + let second = lanes + .claim(|count| { + demanded = Some(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*second, demanded), (1, Some(2))); + } + + #[test] + fn claims_wait_while_every_lane_is_held() { + let lanes = CallLanePool::new(vec![0]); + let held = lanes.claim(no_demand).unwrap(); + let (claimed, claim) = mpsc::channel(); + std::thread::scope(|scope| { + scope.spawn(|| { + let lane = lanes.claim(no_demand).map(|lane| *lane); + claimed.send(lane.is_ok()).unwrap(); + }); + assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); + drop(held); + assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); + }); + } + + #[test] + fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { + let lanes = CallLanePool::new(vec![0]); + let _held = lanes.claim(no_demand).unwrap(); + std::thread::scope(|scope| { + let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); + std::thread::sleep(Duration::from_millis(10)); + assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); + assert!(matches!( + waiter.join().unwrap(), + Err(CallLanePoolError::AssociationFailed(_)) + )); + }); + assert!(!lanes.record_failure(Arc::new(Error::from(ErrorKind::Other)))); + assert_eq!( + lanes.current_failure().unwrap().kind(), + ErrorKind::BrokenPipe + ); + assert!(matches!( + lanes.run_if_live(|| Ok(())), + Err(CallLanePoolError::AssociationFailed(_)) + )); + } +} diff --git a/litebox_broker_transport_windows_userland/src/control_ring.rs b/litebox_broker_transport_windows_userland/src/control_ring.rs index f8b57f0858..a6d58d950d 100644 --- a/litebox_broker_transport_windows_userland/src/control_ring.rs +++ b/litebox_broker_transport_windows_userland/src/control_ring.rs @@ -4,8 +4,8 @@ //! Active Windows-userland broker endpoints over a shared control ring. pub use crate::host::{ - WindowsControlRingHostNotificationChannel, WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, WindowsControlRingHostShutdown, + WindowsControlRingHostCallLane, WindowsControlRingHostCallLaneSource, + WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, }; pub use crate::local::{ WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, diff --git a/litebox_broker_transport_windows_userland/src/host.rs b/litebox_broker_transport_windows_userland/src/host.rs index 7218a32e58..523ab05160 100644 --- a/litebox_broker_transport_windows_userland/src/host.rs +++ b/litebox_broker_transport_windows_userland/src/host.rs @@ -17,12 +17,13 @@ use litebox_broker_protocol::wire::{ encode_notification, encode_response, }; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, PeerCredential, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + BrokerControlRingEndpoints, BrokerControlRingLane, BrokerLaneDemand, ControlRing, + ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, }; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::System::Pipes::GetNamedPipeClientProcessId; @@ -45,16 +46,20 @@ pub struct WindowsNamedPipeHostSetupChannel { negotiated: bool, } -/// Request-reading endpoint of an active host control-ring association. -pub struct WindowsControlRingHostRequestSource { - consumer: ControlRingConsumer, +type BrokerLane = BrokerControlRingLane; + +/// Source of the call lanes of an active host control-ring association, which +/// hands out each lane once the runner starts using it. +pub struct WindowsControlRingHostCallLaneSource { + /// Each lane, or `None` once handed out. + lanes: Vec>, + demand: BrokerLaneDemand, association: Arc, } -/// Shared response-writing endpoint of an active host control-ring association. -#[derive(Clone)] -pub struct WindowsControlRingHostResponseSink { - producer: Arc>>, +/// One call lane of an active host control-ring association. +pub struct WindowsControlRingHostCallLane { + lane: BrokerLane, association: Arc, } @@ -69,12 +74,13 @@ pub struct WindowsControlRingHostShutdown { association: Arc, } +/// State shared by every activated host endpoint of one association: terminal +/// status, the pipe liveness used for teardown, and the wake handles of every +/// ring wait. struct HostRingAssociation { status: Mutex, liveness: PipeLiveness, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } enum HostAssociationStatus { @@ -146,13 +152,14 @@ impl WindowsNamedPipeHostSetupChannel { ) } - /// Activates host request, response, and notification control-ring endpoints. + /// Activates host call-lane, notification, and shutdown control-ring endpoints. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( self, ring: ControlRing, ) -> IoResult<( - WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, + WindowsControlRingHostCallLaneSource, WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, )> { @@ -214,8 +221,7 @@ fn activate_host( setup_deadline: Option, ring: ControlRing, ) -> IoResult<( - WindowsControlRingHostRequestSource, - WindowsControlRingHostResponseSink, + WindowsControlRingHostCallLaneSource, WindowsControlRingHostNotificationChannel, WindowsControlRingHostShutdown, )> { @@ -224,43 +230,47 @@ fn activate_host( "broker host setup channel activated before negotiation completed", )); } + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_broker(); + let wakes = endpoints.wake_handles(); + let BrokerControlRingEndpoints { + lanes, + lane_demand: Some(demand), + notification_producer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; let ready = read_frame(file_handle(&stream), setup_deadline)?.ok_or_else(|| { Error::new( ErrorKind::UnexpectedEof, "runner closed before control-ring setup acknowledgement", ) })?; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "runner sent an invalid control-ring setup acknowledgement", )); } - write_frame(file_handle(&stream), CONTROL_RING_READY, setup_deadline)?; + write_frame(file_handle(&stream), ready_token, setup_deadline)?; - let litebox_broker_transport::control_ring::BrokerControlRingEndpoints { - request_consumer, - response_producer, - notification_producer, - } = ring.into_broker(); let stream = Arc::new(stream); let association = Arc::new(HostRingAssociation { liveness: PipeLiveness::new(Arc::clone(&stream), true)?, status: Mutex::new(HostAssociationStatus::Live), - request_wake: request_consumer.wake_handle(), - response_wake: response_producer.wake_handle(), - notification_wake: notification_producer.wake_handle(), + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() .name("litebox-runner-liveness".to_owned()) .spawn(move || monitor_host_pipe(&stream, &monitor_association))?; Ok(( - WindowsControlRingHostRequestSource { - consumer: request_consumer, - association: Arc::clone(&association), - }, - WindowsControlRingHostResponseSink { - producer: Arc::new(Mutex::new(response_producer)), + WindowsControlRingHostCallLaneSource { + lanes: lanes.into_iter().map(Some).collect(), + demand, association: Arc::clone(&association), }, WindowsControlRingHostNotificationChannel { @@ -298,23 +308,78 @@ impl Drop for WindowsControlRingHostShutdown { } } -impl WindowsControlRingHostRequestSource { - /// Receives one active broker request. +impl WindowsControlRingHostCallLaneSource { + /// Blocks until the runner starts using another lane and returns it. + /// + /// Once every lane is handed out, this blocks until the association ends. + /// Returns `Ok(None)` once the runner closed the association. + /// + /// # Panics + /// + /// Panics if the lane demand hands out a lane more than once. + pub fn next_lane(&mut self) -> IoResult> { + loop { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + match self.demand.try_next_lane() { + Ok(ControlRingDemandStatus::Lane(index)) => { + let lane = self.lanes[index] + .take() + .expect("lane demand hands out each lane once"); + return Ok(Some(WindowsControlRingHostCallLane { + lane, + association: Arc::clone(&self.association), + })); + } + Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) => { + if let Some(ended) = self.association.ended() { + return ended.map(|()| None); + } + if let Err(error) = self.demand.wait_for_demand(wait_epoch) { + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + Err(error) => { + let error = ring_error(error); + let result = Err(copy_io_error(&error)); + let _ = self.association.fail(error); + return result; + } + } + } + } +} + +impl HostCallLaneSource for WindowsControlRingHostCallLaneSource { + type Error = Error; + type Lane = WindowsControlRingHostCallLane; + + fn next_lane(&mut self) -> IoResult> { + Self::next_lane(self) + } +} + +impl WindowsControlRingHostCallLane { + /// Receives this lane's next broker request. pub fn recv_request(&mut self) -> IoResult> { + let consumer = &mut self.lane.request_consumer; loop { if let Some(error) = self.association.current_failure() { return Err(error); } - match self.consumer.try_read(decode_request) { + match consumer.try_read(decode_request) { Ok(ControlRingReadStatus::Message(request)) => { - self.association.acknowledge_request(&mut self.consumer)?; + self.association.acknowledge_request(consumer)?; return Ok(HostReceive::Message(request)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(terminal) = self.association.request_terminal_result() { - return terminal; + if let Some(ended) = self.association.ended() { + return ended.map(|()| HostReceive::PeerClosed); } - if let Err(error) = self.consumer.wait_for_message(wait_epoch) { + if let Err(error) = consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); let _ = self.association.fail(error); return result; @@ -338,26 +403,13 @@ impl WindowsControlRingHostRequestSource { } } } -} - -impl HostRequestSource for WindowsControlRingHostRequestSource { - type Error = Error; - - fn recv_request(&mut self) -> IoResult> { - Self::recv_request(self) - } -} -impl WindowsControlRingHostResponseSink { - /// Serializes and sends one complete active broker response. - pub fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + /// Serializes and sends the response to this lane's last request. + pub fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { let frame = encode_response(response.clone()); - let mut producer = self - .producer - .lock() - .map_err(|_| Error::other("broker response writer mutex poisoned"))?; + let producer = &mut self.lane.response_producer; loop { - match self.association.try_publish(&mut producer, &frame)? { + match self.association.try_publish(producer, &frame)? { ControlRingWriteStatus::Written => return Ok(()), ControlRingWriteStatus::Full { wait_epoch } => { if let Err(error) = producer.wait_for_capacity(wait_epoch) { @@ -371,10 +423,14 @@ impl WindowsControlRingHostResponseSink { } } -impl HostResponseSink for WindowsControlRingHostResponseSink { +impl HostCallLane for WindowsControlRingHostCallLane { type Error = Error; - fn send_response(&self, response: &BrokerResponse) -> IoResult<()> { + fn recv_request(&mut self) -> IoResult> { + Self::recv_request(self) + } + + fn send_response(&mut self, response: &BrokerResponse) -> IoResult<()> { Self::send_response(self, response) } } @@ -453,20 +509,22 @@ impl HostRingAssociation { } fn interrupt_waits(&self) -> IoResult<()> { - self.request_wake - .interrupt_wait() - .and(self.response_wake.interrupt_wait()) - .and(self.notification_wake.interrupt_wait()) + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) } - fn request_terminal_result(&self) -> Option>> { + /// Returns how the association ended, if it has: `Ok` if the runner + /// closed it. + fn ended(&self) -> Option> { match &*self .status .lock() .expect("broker host association mutex poisoned") { HostAssociationStatus::Live => None, - HostAssociationStatus::PeerClosed => Some(Ok(HostReceive::PeerClosed)), + HostAssociationStatus::PeerClosed => Some(Ok(())), HostAssociationStatus::Failed(error) => Some(Err(copy_io_error(error))), } } diff --git a/litebox_broker_transport_windows_userland/src/lib.rs b/litebox_broker_transport_windows_userland/src/lib.rs index 9422859139..ac8e9a6c6c 100644 --- a/litebox_broker_transport_windows_userland/src/lib.rs +++ b/litebox_broker_transport_windows_userland/src/lib.rs @@ -8,12 +8,12 @@ #![cfg(all(windows, target_arch = "x86_64"))] +mod call_lane_pool; pub mod control_ring; pub mod existing_runner; mod host; mod local; pub mod named_pipe; -mod pending_calls; pub mod process_image; mod setup; pub mod shared_memory; diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 8d64923454..262545f261 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -7,8 +7,8 @@ use std::ffi::OsStr; use std::fs::OpenOptions; use std::io::{Error, ErrorKind, Result as IoResult}; use std::os::windows::fs::OpenOptionsExt; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -20,19 +20,21 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; +use litebox_broker_transport::call_lane_pool::CallLanePoolError; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - CONTROL_RING_READY, ControlRing, ControlRingConsumer, ControlRingProducer, - ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingReadError, + ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; +use crate::call_lane_pool::{CallLanePool, call_lane_pool_error}; use crate::control_ring::PipeLiveness; use crate::named_pipe::{TRANSFER_FRAME_TAG, WindowsNamedPipeStream}; -use crate::pending_calls::{PendingCalls, pending_calls_error}; use crate::process_image::WindowsReceivedProcessImage; use crate::setup::{ copy_io_error, invalid_data, read_frame, read_pipe_until_cancelled, ring_error, wire_error, @@ -60,18 +62,19 @@ pub struct WindowsControlRingLocalNotificationChannel { association: Arc, } +type LocalLane = LocalControlRingLane; + +/// State shared by every activated local endpoint of one association: the +/// call lanes and their demand, the pipe liveness used for teardown, and the +/// wake handles of every ring wait. struct LocalRingAssociation { - request_producer: Mutex>, - /// Read by whichever caller holds the pending-call reader role. - response_consumer: Mutex>, - pending_calls: Arc, + lanes: CallLanePool, + lane_demand: LocalLaneDemand, /// Whether the call channel was dropped, which ends the association as a /// local close rather than a failure. closed: AtomicBool, liveness: PipeLiveness, - request_wake: ControlRingWakeHandle, - response_wake: ControlRingWakeHandle, - notification_wake: ControlRingWakeHandle, + wakes: Vec>, } impl WindowsNamedPipeLocalSetupChannel { @@ -123,8 +126,12 @@ impl WindowsNamedPipeLocalSetupChannel { unsafe { WindowsSharedMemory::from_transferred(transfer) } } - /// Receives and maps the shared control ring duplicated into this process. - pub fn receive_control_ring(&mut self) -> IoResult { + /// Receives the shared control ring the broker duplicated into this process and maps it with + /// `layout`. + pub fn receive_control_ring( + &mut self, + layout: ControlRingLayout, + ) -> IoResult { let frame = read_frame(file_handle(&self.stream), self.setup_deadline)?.ok_or_else(|| { Error::new( @@ -135,7 +142,7 @@ impl WindowsNamedPipeLocalSetupChannel { let transfer = decode_transfer(&frame)?; // SAFETY: The authenticated broker duplicated these handles into this process and encoded // their target-process values in the setup frame. - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) } + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, layout) } } /// Receives and maps the optional process image the broker sent with @@ -155,6 +162,8 @@ impl WindowsNamedPipeLocalSetupChannel { } /// Activates calls and notifications over the transferred shared control ring. + /// + /// The ring must have a layout of more than one lane. pub fn into_active( self, ring: ControlRing, @@ -200,35 +209,40 @@ fn activate_local( "broker local setup channel activated before negotiation completed", )); } - write_frame(file_handle(&stream), CONTROL_RING_READY, setup_deadline)?; + let ready_token = ring.layout().ready_token(); + let endpoints = ring.into_local(); + let wakes = endpoints.wake_handles(); + let LocalControlRingEndpoints { + lanes, + lane_demand: Some(lane_demand), + notification_consumer, + } = endpoints + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "broker control-ring layout has a single lane", + )); + }; + write_frame(file_handle(&stream), ready_token, setup_deadline)?; let ready = read_frame(file_handle(&stream), setup_deadline)?.ok_or_else(|| { Error::new( ErrorKind::UnexpectedEof, "broker closed before control-ring setup acknowledgement", ) })?; - if ready != CONTROL_RING_READY { + if ready != ready_token { return Err(invalid_data( "broker sent an invalid control-ring setup acknowledgement", )); } - let litebox_broker_transport::control_ring::LocalControlRingEndpoints { - request_producer, - response_consumer, - notification_consumer, - } = ring.into_local(); - let pending_calls = Arc::new(PendingCalls::new()); let stream = Arc::new(stream); let association = Arc::new(LocalRingAssociation { - liveness: PipeLiveness::new(Arc::clone(&stream), false)?, - request_wake: request_producer.wake_handle(), - request_producer: Mutex::new(request_producer), - response_wake: response_consumer.wake_handle(), - response_consumer: Mutex::new(response_consumer), - notification_wake: notification_consumer.wake_handle(), - pending_calls, + lanes: CallLanePool::new(lanes), + lane_demand, closed: AtomicBool::new(false), + liveness: PipeLiveness::new(Arc::clone(&stream), false)?, + wakes, }); let monitor_association = Arc::clone(&association); thread::Builder::new() @@ -250,41 +264,26 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; - let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; - let frame = encode_request(request); - let write_result = { - let mut producer = association - .request_producer - .lock() - .map_err(|_| Error::other("broker request writer mutex poisoned"))?; - loop { - let status = association - .pending_calls - .run_if_live(|| producer.try_write(&frame).map_err(ring_error)) - .map_err(pending_calls_error); - match status { - Ok(ControlRingWriteStatus::Written) => { - break producer.wake_consumer(); - } - Ok(ControlRingWriteStatus::Full { wait_epoch }) => { - if let Err(error) = producer.wait_for_capacity(wait_epoch) { - break Err(error); - } - } - Err(error) => break Err(error), - } + let mut lane = match association + .lanes + .claim(|lanes| association.demand_lanes(lanes)) + { + Ok(lane) => lane, + Err(CallLanePoolError::AssociationFailed(error)) => return Err(copy_io_error(&error)), + Err(CallLanePoolError::Operation(error)) => { + let result = Err(copy_io_error(&error)); + let _ = association.fail(error); + return result; } }; - if let Err(error) = write_result { - let _ = association.fail(error); + let result = association.call_on_lane(&mut lane, request); + // Fail before freeing the lane, so no other call can find it midway + // through this call. + if let Err(error) = &result { + let _ = association.fail(copy_io_error(error)); } - pending_call - .wait(|| association.read_response()) - .map_err(|error| copy_io_error(&error)) + drop(lane); + result } } @@ -303,7 +302,7 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { - if let Some(error) = self.association.pending_calls.current_failure() { + if let Some(error) = self.association.lanes.current_failure() { return self.association.end_notifications(&error); } match self.consumer.try_read(decode_notification) { @@ -317,7 +316,7 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if let Some(error) = self.association.pending_calls.current_failure() { + if let Some(error) = self.association.lanes.current_failure() { return self.association.end_notifications(&error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { @@ -359,14 +358,14 @@ impl LocalRingAssociation { consumer: &mut ControlRingConsumer, ) -> IoResult<()> { let result = self - .pending_calls + .lanes .run_if_live(|| { consumer .publish_head() .map_err(ring_error) .and_then(|()| consumer.wake_producer()) }) - .map_err(pending_calls_error); + .map_err(call_lane_pool_error); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -375,57 +374,63 @@ impl LocalRingAssociation { Ok(()) } - /// Reads one response for the caller holding the pending-call reader - /// role, blocking until one arrives or the association fails. - fn read_response(&self) { - let Ok(mut consumer) = self.response_consumer.lock() else { - let _ = self.fail(Error::other("broker response reader mutex poisoned")); - return; - }; + /// Raises the lane demand to `lanes` and wakes the broker to serve them. + fn demand_lanes(&self, lanes: usize) -> IoResult<()> { + self.lane_demand.request(lanes).map_err(ring_error)?; + self.lane_demand.wake_broker() + } + + /// Writes `request` to `lane` and reads the broker's response from it. + fn call_on_lane( + &self, + lane: &mut LocalLane, + request: BrokerRequest, + ) -> IoResult { + let request_id = request.request_id; + let request_frame = encode_request(request); + let producer = &mut lane.request_producer; loop { - match consumer.try_read(decode_response) { - Ok(ControlRingReadStatus::Message(response)) => { - if let Err(error) = consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - .and_then(|()| { - self.pending_calls - .complete(response) - .map_err(pending_calls_error) - }) - { - let _ = self.fail(error); - } - return; + let write_status = self + .lanes + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) + .map_err(call_lane_pool_error)?; + match write_status { + ControlRingWriteStatus::Written => break, + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch)?; } + } + } + producer.wake_consumer()?; + + let consumer = &mut lane.response_consumer; + let response = loop { + match consumer.try_read(decode_response) { + Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { - if self.pending_calls.current_failure().is_some() { - return; - } - if let Err(error) = consumer.wait_for_message(wait_epoch) { - let _ = self.fail(error); - return; + if let Some(error) = self.lanes.current_failure() { + return Err(copy_io_error(&error)); } + consumer.wait_for_message(wait_epoch)?; } - Err(ControlRingReadError::Ring(error)) => { - let _ = self.fail(ring_error(error)); - return; - } - Err(ControlRingReadError::Decode(error)) => { - let _ = self.fail(wire_error(error)); - return; - } + Err(ControlRingReadError::Ring(error)) => return Err(ring_error(error)), + Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } + }; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer()?; + if response.request_id != request_id { + return Err(invalid_data("broker answered a different request")); } + Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - self.pending_calls.record_failure(Arc::new(error)); - self.request_wake - .interrupt_wait() - .and(self.response_wake.interrupt_wait()) - .and(self.notification_wake.interrupt_wait()) + self.lanes.record_failure(Arc::new(error)); + self.wakes + .iter() + .map(ControlRingWakeHandle::interrupt_wait) + .fold(Ok(()), IoResult::and) .and(self.liveness.shutdown()) } } diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index d796e31b83..1b0959cf50 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -139,25 +139,43 @@ fn wide_string(value: &OsStr) -> IoResult> { mod tests { use super::*; use std::sync::Arc; + use std::thread; use std::time::{Duration, Instant}; - use crate::setup::{OwnedEvent, read_pipe_until_cancelled, write_frame}; + use crate::control_ring::{ + WindowsControlRingHostCallLaneSource, WindowsControlRingHostShutdown, + WindowsControlRingLocalCallChannel, WindowsControlRingLocalNotificationChannel, + }; + use crate::setup::{OwnedEvent, read_frame, read_pipe_until_cancelled, write_frame}; use crate::shared_memory::WindowsSharedMemory; use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, ReadinessNotification, }; use litebox_broker_protocol::readiness::ReadinessFlags; + use litebox_broker_protocol::wire::{ + decode_request, decode_response, encode_handshake_response, encode_request, encode_response, + }; use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, ObjectHandle, RequestId}; use litebox_broker_transport::channel::{ HostNotificationChannel, HostReceive, HostSetupChannel, LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; - use litebox_broker_transport::control_ring::ControlRing; + use litebox_broker_transport::control_ring::{ + BrokerControlRingEndpoints, ControlRing, ControlRingConsumer, ControlRingDemandStatus, + ControlRingLayout, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, + LocalControlRingEndpoints, + }; use litebox_broker_transport::shared_memory::SharedMemory as _; use windows_sys::Win32::System::Threading::GetCurrentProcessId; + type Producer = ControlRingProducer; + type Consumer = ControlRingConsumer; + type Local = LocalControlRingEndpoints; + type Broker = BrokerControlRingEndpoints; + const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; fn pipe_name(suffix: &str) -> std::ffi::OsString { format!(r"\\.\pipe\litebox-broker-test-{}-{suffix}", unsafe { @@ -170,11 +188,11 @@ mod tests { ControlRing, ControlRing, ) { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let host_memory = local_memory.try_clone_view().unwrap(); ( - ControlRing::new(local_memory).unwrap(), - ControlRing::new(host_memory).unwrap(), + ControlRing::new(local_memory, LAYOUT).unwrap(), + ControlRing::new(host_memory, LAYOUT).unwrap(), ) } @@ -201,6 +219,178 @@ mod tests { } } + fn connect_local(name: &OsStr) -> WindowsNamedPipeLocalSetupChannel { + WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline( + name, + Instant::now() + Duration::from_secs(5), + ) + .unwrap() + } + + fn negotiated() -> BrokerHandshakeResponse { + BrokerHandshakeResponse::Negotiated { + broker_protocol_version: BROKER_PROTOCOL_VERSION, + process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, + } + } + + fn send_handshake_request(local: &mut WindowsNamedPipeLocalSetupChannel) { + local + .send_handshake_request(&BrokerHandshakeRequest { + protocol_version: BROKER_PROTOCOL_VERSION, + }) + .unwrap(); + } + + /// Negotiates one real association over a named pipe, so both halves reach + /// their negotiated state through the same handshake production uses. + fn negotiated_pair( + suffix: &str, + ) -> ( + WindowsNamedPipeLocalSetupChannel, + WindowsNamedPipeHostSetupChannel, + ) { + let name = pipe_name(suffix); + let listener = WindowsNamedPipeListener::bind(&name).unwrap(); + let mut local = connect_local(&name); + let mut host = accept_host_guaranteed(listener); + send_handshake_request(&mut local); + assert!(matches!( + host.recv_handshake_request().unwrap(), + HostReceive::Message(BrokerHandshakeRequest { .. }) + )); + host.send_handshake_response(&negotiated()).unwrap(); + assert!(matches!( + local.recv_handshake_response().unwrap(), + Some(BrokerHandshakeResponse::Negotiated { .. }) + )); + (local, host) + } + + /// Activates a host association against raw local ring endpoints and the + /// local half of its pipe. + fn activate_host( + suffix: &str, + ) -> ( + WindowsControlRingHostCallLaneSource, + WindowsControlRingHostShutdown, + Local, + WindowsNamedPipeLocalSetupChannel, + ) { + let (local, host) = negotiated_pair(suffix); + let (local_ring, host_ring) = control_rings(); + write_frame(local.stream.handle(), LAYOUT.ready_token(), None).unwrap(); + let (lanes, _notifications, shutdown) = host.into_active(host_ring).unwrap(); + assert_eq!( + read_frame(local.stream.handle(), None).unwrap().unwrap(), + LAYOUT.ready_token() + ); + (lanes, shutdown, local_ring.into_local(), local) + } + + /// Activates a local association against raw broker ring endpoints and the + /// host half of its pipe. + fn activate_local( + suffix: &str, + ) -> ( + WindowsControlRingLocalCallChannel, + WindowsControlRingLocalNotificationChannel, + Broker, + WindowsNamedPipeStream, + ) { + let name = pipe_name(suffix); + let listener = WindowsNamedPipeListener::bind(&name).unwrap(); + let mut local = connect_local(&name); + let host = accept_stream_guaranteed(listener); + send_handshake_request(&mut local); + read_frame(host.handle(), None).unwrap().unwrap(); + write_frame( + host.handle(), + &encode_handshake_response(negotiated()), + None, + ) + .unwrap(); + local.recv_handshake_response().unwrap().unwrap(); + let (local_ring, broker_ring) = control_rings(); + let (calls, notifications) = thread::scope(|scope| { + scope.spawn(|| { + assert_eq!( + read_frame(host.handle(), None).unwrap().unwrap(), + LAYOUT.ready_token() + ); + write_frame(host.handle(), LAYOUT.ready_token(), None).unwrap(); + }); + local.into_active(local_ring).unwrap() + }); + (calls, notifications, broker_ring.into_broker(), host) + } + + fn demand_lanes(local: &Local, lanes: usize) { + let demand = local.lane_demand.as_ref().unwrap(); + demand.request(lanes).unwrap(); + demand.wake_broker().unwrap(); + } + + fn read_request(consumer: &mut Consumer) -> BrokerRequest { + loop { + match consumer.try_read(decode_request).unwrap() { + ControlRingReadStatus::Message(request) => { + consumer.publish_head().unwrap(); + consumer.wake_producer().unwrap(); + return request; + } + ControlRingReadStatus::Empty { wait_epoch } => { + consumer.wait_for_message(wait_epoch).unwrap(); + } + } + } + } + + fn read_response(consumer: &mut Consumer) -> BrokerResponse { + loop { + match consumer.try_read(decode_response).unwrap() { + ControlRingReadStatus::Message(response) => { + consumer.publish_head().unwrap(); + consumer.wake_producer().unwrap(); + return response; + } + ControlRingReadStatus::Empty { wait_epoch } => { + consumer.wait_for_message(wait_epoch).unwrap(); + } + } + } + } + + fn write_payload(producer: &mut Producer, payload: &[u8]) { + loop { + match producer.try_write(payload).unwrap() { + ControlRingWriteStatus::Written => { + producer.wake_consumer().unwrap(); + return; + } + ControlRingWriteStatus::Full { wait_epoch } => { + producer.wait_for_capacity(wait_epoch).unwrap(); + } + } + } + } + + fn request(id: u64) -> BrokerRequest { + BrokerRequest { + request_id: RequestId(id), + operation: BrokerOperation::CloseObject(ObjectHandle(id)), + } + } + + fn response(id: RequestId) -> BrokerResponse { + BrokerResponse { + request_id: id, + result: BrokerResult::ObjectClosed, + } + } + #[test] fn accept_without_client_would_block() { let mut listener = @@ -343,21 +533,20 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, responses, mut notifications, _shutdown) = - setup.into_active(host_ring).unwrap(); - let HostReceive::Message(request) = requests.recv_request().unwrap() else { + let (mut lanes, mut notifications, _shutdown) = setup.into_active(host_ring).unwrap(); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let HostReceive::Message(request) = lane.recv_request().unwrap() else { panic!("expected active request"); }; assert_eq!( request.operation, BrokerOperation::CloseObject(ObjectHandle(7)) ); - responses - .send_response(&BrokerResponse { - request_id: request.request_id, - result: BrokerResult::ObjectClosed, - }) - .unwrap(); + lane.send_response(&BrokerResponse { + request_id: request.request_id, + result: BrokerResult::ObjectClosed, + }) + .unwrap(); notifications .send_notification(&BrokerNotification::Readiness(ReadinessNotification { handle: ObjectHandle(7), @@ -402,7 +591,7 @@ mod tests { } #[test] - fn concurrent_calls_accept_responses_in_reverse_order() { + fn concurrent_calls_on_separate_lanes_complete_in_reverse_order() { let control_name = pipe_name("multiplex-control"); let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); let (local_ring, host_ring) = control_rings(); @@ -421,21 +610,20 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, responses, _notifications, _shutdown) = - setup.into_active(host_ring).unwrap(); - let HostReceive::Message(first) = requests.recv_request().unwrap() else { - panic!("expected first active request"); - }; - let HostReceive::Message(second) = requests.recv_request().unwrap() else { - panic!("expected second active request"); - }; - for request in [second, first] { - responses - .send_response(&BrokerResponse { - request_id: request.request_id, - result: BrokerResult::ObjectClosed, - }) - .unwrap(); + let (mut lanes, _notifications, _shutdown) = setup.into_active(host_ring).unwrap(); + // The second lane is handed out only once a concurrent call demands it. + let mut lanes = [ + lanes.next_lane().unwrap().unwrap(), + lanes.next_lane().unwrap().unwrap(), + ]; + let requests = lanes.each_mut().map(|lane| { + let HostReceive::Message(request) = lane.recv_request().unwrap() else { + panic!("expected active request"); + }; + request + }); + for (lane, request) in lanes.iter_mut().zip(requests).rev() { + lane.send_response(&response(request.request_id)).unwrap(); } wait_for_responses.recv().unwrap(); }); @@ -452,32 +640,19 @@ mod tests { setup.recv_handshake_response().unwrap().unwrap(); let (calls, _notifications) = setup.into_active(local_ring).unwrap(); let calls = Arc::new(calls); - let callers = [RequestId(11), RequestId(13)].map(|request_id| { + let callers = [11, 13].map(|id| { let calls = Arc::clone(&calls); - std::thread::spawn(move || { - calls - .call(BrokerRequest { - request_id, - operation: BrokerOperation::CloseObject(ObjectHandle(request_id.0)), - }) - .unwrap() - }) + std::thread::spawn(move || calls.call(request(id)).unwrap()) }); - for (caller, request_id) in callers.into_iter().zip([RequestId(11), RequestId(13)]) { - assert_eq!( - caller.join().unwrap(), - BrokerResponse { - request_id, - result: BrokerResult::ObjectClosed, - } - ); + for (caller, id) in callers.into_iter().zip([11, 13]) { + assert_eq!(caller.join().unwrap(), response(RequestId(id))); } responses_received.send(()).unwrap(); host.join().unwrap(); } #[test] - fn host_shutdown_interrupts_blocked_request_receive() { + fn host_shutdown_interrupts_blocked_request_and_lane_waits() { let control_name = pipe_name("shutdown-control"); let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); let (local_ring, host_ring) = control_rings(); @@ -495,16 +670,24 @@ mod tests { startup: None, }) .unwrap(); - let (mut requests, _responses, _notifications, shutdown) = - setup.into_active(host_ring).unwrap(); + let (mut lanes, _notifications, shutdown) = setup.into_active(host_ring).unwrap(); + let mut lane = lanes.next_lane().unwrap().unwrap(); let (reader_started, wait_for_reader) = std::sync::mpsc::sync_channel(0); let reader = std::thread::spawn(move || { reader_started.send(()).unwrap(); - requests.recv_request() + lane.recv_request() }); + let next = std::thread::spawn(move || lanes.next_lane()); wait_for_reader.recv().unwrap(); shutdown.shutdown().unwrap(); - assert!(reader.join().unwrap().is_err()); + assert_eq!( + reader.join().unwrap().unwrap_err().kind(), + ErrorKind::ConnectionAborted + ); + assert_eq!( + next.join().unwrap().err().map(|error| error.kind()), + Some(ErrorKind::ConnectionAborted) + ); }); let deadline = Instant::now() + Duration::from_secs(5); @@ -520,4 +703,194 @@ mod tests { let (_calls, _notifications) = setup.into_active(local_ring).unwrap(); host.join().unwrap(); } + + #[test] + fn single_lane_layouts_are_rejected() { + let layout = ControlRingLayout::SINGLE_LANE; + let (local, host) = negotiated_pair("single-lane"); + let local_memory = WindowsSharedMemory::create_control_ring(layout).unwrap(); + let host_memory = local_memory.try_clone_view().unwrap(); + + let Err(error) = host.into_active(ControlRing::new(host_memory, layout).unwrap()) else { + panic!("the host activated a single-lane layout"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + let Err(error) = local.into_active(ControlRing::new(local_memory, layout).unwrap()) else { + panic!("the local side activated a single-lane layout"); + }; + assert_eq!(error.kind(), ErrorKind::InvalidInput); + } + + #[test] + fn host_hands_out_lanes_on_demand_and_each_lane_serves_its_own_calls() { + let (mut lanes, _shutdown, mut local, peer) = activate_host("lanes-on-demand"); + let mut first = lanes.next_lane().unwrap().unwrap(); + let next = thread::spawn(move || lanes.next_lane().map(|lane| (lane, lanes))); + thread::sleep(Duration::from_millis(20)); + assert!(!next.is_finished()); + demand_lanes(&local, 2); + let (second, mut lanes) = next.join().unwrap().unwrap(); + let mut second = second.unwrap(); + + for (index, lane) in [&mut first, &mut second].into_iter().enumerate() { + let id = u64::try_from(index).unwrap(); + write_payload( + &mut local.lanes[index].request_producer, + &encode_request(request(id)), + ); + let HostReceive::Message(received) = lane.recv_request().unwrap() else { + panic!("expected ring request"); + }; + assert_eq!(received.request_id, RequestId(id)); + lane.send_response(&response(received.request_id)).unwrap(); + } + for (index, lane) in local.lanes.iter_mut().take(2).enumerate() { + assert_eq!( + read_response(&mut lane.response_consumer).request_id, + RequestId(u64::try_from(index).unwrap()) + ); + } + + // Each remaining lane is handed out once; then the source waits until + // the peer closes. + demand_lanes(&local, LAYOUT.lane_count()); + for _ in 2..LAYOUT.lane_count() { + assert!(lanes.next_lane().unwrap().is_some()); + } + let last = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + assert!(!last.is_finished()); + drop(peer); + assert!(last.join().unwrap().unwrap().is_none()); + } + + #[test] + fn host_clean_close_wakes_request_and_lane_waits_as_peer_closed() { + let (mut lanes, _shutdown, _local, peer) = activate_host("clean-close"); + let mut lane = lanes.next_lane().unwrap().unwrap(); + let receiver = thread::spawn(move || lane.recv_request()); + let next = thread::spawn(move || lanes.next_lane()); + thread::sleep(Duration::from_millis(20)); + drop(peer); + assert_eq!(receiver.join().unwrap().unwrap(), HostReceive::PeerClosed); + assert!(next.join().unwrap().unwrap().is_none()); + } + + #[test] + fn concurrent_local_calls_use_separate_lanes() { + let (calls, _notifications, mut broker, _peer) = activate_local("local-concurrent"); + let calls = Arc::new(calls); + let callers = [3, 7].map(|id| { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }); + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + loop { + match demand.try_next_lane().unwrap() { + ControlRingDemandStatus::Lane(lane) => { + assert_eq!(lane, 1); + break; + } + ControlRingDemandStatus::Satisfied { wait_epoch } => { + demand.wait_for_demand(wait_epoch).unwrap(); + } + } + } + let first = read_request(&mut broker.lanes[0].request_consumer); + let second = read_request(&mut broker.lanes[1].request_consumer); + + // The later lane answers first; neither call waits for the other. + write_payload( + &mut broker.lanes[1].response_producer, + &encode_response(response(second.request_id)), + ); + write_payload( + &mut broker.lanes[0].response_producer, + &encode_response(response(first.request_id)), + ); + for caller in callers { + assert!(caller.join().unwrap().is_ok()); + } + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); + } + + #[test] + fn sequential_local_calls_reuse_the_first_lane() { + let (calls, _notifications, mut broker, _peer) = activate_local("local-sequential"); + let calls = Arc::new(calls); + for id in 1..=3 { + let caller = { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }; + let lane = &mut broker.lanes[0]; + let request = read_request(&mut lane.request_consumer); + assert_eq!(request.request_id, RequestId(id)); + write_payload( + &mut lane.response_producer, + &encode_response(response(request.request_id)), + ); + assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(id)); + } + let demand = broker.lane_demand.as_mut().unwrap(); + assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); + assert!(matches!( + demand.try_next_lane(), + Ok(ControlRingDemandStatus::Satisfied { .. }) + )); + } + + #[test] + fn mismatched_and_malformed_responses_fail_closed() { + for payload_kind in 0..3 { + let (calls, _notifications, mut broker, _peer) = + activate_local(&format!("local-bad-response-{payload_kind}")); + let calls = Arc::new(calls); + let callers = [1, 2].map(|id| { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }); + read_request(&mut broker.lanes[0].request_consumer); + let other = read_request(&mut broker.lanes[1].request_consumer); + + let producer = &mut broker.lanes[0].response_producer; + match payload_kind { + 0 => write_payload(producer, &encode_response(response(RequestId(99)))), + 1 => write_payload(producer, &encode_response(response(other.request_id))), + _ => write_payload(producer, &[u8::MAX]), + } + + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + // The association is failed, so it refuses further calls. + assert!(calls.call(request(3)).is_err()); + } + } + + #[test] + fn broker_pipe_close_wakes_waiting_calls() { + let (calls, _notifications, mut broker, peer) = activate_local("local-pipe-close"); + let calls = Arc::new(calls); + let call = |id| { + let calls = Arc::clone(&calls); + thread::spawn(move || calls.call(request(id))) + }; + let callers: Vec<_> = (0..LAYOUT.lane_count() as u64).map(call).collect(); + for lane in &mut broker.lanes { + read_request(&mut lane.request_consumer); + } + // Every lane is held, so this call waits to claim one. + let waiting = call(u64::MAX); + thread::sleep(Duration::from_millis(10)); + drop(peer); + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + assert!(waiting.join().unwrap().is_err()); + } } diff --git a/litebox_broker_transport_windows_userland/src/pending_calls.rs b/litebox_broker_transport_windows_userland/src/pending_calls.rs deleted file mode 100644 index c430dc5820..0000000000 --- a/litebox_broker_transport_windows_userland/src/pending_calls.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::pending_calls::{ - PendingCalls as GenericPendingCalls, PendingCallsCondvar, PendingCallsError, PendingCallsMutex, - PendingCallsSync, -}; - -use crate::setup::{copy_io_error, invalid_data}; - -pub(crate) type PendingCalls = GenericPendingCalls; - -pub(crate) struct StdPendingCallsSync; - -pub(crate) struct StdPendingCallsMutex(Mutex); - -pub(crate) struct StdPendingCallsCondvar(Condvar, PhantomData); - -impl PendingCallsMutex for StdPendingCallsMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker pending mutex poisoned") - } -} - -impl PendingCallsCondvar for StdPendingCallsCondvar { - type Mutex = StdPendingCallsMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0.wait(guard).expect("broker pending mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl PendingCallsSync for StdPendingCallsSync { - type Mutex = StdPendingCallsMutex; - type Condvar = StdPendingCallsCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdPendingCallsMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdPendingCallsCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn pending_calls_error(error: PendingCallsError) -> Error { - match error { - PendingCallsError::AssociationFailed(error) => copy_io_error(&error), - PendingCallsError::Operation(error) => error, - PendingCallsError::DuplicateRequestId => invalid_data("duplicate broker request ID"), - PendingCallsError::UnknownResponseId => { - invalid_data("broker returned an unknown response ID") - } - } -} diff --git a/litebox_broker_transport_windows_userland/src/shared_memory.rs b/litebox_broker_transport_windows_userland/src/shared_memory.rs index e1a46f44bb..8647f0864a 100644 --- a/litebox_broker_transport_windows_userland/src/shared_memory.rs +++ b/litebox_broker_transport_windows_userland/src/shared_memory.rs @@ -8,8 +8,7 @@ use std::ptr::NonNull; use std::sync::Arc; use litebox_broker_transport::control_ring::{ - CONTROL_RING_MEMORY_SIZE, CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingDirection, - MemoryAccessPolicy, WaitableSharedMemory, + CONTROL_RING_PARALLEL_SPIN_LIMIT, ControlRingLayout, MemoryAccessPolicy, WaitableSharedMemory, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_platform::buffer::RawBytes; @@ -29,15 +28,6 @@ use windows_sys::Win32::System::Threading::{ const CONTROL_RING_WAIT_RECHECK_TIMEOUT_MS: u32 = 100; -const CONTROL_RING_WAKE_OFFSETS: [usize; 6] = [ - ControlRingDirection::Requests.producer_epoch_offset(), - ControlRingDirection::Requests.consumer_epoch_offset(), - ControlRingDirection::Responses.producer_epoch_offset(), - ControlRingDirection::Responses.consumer_epoch_offset(), - ControlRingDirection::Notifications.producer_epoch_offset(), - ControlRingDirection::Notifications.consumer_epoch_offset(), -]; - pub(crate) struct OwnedHandle(pub(crate) HANDLE); impl Drop for OwnedHandle { @@ -60,7 +50,9 @@ pub(crate) struct TransferredSharedMemory { /// One mapped view of a page-file-backed Windows section. pub struct WindowsSharedMemory { mapping: Arc, - wake_handles: Option>, + /// One event per waitable control-ring word, in + /// [`ControlRingLayout::waitable_word_index`] order. + wake_handles: Option>, view: NonNull, length: usize, policy: MemoryAccessPolicy, @@ -80,9 +72,12 @@ impl WindowsSharedMemory { Self::create_with_policy(length, MemoryAccessPolicy::Bytes) } - /// Creates and maps shared memory for one control ring. - pub fn create_control_ring() -> IoResult { - Self::create_with_policy(CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy::ControlRing) + /// Creates and maps shared memory for one control ring with `layout`. + pub fn create_control_ring(layout: ControlRingLayout) -> IoResult { + Self::create_with_policy( + layout.memory_size(), + MemoryAccessPolicy::ControlRing(layout), + ) } fn create_with_policy(length: usize, policy: MemoryAccessPolicy) -> IoResult { @@ -118,7 +113,9 @@ impl WindowsSharedMemory { } let wake_handles = match policy { MemoryAccessPolicy::Bytes => None, - MemoryAccessPolicy::ControlRing => Some(Arc::new(create_wake_handles()?)), + MemoryAccessPolicy::ControlRing(layout) => { + Some(create_wake_handles(layout.waitable_word_count())?) + } }; Self::map(Arc::new(OwnedHandle(mapping)), wake_handles, length, policy) } @@ -135,7 +132,7 @@ impl WindowsSharedMemory { fn map( mapping: Arc, - wake_handles: Option>, + wake_handles: Option>, length: usize, policy: MemoryAccessPolicy, ) -> IoResult { @@ -170,7 +167,7 @@ impl WindowsSharedMemory { .ok_or_else(|| Error::other("NtMapViewOfSection returned a null view"))?; let spin_limit = match policy { MemoryAccessPolicy::Bytes => 0, - MemoryAccessPolicy::ControlRing => parallel_spin_limit(), + MemoryAccessPolicy::ControlRing(_) => parallel_spin_limit(), }; Ok(Self { mapping, @@ -223,24 +220,29 @@ impl WindowsSharedMemory { unsafe { Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::Bytes) } } - /// Reconstructs control-ring memory from handles duplicated into the current process. + /// Reconstructs control-ring memory with `layout` from handles duplicated into the current + /// process. /// /// # Safety /// /// Every handle must be live, owned by the caller, and valid in the current process. The first - /// handle must name a mapping of exactly [`CONTROL_RING_MEMORY_SIZE`] bytes, followed by the six - /// event handles in the transport-defined order. + /// handle must name a mapping of exactly [`ControlRingLayout::memory_size`] bytes, followed by + /// one event handle per waitable word of `layout`, in + /// [`ControlRingLayout::waitable_word_index`] order. pub(crate) unsafe fn control_ring_from_transferred( transfer: TransferredSharedMemory, + layout: ControlRingLayout, ) -> IoResult { - if transfer.length != CONTROL_RING_MEMORY_SIZE { + if transfer.length != layout.memory_size() { return Err(Error::new( ErrorKind::InvalidData, "invalid transferred control-ring length", )); } // SAFETY: The caller upholds the handle validity requirements documented above. - unsafe { Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::ControlRing) } + unsafe { + Self::from_transferred_with_policy(transfer, MemoryAccessPolicy::ControlRing(layout)) + } } unsafe fn from_transferred_with_policy( @@ -249,7 +251,7 @@ impl WindowsSharedMemory { ) -> IoResult { let expected_handles = match policy { MemoryAccessPolicy::Bytes => 1, - MemoryAccessPolicy::ControlRing => 1 + CONTROL_RING_WAKE_OFFSETS.len(), + MemoryAccessPolicy::ControlRing(layout) => 1 + layout.waitable_word_count(), }; if transfer.length == 0 || transfer.handles.len() != expected_handles { return Err(Error::new( @@ -269,16 +271,7 @@ impl WindowsSharedMemory { )); } let mapping = Arc::new(handles.remove(0)); - let wake_handles = if handles.is_empty() { - None - } else { - Some(Arc::new(handles.try_into().map_err(|_| { - Error::new( - ErrorKind::InvalidData, - "invalid transferred control-ring wake handle count", - ) - })?)) - }; + let wake_handles = (!handles.is_empty()).then(|| Arc::from(handles)); Self::map(mapping, wake_handles, transfer.length, policy) } @@ -317,16 +310,18 @@ impl WindowsSharedMemory { } fn wake_handle(&self, offset: usize) -> IoResult { - let index = CONTROL_RING_WAKE_OFFSETS - .iter() - .position(|candidate| *candidate == offset) - .ok_or_else(|| { - Error::new(ErrorKind::InvalidInput, "invalid control-ring wait offset") - })?; - self.wake_handles - .as_ref() - .map(|handles| handles[index].0) - .ok_or_else(|| Error::new(ErrorKind::InvalidInput, "shared memory is not waitable")) + let (MemoryAccessPolicy::ControlRing(layout), Some(handles)) = + (self.policy, &self.wake_handles) + else { + return Err(Error::new( + ErrorKind::InvalidInput, + "shared memory is not waitable", + )); + }; + let index = layout.waitable_word_index(offset).ok_or_else(|| { + Error::new(ErrorKind::InvalidInput, "invalid control-ring wait offset") + })?; + Ok(handles[index].0) } } @@ -339,19 +334,18 @@ fn parallel_spin_limit() -> u32 { } } -fn create_wake_handles() -> IoResult<[OwnedHandle; CONTROL_RING_WAKE_OFFSETS.len()]> { - let mut handles = Vec::with_capacity(CONTROL_RING_WAKE_OFFSETS.len()); - for _ in CONTROL_RING_WAKE_OFFSETS { - // SAFETY: The event is unnamed, auto-reset, initially nonsignaled, and has no custom ACL. - let handle = unsafe { CreateEventW(std::ptr::null(), 0, 0, std::ptr::null()) }; - if handle.is_null() { - return Err(Error::last_os_error()); - } - handles.push(OwnedHandle(handle)); - } - handles - .try_into() - .map_err(|_| Error::other("incorrect control-ring wake handle count")) +fn create_wake_handles(count: usize) -> IoResult> { + (0..count) + .map(|_| { + // SAFETY: The event is unnamed, auto-reset, initially nonsignaled, and has no custom + // ACL. + let handle = unsafe { CreateEventW(std::ptr::null(), 0, 0, std::ptr::null()) }; + if handle.is_null() { + return Err(Error::last_os_error()); + } + Ok(OwnedHandle(handle)) + }) + .collect() } /// Duplicates `handle` into `target_process` with `access`, or with the same access if `None`, @@ -558,7 +552,7 @@ impl WaitableSharedMemory for WindowsSharedMemory { mod tests { use super::*; use litebox_broker_transport::control_ring::{ - ControlRing, ControlRingReadStatus, ControlRingWriteStatus, + ControlRing, ControlRingDemandStatus, ControlRingReadStatus, ControlRingWriteStatus, }; use windows_sys::Win32::System::Memory::{ FILE_MAP_ALL_ACCESS, MEM_COMMIT, MEM_FREE, MEMORY_BASIC_INFORMATION, MapViewOfFile, @@ -641,13 +635,14 @@ mod tests { #[test] fn native_clones_do_not_inherit_association_views() { let pool = WindowsSharedMemory::create(64).unwrap(); - let ring = WindowsSharedMemory::create_control_ring().unwrap(); + let ring = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let transfer = ring .duplicate_to_process(unsafe { GetCurrentProcess() }) .unwrap(); // SAFETY: DuplicateHandle created every transferred handle in this process. let received_ring = - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) }.unwrap(); + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, LAYOUT) } + .unwrap(); // SAFETY: `pool.mapping` is a live section handle at least 64 bytes long. let inherited = unsafe { MapViewOfFile(pool.mapping.0, FILE_MAP_ALL_ACCESS, 0, 0, 64) }; assert!(!inherited.Value.is_null(), "{}", Error::last_os_error()); @@ -672,19 +667,26 @@ mod tests { assert_eq!(byte, [5]); } + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + #[test] fn cloned_views_share_control_ring_messages() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = local_memory.try_clone_view().unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); + let mut broker = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_broker(); assert_eq!( - local.request_producer.try_write(b"request").unwrap(), + local.lanes[0] + .request_producer + .try_write(b"request") + .unwrap(), ControlRingWriteStatus::Written ); assert_eq!( - broker + broker.lanes[0] .request_consumer .try_read(|frame| Ok::<_, ()>(frame.to_vec())), Ok(ControlRingReadStatus::Message(b"request".to_vec())) @@ -693,10 +695,21 @@ mod tests { #[test] fn cloned_views_wake_blocked_control_ring_consumer() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = local_memory.try_clone_view().unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + // The last lane waits on different events than lane 0. + let mut local = ControlRing::new(local_memory, LAYOUT) + .unwrap() + .into_local() + .lanes + .pop() + .unwrap(); + let mut broker = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_broker() + .lanes + .pop() + .unwrap(); let wait_epoch = match broker.request_consumer.try_read(|_| Ok::<_, ()>(())) { Ok(ControlRingReadStatus::Empty { wait_epoch }) => wait_epoch, @@ -722,10 +735,64 @@ mod tests { writer.join().unwrap(); } + #[test] + fn cloned_views_wake_broker_waiting_for_lane_demand() { + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); + let broker_memory = local_memory.try_clone_view().unwrap(); + let local_demand = ControlRing::new(local_memory, LAYOUT) + .unwrap() + .into_local() + .lane_demand + .unwrap(); + let mut broker_demand = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_broker() + .lane_demand + .unwrap(); + + assert_eq!( + broker_demand.try_next_lane(), + Ok(ControlRingDemandStatus::Lane(0)) + ); + let demander = std::thread::spawn(move || { + local_demand.request(2).unwrap(); + local_demand.wake_broker().unwrap(); + }); + loop { + match broker_demand.try_next_lane().unwrap() { + ControlRingDemandStatus::Lane(lane) => { + assert_eq!(lane, 1); + break; + } + ControlRingDemandStatus::Satisfied { wait_epoch } => { + broker_demand.wait_for_demand(wait_epoch).unwrap(); + } + } + } + demander.join().unwrap(); + } + + #[test] + fn every_waitable_control_ring_word_has_its_own_event() { + for layout in [ + ControlRingLayout::SINGLE_LANE, + ControlRingLayout::MULTI_LANE, + ] { + let memory = WindowsSharedMemory::create_control_ring(layout).unwrap(); + let mut events = (0..layout.memory_size()) + .filter(|offset| layout.waitable_word_index(*offset).is_some()) + .map(|offset| memory.wake_handle(offset).unwrap()) + .collect::>(); + events.sort_unstable(); + events.dedup(); + assert_eq!(events.len(), layout.waitable_word_count()); + } + } + #[test] fn event_wait_returns_without_peer_cooperation() { - let memory = WindowsSharedMemory::create_control_ring().unwrap(); - let epoch_offset = CONTROL_RING_WAKE_OFFSETS[0]; + let memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); + let epoch_offset = LAYOUT.requests(0).producer_epoch_offset(); let start = std::time::Instant::now(); memory.wait_while_equal(epoch_offset, 0).unwrap(); @@ -735,8 +802,8 @@ mod tests { #[test] fn byte_memory_does_not_infer_control_ring_from_length() { - let memory = WindowsSharedMemory::create(CONTROL_RING_MEMORY_SIZE).unwrap(); - let epoch_offset = CONTROL_RING_WAKE_OFFSETS[0]; + let memory = WindowsSharedMemory::create(LAYOUT.memory_size()).unwrap(); + let epoch_offset = LAYOUT.requests(0).producer_epoch_offset(); assert_eq!( memory.wake_one(epoch_offset).unwrap_err().kind(), @@ -752,23 +819,30 @@ mod tests { #[test] fn duplicated_handles_reconstruct_control_ring_view() { - let local_memory = WindowsSharedMemory::create_control_ring().unwrap(); + let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let transfer = local_memory .duplicate_to_process(unsafe { GetCurrentProcess() }) .unwrap(); + assert_eq!(transfer.handles.len(), 1 + LAYOUT.waitable_word_count()); // SAFETY: DuplicateHandle created every transferred handle in this process, and the source // mapping remains alive for the duration of the reconstructed view. let broker_memory = - unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer) }.unwrap(); - let mut local = ControlRing::new(local_memory).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory).unwrap().into_broker(); + unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, LAYOUT) } + .unwrap(); + let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); + let mut broker = ControlRing::new(broker_memory, LAYOUT) + .unwrap() + .into_broker(); assert_eq!( - local.request_producer.try_write(b"transferred").unwrap(), + local.lanes[0] + .request_producer + .try_write(b"transferred") + .unwrap(), ControlRingWriteStatus::Written ); assert_eq!( - broker + broker.lanes[0] .request_consumer .try_read(|frame| Ok::<_, ()>(frame.to_vec())), Ok(ControlRingReadStatus::Message(b"transferred".to_vec())) diff --git a/litebox_broker_userland/src/lib.rs b/litebox_broker_userland/src/lib.rs index a63d519c89..927a694ccb 100644 --- a/litebox_broker_userland/src/lib.rs +++ b/litebox_broker_userland/src/lib.rs @@ -30,10 +30,3 @@ pub mod random; pub mod stdio; mod terminal; mod timer; - -/// The maximum number of worker threads that serve one association's requests. -/// -/// An association starts with a single worker, its own thread. Another worker is started only -/// when a request is about to wait, such as for a child process to start, and no worker is idle -/// to receive the next request. -const WORKER_COUNT: usize = 8; diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 4c2951817f..a2cac1b128 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -6,23 +6,22 @@ //! This module owns association setup, request dispatch, readiness //! publication, cancellation, and teardown for one broker association. //! It is transport-neutral: it depends only on the [`HostSetupChannel`], -//! [`HostRequestSource`], [`HostResponseSink`], [`HostNotificationChannel`], -//! and [`HostAssociationShutdown`] contracts from +//! [`HostCallLaneSource`], [`HostCallLane`], [`HostNotificationChannel`], and +//! [`HostAssociationShutdown`] contracts from //! `litebox_broker_transport::channel`, so a deployment supplies the //! transport-specific setup channel, activation, and shared-memory plumbing //! and this runtime serves the association the same way regardless of //! transport. //! -//! Concurrency and worker sizing are deliberately not part of the public -//! surface. Both association entry points delegate to one internal runtime -//! that owns the worker count. +//! Concurrency is deliberately not part of the public surface. Both +//! association entry points delegate to one internal runtime that chooses the +//! control-ring layout and serves each of its call lanes with one worker. use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::sync::{ - Arc, Mutex, MutexGuard, + Arc, Mutex, atomic::{AtomicBool, Ordering}, }; -use std::thread::{JoinHandle, Thread}; use litebox_broker_core::BrokerCore; use litebox_broker_host::{ @@ -30,13 +29,12 @@ use litebox_broker_host::{ setup_connection, }; use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest, ProcessRequest}; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ - HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, - HostResponseSink, HostSetupChannel, + HostAssociationShutdown, HostCallLane, HostCallLaneSource, HostNotificationChannel, + HostReceive, HostSetupChannel, }; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; use crate::process_launcher::{PendingRunnerAssociation, UserlandProcessLauncher}; @@ -62,10 +60,11 @@ pub(crate) fn is_peer_closed_error(error: &IoError) -> bool { /// `control_channel` must already be configured with whatever deadline and /// peer authentication its transport requires; this function only negotiates /// the broker protocol and control-ring setup over it. `create_shared_memory` -/// and `create_control_memory` allocate the transport's shared-memory -/// resources, `send_shared_memory` transfers them to the peer, and `activate` -/// consumes the negotiated setup channel and control ring into the transport's -/// active request, response, notification, and shutdown endpoints. +/// and `create_control_memory` (for the given control-ring layout) allocate +/// the transport's shared-memory resources, `send_shared_memory` transfers +/// them to the peer, and `activate` consumes the negotiated setup channel and +/// control ring into the transport's active call lanes, notification, and +/// shutdown endpoints. /// /// Returns once the association ends, whether by a clean peer close or a /// failure. Layout mismatches and broker setup failures are mapped to a @@ -73,28 +72,27 @@ pub(crate) fn is_peer_closed_error(error: &IoError) -> bool { pub fn serve_in_process_runner_association< Memory, SetupChannel, - RequestSource, - ResponseSink, + LaneSource, NotificationChannel, Shutdown, >( broker: &BrokerCore, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, ) -> IoResult<()> where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { serve_association_inner( broker, @@ -117,8 +115,7 @@ where pub(crate) fn serve_out_of_process_runner_association< Memory, SetupChannel, - RequestSource, - ResponseSink, + LaneSource, NotificationChannel, Shutdown, >( @@ -126,21 +123,21 @@ pub(crate) fn serve_out_of_process_runner_association< broker: BrokerCore, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, launcher: Arc, ) -> AssociationOutcome where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { let outcome = serve_association_inner( &broker, @@ -168,33 +165,26 @@ where } #[allow(clippy::too_many_arguments)] -fn serve_association_inner< - Memory, - SetupChannel, - RequestSource, - ResponseSink, - NotificationChannel, - Shutdown, ->( +fn serve_association_inner( broker: &BrokerCore, mut control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce(ControlRingLayout) -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, activate: impl FnOnce( SetupChannel, ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + ) -> IoResult<(LaneSource, NotificationChannel, Shutdown)>, launcher: Option>, startup: Option, ) -> IoResult where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { let (process, startup) = match startup { Some(startup) => { @@ -209,8 +199,9 @@ where SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?, ); - let control_memory = create_control_memory()?; - let control_ring = ControlRing::new(control_memory) + let layout = ControlRingLayout::MULTI_LANE; + let control_memory = create_control_memory(layout)?; + let control_ring = ControlRing::new(control_memory, layout) .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; let readiness = Arc::new(ReadinessPublisherRuntime::new()); let association = match setup_connection( @@ -245,7 +236,7 @@ where )); } }; - let (request_source, response_sink, notification_channel, shutdown) = + let (lane_source, notification_channel, shutdown) = match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { @@ -258,8 +249,7 @@ where Ok(dispatch_requests( association, readiness, - request_source, - response_sink, + lane_source, notification_channel, shutdown, launcher, @@ -377,12 +367,12 @@ impl> /// Fails the association if readiness publication unwinds. /// -/// The request reader owns association termination but does not depend on the -/// publisher, so an unwinding publisher would otherwise leave a live +/// The association thread owns association termination but does not depend on +/// the publisher, so an unwinding publisher would otherwise leave a live /// association with no notification source. The join that turns that panic into -/// a reported failure is reached only once the reader has returned, and a peer -/// that is waiting for a readiness change it will never be told about does not -/// return it. Failing the association here ends that wait instead. +/// a reported failure is reached only once the association has ended, and a +/// peer that is waiting for a readiness change it will never be told about does +/// not end it. Failing the association here ends it instead. struct PublisherPanicGuard<'association, Shutdown: HostAssociationShutdown> { failure_coordinator: &'association HostAssociationFailureCoordinator, } @@ -404,9 +394,10 @@ impl> Drop /// panic out of the association, and both states it can rest in have to end for /// that join to complete. Closing publication returns a publisher parked for /// work, and ending the transport returns one blocked on notification capacity -/// that a local endpoint stopped draining. The failure coordinator owns the -/// association until `dispatch_requests` returns, which is after that join, so -/// an unwind cannot leave ending the transport to dropping it. +/// that a local endpoint stopped draining, as well as any worker waiting for a +/// request. The failure coordinator owns the association until +/// `dispatch_requests` returns, which is after that join, so an unwind cannot +/// leave ending the transport to dropping it. struct ReadinessPublicationGuard<'association, Shutdown: HostAssociationShutdown> { readiness: &'association ReadinessPublisherRuntime, failure_coordinator: &'association HostAssociationFailureCoordinator, @@ -434,15 +425,26 @@ impl Drop for AssociationCancellationGuard<'_, Memory> { /// Serves one association until it ends, then reports its first failure. /// +/// Each call lane has its own worker from the runner's first use of it, so the +/// association has as many workers as the runner has had calls in flight at +/// once. A worker serves only its own lane, so a call never waits for another +/// call to finish, and back-to-back calls stay on the first lane's worker. +/// +/// This thread is the first lane's worker. Having set the association up with +/// the runner, it tends to run where the runner's writes reach it fastest, +/// whereas the scheduler places a new thread on the least busy CPUs, which can +/// be across a cache boundary from the runner. An observer thread starts the +/// workers of further lanes, observes the end of the association, and cancels +/// the requests still executing; this thread then joins every worker. +/// /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network /// reactor is currently its production source. #[allow(clippy::too_many_arguments)] -fn dispatch_requests( +fn dispatch_requests( association: BrokerHostAssociation, readiness: Arc, - request_source: RequestSource, - response_sink: ResponseSink, + mut lane_source: LaneSource, mut notification_channel: NotificationChannel, shutdown: Shutdown, launcher: Option>, @@ -450,13 +452,12 @@ fn dispatch_requests AssociationOutcome where Memory: SharedMemory, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, + LaneSource: HostCallLaneSource + Send, + LaneSource::Lane: Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, + Shutdown: HostAssociationShutdown + Sync, { - let association = Arc::new(association); - let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(shutdown)); + let failure_coordinator = HostAssociationFailureCoordinator::new(shutdown); if let Err(error) = association.activate_process() { return AssociationOutcome { result: Err(IoError::other(format!( @@ -465,31 +466,26 @@ where abnormal: true, }; } - let workers = Arc::new(Workers { - association: Arc::clone(&association), - requests: Mutex::new(request_source), - response_sink, - failure_coordinator: Arc::clone(&failure_coordinator), - launcher, - role: ReceivingRole::new(), - }); + let context = AssociationContext { + association: &association, + failure_coordinator: &failure_coordinator, + launcher: launcher.as_ref(), + }; std::thread::scope(|scope| { - let publisher_readiness = Arc::clone(&readiness); - let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() .name("litebox-broker-notifier".to_owned()) - .spawn_scoped(scope, move || { + .spawn_scoped(scope, || { let _panicking = PublisherPanicGuard { - failure_coordinator: &publisher_failure_coordinator, + failure_coordinator: &failure_coordinator, }; - // The request workers own association termination. A failing - // notification transport must fail the association before - // returning an error, so a worker still receiving observes and - // reports the same failure. Reporting here would instead turn - // a clean peer close into an error when its transport teardown - // releases a blocked notification send. - let _ = publisher_readiness.run(&mut notification_channel); + // The association thread owns association termination. A + // failing notification transport must fail the association + // before returning an error, so the association thread + // observes and reports the same failure. Reporting here would + // instead turn a clean peer close into an error when its + // transport teardown releases a blocked notification send. + let _ = readiness.run(&mut notification_channel); }); let publisher = match publisher { Ok(publisher) => Some(publisher), @@ -507,26 +503,40 @@ where readiness: &readiness, failure_coordinator: &failure_coordinator, }; - let cancellation = AssociationCancellationGuard { - association: &association, + let workers = match context.next_lane(&mut lane_source) { + Some(first_lane) => { + let observer = std::thread::Builder::new() + .name("litebox-broker-lanes".to_owned()) + .spawn_scoped(scope, move || { + context.serve_further_lanes(scope, lane_source) + }); + match observer { + Ok(observer) => { + context.serve(first_lane); + observer.join().unwrap_or_else(|_| { + failure_coordinator + .report_panic(IoError::other("broker call-lane observer panicked")); + Vec::new() + }) + } + Err(error) => { + failure_coordinator.report(error); + Vec::new() + } + } + } + None => Vec::new(), }; - - // This thread is the first worker; others start when requests wait. - workers.run(false); - drop(cancellation); - for worker in workers.role.take_started() { + association.association_ending(); + for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); } } - // Readiness publication lives exactly as long as the association. The - // request reader returns only once the association is over, but workers - // keep draining already-queued requests after that, so publication must - // outlive them or a late readiness change would be discarded. Ending it + // Readiness publication lives exactly as long as the association, and + // workers can still change readiness until they are joined. Ending it // here rather than leaving it to the scope orders it before the join - // that observes a panicking publisher, and dropping the guard is what - // ends both states the publisher can rest in without depending on the - // reader having failed the association already. + // that observes a panicking publisher. drop(publication); if let Some(publisher) = publisher && publisher.join().is_err() @@ -535,14 +545,10 @@ where } }); - drop(workers); let result = match failure_coordinator.take_error() { Some(error) => Err(error), None => Ok(()), }; - let Ok(association) = Arc::try_unwrap(association) else { - panic!("all broker association workers must be joined before teardown"); - }; let panicked = failure_coordinator.panicked(); let abnormal = failure_coordinator.abnormal(); if panicked || !finish_process { @@ -553,242 +559,119 @@ where AssociationOutcome { result, abnormal } } -/// The workers serving one association's requests. -/// -/// The association's own thread is the first worker; [`ReceivingRole::pass`] -/// starts others only when a request is about to wait while no worker is -/// idle. -struct Workers { - association: Arc>, - /// Locked only by the worker holding `role`. - requests: Mutex, - response_sink: ResponseSink, - failure_coordinator: Arc>, - launcher: Option>, - role: ReceivingRole, -} - -/// The right to receive an association's next request. -/// -/// One worker at a time receives, and it keeps the role while it executes the -/// request it received, so the requests of a mostly sequential peer keep going -/// to one worker with a warm cache instead of rotating through ones that have -/// gone cold. A worker passes the role on only before executing a request that -/// waits (see [`Workers::run`]), so a slow request does not hold up later ones. -struct ReceivingRole { - state: Mutex, -} - -struct Receivers { - /// Whether the receiving worker has observed the end of the requests. - ended: bool, - /// Whether a worker holds the receiving role. - held: bool, - /// Workers waiting for the receiving role, most recently idle last. - /// - /// A worker waits only while another one holds the role, so this is empty - /// whenever `held` is false. - idle: Vec, - /// Workers started by [`ReceivingRole::pass`]. - started: Vec>, +/// What the threads serving an association share. +struct AssociationContext<'association, Memory: SharedMemory, Shutdown> { + association: &'association BrokerHostAssociation, + failure_coordinator: &'association HostAssociationFailureCoordinator, + launcher: Option<&'association Arc>, } -impl ReceivingRole { - fn new() -> Self { - Self { - state: Mutex::new(Receivers { - ended: false, - held: false, - idle: Vec::with_capacity(crate::WORKER_COUNT), - started: Vec::with_capacity(crate::WORKER_COUNT), - }), - } +impl Clone for AssociationContext<'_, Memory, Shutdown> { + fn clone(&self) -> Self { + *self } +} - fn state(&self) -> MutexGuard<'_, Receivers> { - self.state - .lock() - .expect("broker receiving role mutex poisoned") - } +impl Copy for AssociationContext<'_, Memory, Shutdown> {} - /// Waits for the role, or returns `false` once requests have ended. - fn take(&self) -> bool { - let mut state = self.state(); - if state.ended { - return false; - } - if !state.held { - state.held = true; - return true; - } - let worker = std::thread::current(); - let id = worker.id(); - state.idle.push(worker); - loop { - if state.ended { - return false; +impl AssociationContext<'_, Memory, Shutdown> +where + Memory: SharedMemory, + Shutdown: HostAssociationShutdown, +{ + /// Returns the next lane the runner uses, or `None` once the association + /// has ended, reporting why it ended abnormally. + fn next_lane>( + self, + lane_source: &mut LaneSource, + ) -> Option { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| lane_source.next_lane())) { + Ok(Ok(lane)) => lane, + Ok(Err(error)) => { + self.failure_coordinator.report(error); + None } - // Only a worker passing the role removes a waiting one. - if !state.idle.iter().any(|idle| idle.id() == id) { - return true; + Err(_) => { + self.failure_coordinator + .report_panic(IoError::other("broker call-lane source panicked")); + None } - drop(state); - std::thread::park(); - state = self.state(); - } - } - - /// Passes the role to the most recently idle worker or, while there are - /// fewer than [`crate::WORKER_COUNT`] workers, to one that `start` starts - /// already holding it given its index. - /// - /// Without either, the role is left for the next worker to finish. - fn pass(&self, start: impl FnOnce(usize) -> IoResult>) { - let mut state = self.state(); - if let Some(next) = state.idle.pop() { - drop(state); - next.unpark(); - return; - } - let index = state.started.len() + 1; - // Starting under the lock records the new worker before it can end - // the requests, so the association thread joins it. - if index < crate::WORKER_COUNT - && let Ok(worker) = start(index) - { - state.started.push(worker); - } else { - state.held = false; } } - /// Ends receiving and returns every idle worker. - fn end(&self) { - let idle = { - let mut state = self.state(); - state.ended = true; - std::mem::take(&mut state.idle) + /// Starts a worker for each further lane the runner uses until the + /// association ends, then cancels the requests still executing and returns + /// the workers to join. + fn serve_further_lanes<'scope, LaneSource>( + self, + scope: &'scope std::thread::Scope<'scope, '_>, + mut lane_source: LaneSource, + ) -> Vec> + where + LaneSource: HostCallLaneSource, + LaneSource::Lane: Send + 'scope, + Self: Send + 'scope, + { + // Cancelling on an unwind too keeps the scope from waiting forever on + // a worker blocked in a request. + let cancellation = AssociationCancellationGuard { + association: self.association, }; - for worker in idle { - worker.unpark(); + let mut workers = Vec::new(); + while let Some(lane) = self.next_lane(&mut lane_source) { + let started = std::thread::Builder::new() + .name(format!("litebox-broker-worker-{}", workers.len() + 1)) + .spawn_scoped(scope, move || self.serve(lane)); + match started { + Ok(started) => workers.push(started), + Err(error) => { + self.failure_coordinator.report(error); + break; + } + } } + drop(cancellation); + workers } - /// Takes the workers [`Self::pass`] started. - /// - /// Once requests have ended, no worker holds the role to start another. - fn take_started(&self) -> Vec> { - let mut state = self.state(); - debug_assert!(state.ended, "broker requests must end before joining"); - std::mem::take(&mut state.started) - } -} - -impl - Workers -where - Memory: SharedMemory, - RequestSource: HostRequestSource + Send + 'static, - ResponseSink: HostResponseSink + Send + Sync + 'static, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, -{ - /// Serves requests until the association's requests end, starting with - /// the receiving role if `holds_role`. - /// - /// Every worker takes its turn receiving and executes each request it - /// receives. Once every worker is busy, requests wait in the transport - /// until one finishes. - fn run(self: &Arc, mut holds_role: bool) { - loop { - if !holds_role && !self.role.take() { - return; - } - let Some(request) = self.next_request() else { - return; - }; - // Starting a child process waits for its runner's setup, so another - // worker receives meanwhile. Other requests wait at most briefly for - // broker threads such as the socket reactor, and never for the - // guest, which learns of readiness through notifications instead. - holds_role = !matches!( - request.operation, - BrokerOperation::Process( - ProcessRequest::StartChild(_) | ProcessRequest::StartChildWithExistingRunner(_) - ) - ); - if !holds_role { - self.pass_role(); - } - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.association.execute_request_with( + /// Serves one lane's requests until the association ends. + fn serve>(self, mut lane: Lane) { + let served = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + while !self.failure_coordinator.failed() { + let request = match lane.recv_request() { + Ok(HostReceive::Message(request)) => request, + Ok(HostReceive::PeerClosed) => return, + Ok(HostReceive::ProtocolViolation) => { + self.failure_coordinator.report(IoError::new( + ErrorKind::InvalidData, + "runner sent a request for the wrong protocol phase", + )); + return; + } + Err(error) => { + self.failure_coordinator.report(error); + return; + } + }; + match self.association.execute_request_with( request, |process, operation, shared_buffers| { - self.launcher.as_ref().and_then(|launcher| { + self.launcher.and_then(|launcher| { handle_process_operation(launcher, process, operation, shared_buffers) }) }, - |response| self.response_sink.send_response(response), - ) - })) { - Ok(Ok(()) | Err(BrokerHostError::AssociationFailed)) => {} - Ok(Err(error)) => self.failure_coordinator.report(map_host_error(error)), - Err(_) => { - self.failure_coordinator - .report_panic(IoError::other("broker request worker panicked")); + |response| lane.send_response(response), + ) { + Ok(()) | Err(BrokerHostError::AssociationFailed) => {} + Err(error) => self.failure_coordinator.report(map_host_error(error)), } } + })); + if served.is_err() { + self.failure_coordinator + .report_panic(IoError::other("broker request worker panicked")); } } - - /// Passes the receiving role on before executing a request that waits. - /// - /// See [`ReceivingRole::pass`] for which worker takes it. - fn pass_role(self: &Arc) { - self.role.pass(|index| { - let workers = Arc::clone(self); - std::thread::Builder::new() - .name(format!("litebox-broker-worker-{index}")) - .spawn(move || workers.run(true)) - }); - } - - /// Receives the next request while holding the receiving role, or returns - /// `None` once the association's requests have ended. - /// - /// The worker that observes the end reports why and starts association - /// teardown, so workers still executing requests see cancellation - /// promptly. - fn next_request(&self) -> Option { - let received = (!self.failure_coordinator.failed()).then(|| { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.requests - .lock() - .expect("broker request source mutex poisoned") - .recv_request() - })) - }); - if let Some(Ok(Ok(HostReceive::Message(request)))) = received { - return Some(request); - } - self.role.end(); - match received { - Some(Ok(Ok(HostReceive::ProtocolViolation))) => { - self.failure_coordinator.report(IoError::new( - ErrorKind::InvalidData, - "runner sent a request for the wrong protocol phase", - )); - } - Some(Ok(Err(error))) => self.failure_coordinator.report(error), - Some(Err(_)) => { - self.failure_coordinator - .report_panic(IoError::other("broker request reader panicked")); - } - Some(Ok(Ok(HostReceive::PeerClosed | HostReceive::Message(_)))) | None => {} - } - self.association.request_cancellation(); - self.association.association_ending(); - None - } } #[cfg(all(test, target_os = "linux"))] @@ -807,16 +690,16 @@ mod tests { use litebox_broker_protocol::message::{BrokerHandshakeResponse, BrokerNotification}; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_transport::channel::{ - HostNotificationChannel, HostReceive, HostSetupChannel, LocalSetupChannel, + HostNotificationChannel, HostSetupChannel, LocalSetupChannel, }; - use litebox_broker_transport::control_ring::ControlRing; + use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; use litebox_broker_transport::shared_memory::SharedBufferPool; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostRequestSource, - UnixControlRingHostResponseSink, UnixControlRingHostShutdown, - UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, - UnixControlRingLocalShutdown, UnixStreamHostSetupChannel, UnixStreamLocalSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixControlRingLocalCallChannel, + UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, + UnixStreamHostSetupChannel, UnixStreamLocalSetupChannel, }; use super::*; @@ -826,6 +709,8 @@ mod tests { /// the userland binary chooses for real runner processes. const TEST_SETUP_TIMEOUT: Duration = Duration::from_secs(5); + const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + #[test] fn host_errors_preserve_io_categories() { for (error, expected) in [ @@ -863,10 +748,9 @@ mod tests { /// One live host association: the endpoints teardown acts on, and the rest /// held open so the association stays up for the duration of a test. struct LiveAssociation { - request_source: UnixControlRingHostRequestSource, + lane_source: UnixControlRingHostCallLaneSource, notifications: UnixControlRingHostNotificationChannel, shutdown: UnixControlRingHostShutdown, - _response_sink: UnixControlRingHostResponseSink, _local: ( UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -887,26 +771,42 @@ mod tests { }) .unwrap(); local_setup.recv_handshake_response().unwrap().unwrap(); - let local_memory = MemfdSharedMemory::create_control_ring().unwrap(); + let local_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); let host_memory = MemfdSharedMemory::control_ring_from_received_fd( local_memory.as_fd().try_clone_to_owned().unwrap(), + LAYOUT, ) .unwrap(); - let local_ring = ControlRing::new(local_memory).unwrap(); - let host_ring = ControlRing::new(host_memory).unwrap(); + let local_ring = ControlRing::new(local_memory, LAYOUT).unwrap(); + let host_ring = ControlRing::new(host_memory, LAYOUT).unwrap(); let local_activation = std::thread::spawn(move || local_setup.into_active(local_ring, || {}).unwrap()); - let (request_source, response_sink, notifications, shutdown) = + let (lane_source, notifications, shutdown) = control_channel.into_active(host_ring).unwrap(); LiveAssociation { - request_source, + lane_source, notifications, shutdown, - _response_sink: response_sink, _local: local_activation.join().unwrap(), } } + /// Waits for a call lane beyond the always-demanded first one, as the lane + /// observer does. + fn wait_for_more_lanes( + mut lane_source: UnixControlRingHostCallLaneSource, + ) -> Receiver> { + let first = lane_source.next_lane().unwrap().unwrap(); + let (result_sender, result_receiver) = sync_channel(1); + std::thread::spawn(move || { + let _first = first; + result_sender + .send(lane_source.next_lane().map(|lane| lane.is_some())) + .unwrap(); + }); + result_receiver + } + /// A notification channel that accepts every send and keeps nothing. struct DiscardingChannel; @@ -931,13 +831,14 @@ mod tests { UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - IoError::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = + ControlRing::new(control_memory, LAYOUT).map_err(|error| { + IoError::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; let (call_channel, notifications, shutdown) = setup.into_active(control_ring, || {})?; Ok(( @@ -979,8 +880,8 @@ mod tests { let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let shared_buffers = Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let control_ring = ControlRing::new(control_memory).unwrap(); + let control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let control_ring = ControlRing::new(control_memory, LAYOUT).unwrap(); let mut control = UnixStreamHostSetupChannel::from_host_guaranteed( host_stream, Instant::now() + TEST_SETUP_TIMEOUT, @@ -1000,16 +901,14 @@ mod tests { ) .unwrap() .unwrap(); - let (request_source, response_sink, notifications, shutdown) = - control.into_active(control_ring).unwrap(); + let (lane_source, notifications, shutdown) = control.into_active(control_ring).unwrap(); started.recv().unwrap(); outcome_sender .send( dispatch_requests( association, readiness, - request_source, - response_sink, + lane_source, notifications, shutdown, None, @@ -1103,18 +1002,14 @@ mod tests { } #[test] - fn a_panicking_publisher_ends_a_blocked_request_reader() { + fn a_panicking_publisher_ends_the_wait_for_lanes() { let association = live_association(); - let mut request_source = association.request_source; let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(association.shutdown)); - let (result_sender, result_receiver) = sync_channel(1); - let reader = std::thread::spawn(move || { - result_sender.send(request_source.recv_request()).unwrap(); - }); + let result_receiver = wait_for_more_lanes(association.lane_source); - // The peer sends nothing and never closes, so the reader returns only if - // the publisher's unwind fails the association. + // The peer sends nothing and never closes, so the wait ends only if the + // publisher's unwind fails the association. let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::spawn(move || { let _panicking = PublisherPanicGuard { @@ -1123,38 +1018,26 @@ mod tests { panic!("readiness publication panicked"); }); - let receive_result = result_receiver + let wait_result = result_receiver .recv_timeout(TEST_SETUP_TIMEOUT) - .expect("a panicking publisher must end a blocked request reader"); - assert!(matches!( - receive_result, - Ok(HostReceive::PeerClosed) | Err(_) - )); - reader.join().unwrap(); + .expect("a panicking publisher must end the wait for lanes"); + assert!(matches!(wait_result, Ok(false) | Err(_))); assert!(publisher.join().is_err()); assert!(failure_coordinator.panicked()); assert!(failure_coordinator.take_error().is_some()); } #[test] - fn first_failure_is_preserved_and_unblocks_request_reading() { + fn first_failure_is_preserved_and_ends_the_wait_for_lanes() { let association = live_association(); - let mut request_source = association.request_source; let failure_coordinator = HostAssociationFailureCoordinator::new(association.shutdown); - let (result_sender, result_receiver) = sync_channel(1); - let reader = std::thread::spawn(move || { - result_sender.send(request_source.recv_request()).unwrap(); - }); + let result_receiver = wait_for_more_lanes(association.lane_source); failure_coordinator.report(IoError::new(ErrorKind::TimedOut, "first failure")); failure_coordinator.report(IoError::other("second failure")); - let receive_result = result_receiver.recv_timeout(TEST_SETUP_TIMEOUT); - reader.join().unwrap(); + let wait_result = result_receiver.recv_timeout(TEST_SETUP_TIMEOUT); - assert!(matches!( - receive_result.unwrap(), - Ok(HostReceive::PeerClosed) | Err(_) - )); + assert!(matches!(wait_result.unwrap(), Ok(false) | Err(_))); let error = failure_coordinator.take_error().unwrap(); assert!(!failure_coordinator.panicked()); assert_eq!(error.kind(), ErrorKind::TimedOut); @@ -1225,6 +1108,38 @@ mod tests { host.join().unwrap(); } + #[test] + fn dispatching_requests_serves_more_concurrent_calls_than_lanes() { + use litebox_broker_local::BrokerLocalError; + + let readiness = Arc::new(ReadinessPublisherRuntime::new()); + let (local, _notifications, _shutdown, outcome, host) = spawn_dispatch(readiness); + + // Callers beyond the lane count wait for a lane, and every call is + // answered on the lane it was made on. + std::thread::scope(|scope| { + for _ in 0..2 * LAYOUT.lane_count() { + scope.spawn(|| { + for _ in 0..50 { + assert!(matches!( + local.check_readiness(litebox_broker_protocol::ObjectHandle(u64::MAX)), + Err(BrokerLocalError::Broker(ErrorCode::UnknownObject)) + )); + } + }); + } + }); + + // Every started worker is waiting on its lane, and a clean local close + // must end them all along with dispatch. + drop(local); + outcome + .recv_timeout(TEST_SETUP_TIMEOUT) + .expect("a clean local close must end dispatch") + .unwrap(); + host.join().unwrap(); + } + #[test] fn dispatching_requests_fails_when_its_readiness_publisher_panics() { // Publication is one-shot, so a runtime that has already run makes the @@ -1234,7 +1149,7 @@ mod tests { readiness.run(&mut DiscardingChannel).unwrap(); // The local half stays connected and idle, so nothing but the panic can - // release the request reader that owns association termination. + // release the association thread that owns association termination. let (local, _notifications, _shutdown, outcome, host) = spawn_dispatch(Arc::clone(&readiness)); let error = outcome diff --git a/litebox_broker_userland/tests/notification_runtime.rs b/litebox_broker_userland/tests/notification_runtime.rs index a6f1738a6a..af54b7efc5 100644 --- a/litebox_broker_userland/tests/notification_runtime.rs +++ b/litebox_broker_userland/tests/notification_runtime.rs @@ -12,7 +12,7 @@ use std::time::Duration; use litebox_broker_core::test_support::TestBrokerCoreBuilder; use litebox_broker_core::{ObjectRights, PolicyEngine}; -use litebox_broker_host::{ConnectionTermination, setup_connection}; +use litebox_broker_host::{BrokerHostAssociation, setup_connection}; use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::ObjectHandle; use litebox_broker_protocol::fs::FileOpenFlags; @@ -20,11 +20,14 @@ use litebox_broker_protocol::message::{BrokerNotification, ReadinessNotification use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_transport::channel::{HostNotificationChannel, HostReceive}; -use litebox_broker_transport::control_ring::{CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRing}; +use litebox_broker_transport::control_ring::{ + CONTROL_RING_NOTIFICATION_SLOT_COUNT, ControlRing, ControlRingLayout, +}; use litebox_broker_transport::shared_memory::SharedBufferPool; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ - UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown, UnixStreamHostSetupChannel, + UnixControlRingHostCallLaneSource, UnixControlRingHostNotificationChannel, + UnixControlRingHostShutdown, UnixStreamHostSetupChannel, }; use litebox_broker_transport_linux_userland::unix_socket::{ UnixControlRingLocalCallChannel, UnixControlRingLocalNotificationChannel, @@ -38,11 +41,33 @@ const TEST_TIMEOUT: Duration = Duration::from_secs(10); const BLOCK_DELAY: Duration = Duration::from_millis(50); /// More objects than the notification ring holds, so publication must block. const OVERSUBSCRIBED_OBJECT_COUNT: u64 = CONTROL_RING_NOTIFICATION_SLOT_COUNT * 3; +/// Layout of every test association's control ring. +const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; + +/// Serves each call lane the local endpoint starts using on its own thread, +/// until the association ends: `Ok` if the local endpoint closed it. +fn serve_lanes( + mut lanes: UnixControlRingHostCallLaneSource, + association: &BrokerHostAssociation, +) -> std::io::Result<()> { + std::thread::scope(|scope| { + while let Some(mut lane) = lanes.next_lane()? { + scope.spawn(move || { + while let Ok(HostReceive::Message(request)) = lane.recv_request() { + association + .execute_request(request, |response| lane.send_response(response)) + .unwrap(); + } + }); + } + Ok(()) + }) +} /// Runs one host association whose only traffic is readiness notifications. /// -/// The reader thread mirrors production: it is the endpoint that observes local -/// termination and interrupts every ring wait of the association. +/// Its call lanes are served as in production, so the association ends when +/// the local endpoint closes it. fn spawn_host( stream: UnixStream, host: impl FnOnce(UnixControlRingHostNotificationChannel, UnixControlRingHostShutdown) @@ -58,8 +83,8 @@ fn spawn_host( let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let shared_buffers = Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let control_ring = ControlRing::new(control_memory).unwrap(); + let control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let control_ring = ControlRing::new(control_memory, LAYOUT).unwrap(); let mut control = UnixStreamHostSetupChannel::from_accepted(stream); let association = setup_connection( &broker, @@ -76,15 +101,12 @@ fn spawn_host( ) .unwrap() .unwrap(); - let (mut request_source, response_sink, notifications, shutdown) = - control.into_active(control_ring).unwrap(); + let (lanes, notifications, shutdown) = control.into_active(control_ring).unwrap(); std::thread::scope(|scope| { + // Dropping `shutdown` when `host` returns can end the association + // before the local endpoint closes it. scope.spawn(|| { - while let Ok(HostReceive::Message(request)) = request_source.recv_request() { - association - .execute_request(request, |response| response_sink.send_response(response)) - .unwrap(); - } + let _ = serve_lanes(lanes, &association); }); host(notifications, shutdown); }); @@ -102,8 +124,8 @@ fn negotiate_local( UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = ControlRing::new(control_memory, LAYOUT).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid test control ring: {error:?}"), @@ -145,8 +167,8 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { let host_shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let host_shared_buffers = Arc::new(SharedBufferPool::new(host_shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); - let host_control_memory = MemfdSharedMemory::create_control_ring().unwrap(); - let host_control_ring = ControlRing::new(host_control_memory).unwrap(); + let host_control_memory = MemfdSharedMemory::create_control_ring(LAYOUT).unwrap(); + let host_control_ring = ControlRing::new(host_control_memory, LAYOUT).unwrap(); let notification = BrokerNotification::Readiness(ReadinessNotification { handle: ObjectHandle(7), readiness: ReadinessFlags::READ, @@ -170,28 +192,17 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { ) .unwrap() .unwrap(); - let (mut request_source, response_sink, mut notifications, _shutdown) = - control.into_active(host_control_ring).unwrap(); + let (lanes, mut notifications, _shutdown) = control.into_active(host_control_ring).unwrap(); notifications.send_notification(&host_notification).unwrap(); - loop { - match request_source.recv_request().unwrap() { - HostReceive::Message(request) => association - .execute_request(request, |response| response_sink.send_response(response)) - .unwrap(), - HostReceive::PeerClosed => return ConnectionTermination::PeerClosed, - HostReceive::ProtocolViolation => { - return ConnectionTermination::ProtocolViolation; - } - } - } + serve_lanes(lanes, &association) }); let (local, _startup, notification_channel) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(local_control), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { + let control_memory = setup.receive_control_ring(LAYOUT, None)?; + let control_ring = ControlRing::new(control_memory, LAYOUT).map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("invalid test control ring: {error:?}"), @@ -216,10 +227,10 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { assert_eq!(local.add_event(handle, 1).unwrap(), readiness); drop(local); - assert_eq!( - host_thread.join().unwrap(), - ConnectionTermination::PeerClosed - ); + host_thread + .join() + .unwrap() + .expect("the host must see a clean close"); } #[test] diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 2856d5ceb9..a0f4ff3c5f 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -20,7 +20,7 @@ use litebox_broker_protocol::shared_buffer::{ #[cfg(target_os = "linux")] use litebox_broker_protocol::socket::{ReceiveFromFlags, SendFlags, SocketConnectionStatus}; use litebox_broker_transport::channel::LocalCallChannel; -use litebox_broker_transport::control_ring::ControlRing; +use litebox_broker_transport::control_ring::{ControlRing, ControlRingLayout}; #[cfg(target_os = "linux")] use litebox_broker_transport_linux_userland::unix_socket::UnixStreamLocalSetupChannel; #[cfg(windows)] @@ -159,8 +159,10 @@ fn run_fake_runner(args: &[OsString]) { SHARED_BUFFER_POOL_SIZE, Some(Instant::now() + Duration::from_secs(5)), )?; - let control_memory = - setup.receive_control_ring(Some(Instant::now() + Duration::from_secs(5)))?; + let control_memory = setup.receive_control_ring( + ControlRingLayout::MULTI_LANE, + Some(Instant::now() + Duration::from_secs(5)), + )?; let process_image = setup.receive_process_image(Some(Instant::now() + Duration::from_secs(5)))?; (shared_memory, control_memory, process_image) @@ -168,17 +170,18 @@ fn run_fake_runner(args: &[OsString]) { #[cfg(windows)] let (shared_memory, control_memory, process_image) = { let shared_memory = setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?; - let control_memory = setup.receive_control_ring()?; + let control_memory = setup.receive_control_ring(ControlRingLayout::MULTI_LANE)?; let process_image = setup.receive_process_image()?; (shared_memory, control_memory, process_image) }; assert!(process_image.is_none()); - let control_ring = ControlRing::new(control_memory).map_err(|error| { - std::io::Error::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; + let control_ring = ControlRing::new(control_memory, ControlRingLayout::MULTI_LANE) + .map_err(|error| { + std::io::Error::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; #[cfg(target_os = "linux")] let (call_channel, _notifications, _shutdown) = setup.into_active(control_ring, || {})?; #[cfg(windows)] diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index de10598ebb..b32a78f74c 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -80,9 +80,8 @@ use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER use litebox_broker_protocol::wire; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, CONTROL_RING_MEMORY_SIZE, ControlRing, ControlRingConsumer, - ControlRingDirection, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, - MemoryAccessPolicy, + BrokerControlRingEndpoints, ControlRing, ControlRingConsumer, ControlRingLayout, + ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool}; use litebox_common_vm_abi::{Status, UserRange, WireFrame}; @@ -99,8 +98,12 @@ use timers::KernelTimers; /// Size of a process's broker shared memory. pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; +/// Layout of a process's broker control ring: the kernel serves one call at a +/// time, so a single lane. +pub const CONTROL_RING_LAYOUT: ControlRingLayout = ControlRingLayout::SINGLE_LANE; + /// Size of a process's broker control ring. -pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; +pub const CONTROL_RING_SIZE: usize = CONTROL_RING_LAYOUT.memory_size(); /// What the broker waits for: the runner's event sources (e.g., device /// interrupts and a timer). @@ -272,7 +275,8 @@ impl Broker { ) -> Association { let memory = PinnedControlRing::new(control_ring_pages).expect("the pages hold the control ring"); - let ring = ControlRing::new(memory).expect("a pinned control ring has the exact size"); + let ring = ControlRing::new(memory, CONTROL_RING_LAYOUT) + .expect("a pinned control ring has the exact size"); Association { broker: self.clone(), shared_memory, @@ -569,15 +573,17 @@ impl Association { } let words = ring.memory().clone(); let BrokerControlRingEndpoints { - request_consumer, - response_producer, + lanes, + lane_demand: _, notification_producer, } = ring.into_broker(); + let [lane] = <[_; 1]>::try_from(lanes) + .unwrap_or_else(|_| unreachable!("a single-lane layout has one lane")); self.readiness.activate(notification_producer); *state = State::Active(Box::new(Active { association: host, - requests: request_consumer, - responses: response_producer, + requests: lane.request_consumer, + responses: lane.response_producer, words, overflow: None, })); @@ -600,12 +606,12 @@ impl Association { if !matches!(*self.state.borrow(), State::Active(_)) { return Err(EnterError::Status(Status::Denied)); } - if !MemoryAccessPolicy::ControlRing.permits_u32(offset) { + if !CONTROL_RING_LAYOUT.permits_u32(offset) { return Err(EnterError::Status(Status::InvalidArgument)); } let wait = match op { RingFutex::Wake => { - if offset != ControlRingDirection::Requests.producer_epoch_offset() { + if offset != CONTROL_RING_LAYOUT.requests(0).producer_epoch_offset() { // The broker waits on no other word. return Ok(Entered::Done); } @@ -616,7 +622,7 @@ impl Association { expected, deadline, notifications: offset - == ControlRingDirection::Notifications.producer_epoch_offset(), + == CONTROL_RING_LAYOUT.notifications().producer_epoch_offset(), }), }; self.submitted.set(true); @@ -1079,14 +1085,17 @@ impl ProcessLauncher for KernelProcessLauncher { #[cfg(test)] mod tests { - use litebox_broker_transport::control_ring::CONTROL_RING_READY; + use super::CONTROL_RING_LAYOUT; /// `ABI_VERSION` stands in for the userland broker's ring-layout token; /// change both together. #[test] fn ring_layout_change_needs_an_abi_version_change() { assert_eq!( - (CONTROL_RING_READY, litebox_common_vm_abi::ABI_VERSION), + ( + CONTROL_RING_LAYOUT.ready_token(), + litebox_common_vm_abi::ABI_VERSION + ), (&b"litebox-control-ring-ready-v1"[..], 1) ); } diff --git a/litebox_broker_vm_kernel/src/memory.rs b/litebox_broker_vm_kernel/src/memory.rs index bbf2d1012f..0813fa01e0 100644 --- a/litebox_broker_vm_kernel/src/memory.rs +++ b/litebox_broker_vm_kernel/src/memory.rs @@ -4,11 +4,11 @@ //! The association's shared memory: the shared buffers in the process's lazy //! mapping, and the control ring in pinned pages. +use crate::{CONTROL_RING_LAYOUT, CONTROL_RING_SIZE}; use alloc::sync::Arc; use core::ops::Range; use litebox::mm::exception_table::memcpy_fallible; use litebox::platform::common_providers::userspace_pointers::ValidateAccess as _; -use litebox_broker_transport::control_ring::{CONTROL_RING_MEMORY_SIZE, MemoryAccessPolicy}; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedMemory, SharedMemoryError}; use litebox_common_vm_abi::UserRange; use litebox_platform::buffer::Fault; @@ -71,14 +71,14 @@ impl SharedMemory for UserSharedMemory { } /// The control ring at the start of pinned pages, through the kernel's -/// mapping. Accesses keep to [`MemoryAccessPolicy::ControlRing`]. +/// mapping. Accesses keep to [`CONTROL_RING_LAYOUT`]. #[derive(Clone)] pub(crate) struct PinnedControlRing(Arc); impl PinnedControlRing { /// `None` if `pages` cannot hold the ring. pub(crate) fn new(pages: PinnedUserPages) -> Option { - (pages.len() >= CONTROL_RING_MEMORY_SIZE).then_some(Self(Arc::new(pages))) + (pages.len() >= CONTROL_RING_SIZE).then_some(Self(Arc::new(pages))) } fn word(&self, offset: usize, permitted: bool) -> Result<*mut T, SharedMemoryError> { @@ -97,11 +97,11 @@ impl PinnedControlRing { } fn u32_at(&self, offset: usize) -> Result<*mut u32, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u32(offset)) + self.word(offset, CONTROL_RING_LAYOUT.permits_u32(offset)) } fn u64_at(&self, offset: usize) -> Result<*mut u64, SharedMemoryError> { - self.word(offset, MemoryAccessPolicy::ControlRing.permits_u64(offset)) + self.word(offset, CONTROL_RING_LAYOUT.permits_u64(offset)) } /// Calls `copy` for each page's part of `offset..offset + len`, with that @@ -112,7 +112,7 @@ impl PinnedControlRing { len: usize, mut copy: impl FnMut(*mut u8, Range), ) -> Result<(), SharedMemoryError> { - if !MemoryAccessPolicy::ControlRing.permits_byte_range(offset, len) { + if !CONTROL_RING_LAYOUT.permits_byte_range(offset, len) { return Err(SharedMemoryError::InvalidRange); } let mut done = 0; @@ -133,7 +133,7 @@ impl PinnedControlRing { // (checked), and disjoint from private memory. impl SharedMemory for PinnedControlRing { fn len(&self) -> usize { - CONTROL_RING_MEMORY_SIZE + CONTROL_RING_SIZE } fn read(&self, offset: usize, destination: &mut [u8]) -> Result<(), SharedMemoryError> { diff --git a/litebox_common_vm_abi/src/lib.rs b/litebox_common_vm_abi/src/lib.rs index e3c6a70e11..1b80fd047a 100644 --- a/litebox_common_vm_abi/src/lib.rs +++ b/litebox_common_vm_abi/src/lib.rs @@ -194,8 +194,8 @@ //! no further request. //! - A malformed ring or request kills the process. Before the handshake, a //! futex operation on the control ring is [`Status::Denied`]; on the ring, -//! only the words that `litebox_broker_transport`'s -//! `MemoryAccessPolicy::ControlRing` lets a peer access as `u32` are valid. +//! only the words that `litebox_broker_transport`'s single-lane +//! `ControlRingLayout` lets a peer access as `u32` are valid. #![no_std] diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index e03f6eb803..ebf99ca63f 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -1774,11 +1774,15 @@ fn run_test_broker_connection( ) .expect("failed to attach broker test shared-buffer layout"), ); + let layout = litebox_broker_transport::control_ring::ControlRingLayout::MULTI_LANE; let control_memory = - litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory::create_control_ring() - .expect("failed to create broker test control ring"); - let control_ring = litebox_broker_transport::control_ring::ControlRing::new(control_memory) - .expect("failed to attach broker test control ring"); + litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory::create_control_ring( + layout, + ) + .expect("failed to create broker test control ring"); + let control_ring = + litebox_broker_transport::control_ring::ControlRing::new(control_memory, layout) + .expect("failed to attach broker test control ring"); let setup_deadline = std::time::Instant::now() + BROKER_HELPER_TIMEOUT; let mut channel = litebox_broker_transport_linux_userland::unix_socket::UnixStreamHostSetupChannel::from_host_guaranteed( @@ -1803,57 +1807,73 @@ fn run_test_broker_connection( ) .expect("broker host setup failed") .expect("broker setup terminated before activation"); - let (mut request_source, response_sink, mut notifications, _shutdown) = channel + let (mut lanes, mut notifications, shutdown) = channel .into_active(control_ring) .expect("failed to activate broker test control ring"); let publisher_readiness = readiness.clone(); let publisher = std::thread::spawn(move || publisher_readiness.run(&mut notifications)); - let mut close_object_count = 0; - // Files and the process's signals are not counted, since every runner opens them. - let mut uncounted_handles = Vec::new(); - let termination = loop { - match request_source - .recv_request() - .expect("failed to receive broker test request") + // The number of objects closed, and the open file and signal handles excluded from it since + // every runner opens them. + let counts = std::sync::Mutex::new((0, Vec::new())); + // Serves each call lane the runner starts using on its own thread, until the runner closes the + // association. + std::thread::scope(|scope| { + while let Some(mut lane) = lanes + .next_lane() + .expect("failed to receive a broker test call lane") { - litebox_broker_transport::channel::HostReceive::Message(request) => { - if let litebox_broker_protocol::message::BrokerOperation::CloseObject(handle) = - &request.operation - { - if let Some(index) = uncounted_handles.iter().position(|value| value == handle) + let (association, counts, shutdown) = (&association, &counts, &shutdown); + scope.spawn(move || { + loop { + match lane + .recv_request() + .expect("failed to receive broker test request") { - uncounted_handles.swap_remove(index); - } else { - close_object_count += 1; + litebox_broker_transport::channel::HostReceive::Message(request) => { + if let litebox_broker_protocol::message::BrokerOperation::CloseObject( + handle, + ) = &request.operation + { + let (close_object_count, uncounted_handles) = + &mut *counts.lock().unwrap(); + if let Some(index) = + uncounted_handles.iter().position(|value| value == handle) + { + uncounted_handles.swap_remove(index); + } else { + *close_object_count += 1; + } + } + association + .execute_request(request, |response| { + match &response.result { + litebox_broker_protocol::message::BrokerResult::File( + litebox_broker_protocol::message::FileResponse::Open( + open, + ), + ) => counts.lock().unwrap().1.push(open.handle), + litebox_broker_protocol::message::BrokerResult::Signal( + litebox_broker_protocol::message::SignalResponse::Open( + open, + ), + ) => counts.lock().unwrap().1.push(open.handle), + _ => {} + } + lane.send_response(response) + }) + .expect("failed to execute broker test request"); + } + litebox_broker_transport::channel::HostReceive::PeerClosed => return, + litebox_broker_transport::channel::HostReceive::ProtocolViolation => { + // Ends the wait for lanes, so the test fails instead of hanging. + let _ = shutdown.shutdown(); + panic!("broker test request violated the protocol"); + } } } - association - .execute_request(request, |response| { - match &response.result { - litebox_broker_protocol::message::BrokerResult::File( - litebox_broker_protocol::message::FileResponse::Open(open), - ) => uncounted_handles.push(open.handle), - litebox_broker_protocol::message::BrokerResult::Signal( - litebox_broker_protocol::message::SignalResponse::Open(open), - ) => uncounted_handles.push(open.handle), - _ => {} - } - response_sink.send_response(response) - }) - .expect("failed to execute broker test request"); - } - litebox_broker_transport::channel::HostReceive::PeerClosed => { - break litebox_broker_host::ConnectionTermination::PeerClosed; - } - litebox_broker_transport::channel::HostReceive::ProtocolViolation => { - break litebox_broker_host::ConnectionTermination::ProtocolViolation; - } + }); } - }; - assert_eq!( - termination, - litebox_broker_host::ConnectionTermination::PeerClosed - ); + }); readiness.close(); // The runner's clean close can fail a notification send already in flight, which the broker // runtime likewise ignores; any other publication failure is a real error. @@ -1873,6 +1893,7 @@ fn run_test_broker_connection( ); } association.finish(); + let (close_object_count, _) = counts.into_inner().unwrap(); close_object_count_tx .send(close_object_count) .expect("failed to report broker close-object count"); From 07595a29e57b935cb2dbed7825053d6d0f9def44 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Tue, 20 Oct 2026 23:21:11 -0700 Subject: [PATCH 02/12] End lanes associations promptly and reject late responses The lane observer now applies owner-death handling as soon as it observes the association's end, so a request waiting for a child to start, such as StartChild on the first lane, is released and the child fails as before. Having ended the association, the observer then joins the workers it started. Calls accept a response only while the association is live, and the runner test broker ends its association when a lane thread panics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- .../src/unix_socket/local.rs | 6 +- .../src/local.rs | 6 +- litebox_broker_userland/src/runtime.rs | 109 ++++++++---------- litebox_runner_linux_userland/tests/run.rs | 20 +++- 4 files changed, 74 insertions(+), 67 deletions(-) diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index e5e298537f..bf6a3fd3cc 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -486,7 +486,11 @@ impl LocalRingAssociation { Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } }; - consumer.publish_head().map_err(ring_error)?; + // Accepting the response excludes failure recording, so no call + // succeeds once the association has failed. + self.lanes + .run_if_live(|| consumer.publish_head().map_err(ring_error)) + .map_err(call_lane_pool_error)?; consumer.wake_producer()?; if response.request_id != request_id { return Err(invalid_data("broker answered a different request")); diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 262545f261..f8c125783b 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -417,7 +417,11 @@ impl LocalRingAssociation { Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } }; - consumer.publish_head().map_err(ring_error)?; + // Accepting the response excludes failure recording, so no call + // succeeds once the association has failed. + self.lanes + .run_if_live(|| consumer.publish_head().map_err(ring_error)) + .map_err(call_lane_pool_error)?; consumer.wake_producer()?; if response.request_id != request_id { return Err(invalid_data("broker answered a different request")); diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index a2cac1b128..28ce1db0c7 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -412,30 +412,32 @@ impl> Drop } } -/// Requests cancellation before an association scope joins its workers. -struct AssociationCancellationGuard<'association, Memory: SharedMemory> { +/// Ends an association once its end is observed, including on an unwind: +/// cancels the requests still executing and applies owner-death handling, which +/// fails the children still starting. +struct AssociationEndingGuard<'association, Memory: SharedMemory> { association: &'association BrokerHostAssociation, } -impl Drop for AssociationCancellationGuard<'_, Memory> { +impl Drop for AssociationEndingGuard<'_, Memory> { fn drop(&mut self) { self.association.request_cancellation(); + self.association.association_ending(); } } /// Serves one association until it ends, then reports its first failure. /// -/// Each call lane has its own worker from the runner's first use of it, so the +/// Each call lane gets its own worker once the runner first uses it, so the /// association has as many workers as the runner has had calls in flight at -/// once. A worker serves only its own lane, so a call never waits for another -/// call to finish, and back-to-back calls stay on the first lane's worker. +/// once. Calls on different lanes never wait for each other, and back-to-back +/// calls stay on the first lane's worker. /// -/// This thread is the first lane's worker. Having set the association up with -/// the runner, it tends to run where the runner's writes reach it fastest, -/// whereas the scheduler places a new thread on the least busy CPUs, which can -/// be across a cache boundary from the runner. An observer thread starts the -/// workers of further lanes, observes the end of the association, and cancels -/// the requests still executing; this thread then joins every worker. +/// This thread serves the first lane: having run setup with the runner, it +/// tends to stay near the runner, whereas a new thread can land across a cache +/// boundary. An observer thread starts and joins the workers of further lanes +/// and ends the association as soon as it observes its end, so requests still +/// executing see that promptly. /// /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network @@ -503,36 +505,24 @@ where readiness: &readiness, failure_coordinator: &failure_coordinator, }; - let workers = match context.next_lane(&mut lane_source) { - Some(first_lane) => { - let observer = std::thread::Builder::new() - .name("litebox-broker-lanes".to_owned()) - .spawn_scoped(scope, move || { - context.serve_further_lanes(scope, lane_source) - }); - match observer { - Ok(observer) => { - context.serve(first_lane); - observer.join().unwrap_or_else(|_| { - failure_coordinator - .report_panic(IoError::other("broker call-lane observer panicked")); - Vec::new() - }) - } - Err(error) => { - failure_coordinator.report(error); - Vec::new() + if let Some(first_lane) = context.next_lane(&mut lane_source) { + let observer = std::thread::Builder::new() + .name("litebox-broker-lanes".to_owned()) + .spawn_scoped(scope, move || context.serve_further_lanes(lane_source)); + match observer { + Ok(observer) => { + context.serve(first_lane); + if observer.join().is_err() { + failure_coordinator + .report_panic(IoError::other("broker call-lane observer panicked")); } } - } - None => Vec::new(), - }; - association.association_ending(); - for worker in workers { - if worker.join().is_err() { - failure_coordinator.report_panic(IoError::other("broker request worker panicked")); + Err(error) => failure_coordinator.report(error), } } + // Ends the association if no observer did; ending it again is + // harmless. + association.association_ending(); // Readiness publication lives exactly as long as the association, and // workers can still change readiness until they are joined. Ending it // here rather than leaving it to the scope orders it before the join @@ -600,38 +590,31 @@ where } /// Starts a worker for each further lane the runner uses until the - /// association ends, then cancels the requests still executing and returns - /// the workers to join. - fn serve_further_lanes<'scope, LaneSource>( - self, - scope: &'scope std::thread::Scope<'scope, '_>, - mut lane_source: LaneSource, - ) -> Vec> + /// association ends, then ends the association and joins the workers. + fn serve_further_lanes(self, mut lane_source: LaneSource) where LaneSource: HostCallLaneSource, - LaneSource::Lane: Send + 'scope, - Self: Send + 'scope, + LaneSource::Lane: Send, + Self: Send, { - // Cancelling on an unwind too keeps the scope from waiting forever on - // a worker blocked in a request. - let cancellation = AssociationCancellationGuard { - association: self.association, - }; - let mut workers = Vec::new(); - while let Some(lane) = self.next_lane(&mut lane_source) { - let started = std::thread::Builder::new() - .name(format!("litebox-broker-worker-{}", workers.len() + 1)) - .spawn_scoped(scope, move || self.serve(lane)); - match started { - Ok(started) => workers.push(started), - Err(error) => { + std::thread::scope(|scope| { + // Dropped before the scope joins the workers, including on an + // unwind, so the join never waits on a worker blocked in a request. + let _ending = AssociationEndingGuard { + association: self.association, + }; + let mut workers = 0; + while let Some(lane) = self.next_lane(&mut lane_source) { + workers += 1; + let started = std::thread::Builder::new() + .name(format!("litebox-broker-worker-{workers}")) + .spawn_scoped(scope, move || self.serve(lane)); + if let Err(error) = started { self.failure_coordinator.report(error); break; } } - } - drop(cancellation); - workers + }); } /// Serves one lane's requests until the association ends. diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index ebf99ca63f..61a28dff65 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -1757,6 +1757,22 @@ fn spawn_test_broker_with_mode( } } +/// Shuts a test broker association down if one of its lane threads, or the thread starting them, +/// panics, ending every wait on it so the test fails instead of hanging. +#[cfg(target_os = "linux")] +struct ShutdownOnPanic<'a>( + &'a litebox_broker_transport_linux_userland::unix_socket::UnixControlRingHostShutdown, +); + +#[cfg(target_os = "linux")] +impl Drop for ShutdownOnPanic<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + let _ = self.0.shutdown(); + } + } +} + #[cfg(target_os = "linux")] fn run_test_broker_connection( broker: &litebox_broker_core::BrokerCore, @@ -1818,12 +1834,14 @@ fn run_test_broker_connection( // Serves each call lane the runner starts using on its own thread, until the runner closes the // association. std::thread::scope(|scope| { + let _shutdown_on_panic = ShutdownOnPanic(&shutdown); while let Some(mut lane) = lanes .next_lane() .expect("failed to receive a broker test call lane") { let (association, counts, shutdown) = (&association, &counts, &shutdown); scope.spawn(move || { + let _shutdown_on_panic = ShutdownOnPanic(shutdown); loop { match lane .recv_request() @@ -1865,8 +1883,6 @@ fn run_test_broker_connection( } litebox_broker_transport::channel::HostReceive::PeerClosed => return, litebox_broker_transport::channel::HostReceive::ProtocolViolation => { - // Ends the wait for lanes, so the test fails instead of hanging. - let _ = shutdown.shutdown(); panic!("broker test request violated the protocol"); } } From 3fe44c480a56e03db51eb4bb3122e94de9b41a29 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Tue, 20 Oct 2026 23:29:28 -0700 Subject: [PATCH 03/12] Accept responses as calls free their lanes Accept a call's response and free its lane in one admission critical section, so a call still cannot succeed after the association fails without taking an extra lock per call. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- .../src/call_lane_pool.rs | 38 +++++++++++++++++-- .../src/call_lane_pool.rs | 27 ++++++++++++- .../src/unix_socket/local.rs | 25 ++++++------ .../src/call_lane_pool.rs | 27 ++++++++++++- .../src/local.rs | 25 ++++++------ 5 files changed, 114 insertions(+), 28 deletions(-) diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs index f37d3079ca..71b1a6aa92 100644 --- a/litebox_broker_transport/src/call_lane_pool.rs +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -89,10 +89,11 @@ struct CallLanePoolInner { failure: Option>, } -/// A lane held by one call; dropping it frees the lane. +/// A lane held by one call; releasing or dropping the claim frees the lane. pub struct LaneClaim<'lanes, Sync: CallLanePoolSync, Lane, Error> { lanes: &'lanes CallLanePool, index: usize, + /// The lane, or `None` once released. lane: Option, } @@ -102,7 +103,7 @@ impl Deref for LaneClaim<'_, Sync, Lane, Er fn deref(&self) -> &Lane { self.lane .as_ref() - .expect("claimed lane is present until drop") + .expect("claimed lane is present until released") } } @@ -110,13 +111,42 @@ impl DerefMut for LaneClaim<'_, Sync, Lane, fn deref_mut(&mut self) -> &mut Lane { self.lane .as_mut() - .expect("claimed lane is present until drop") + .expect("claimed lane is present until released") } } impl Drop for LaneClaim<'_, Sync, Lane, Error> { fn drop(&mut self) { - let mut state = self.lanes.state.lock(); + if self.lane.is_some() { + let lanes = self.lanes; + self.free(&mut lanes.state.lock()); + } + } +} + +impl LaneClaim<'_, Sync, Lane, Error> { + /// Runs `operation` on the lane and frees the lane, both while excluding + /// failure recording, so a call completes only while the association is + /// live. + /// + /// If the association has failed or `operation` fails, the claim keeps + /// the lane until dropped, so the caller can fail the association first. + pub fn release_if_live( + &mut self, + operation: impl FnOnce(&mut Lane) -> Result, + ) -> Result> { + let lanes = self.lanes; + let mut state = lanes.state.lock(); + if let Some(error) = state.failure.as_ref() { + return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); + } + let value = operation(self).map_err(CallLanePoolError::Operation)?; + self.free(&mut state); + Ok(value) + } + + /// Returns the lane to its slot, under the admission lock. + fn free(&mut self, state: &mut CallLanePoolInner) { state.lanes[self.index] = self.lane.take(); if state.waiting > 0 { self.lanes.lane_freed.notify_one(); diff --git a/litebox_broker_transport_linux_userland/src/call_lane_pool.rs b/litebox_broker_transport_linux_userland/src/call_lane_pool.rs index f10d1108fe..65c7ff64e2 100644 --- a/litebox_broker_transport_linux_userland/src/call_lane_pool.rs +++ b/litebox_broker_transport_linux_userland/src/call_lane_pool.rs @@ -7,13 +7,16 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use litebox_broker_transport::call_lane_pool::{ CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, - CallLanePoolSync, + CallLanePoolSync, LaneClaim as GenericLaneClaim, }; use crate::setup::copy_io_error; pub(crate) type CallLanePool = GenericCallLanePool; +pub(crate) type LaneClaim<'lanes, Lane> = + GenericLaneClaim<'lanes, StdCallLanePoolSync, Lane, Error>; + pub(crate) struct StdCallLanePoolSync; pub(crate) struct StdCallLanePoolMutex(Mutex); @@ -146,6 +149,28 @@ mod tests { }); } + #[test] + fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { + let lanes = CallLanePool::new(vec![0, 1]); + let mut released = lanes.claim(no_demand).unwrap(); + assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); + // Dropping a released claim leaves its lane free. + drop(released); + let mut held = lanes.claim(no_demand).unwrap(); + assert_eq!(*held, 0); + assert!(matches!( + held.release_if_live(|_| Err::<(), _>(Error::other("release failed"))), + Err(CallLanePoolError::Operation(_)) + )); + assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); + assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); + assert!(matches!( + held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), + Err(CallLanePoolError::AssociationFailed(_)) + )); + assert_eq!(*held, 0); + } + #[test] fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { let lanes = CallLanePool::new(vec![0]); diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index bf6a3fd3cc..71a741041b 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -40,7 +40,7 @@ use litebox_broker_transport::control_ring::{ LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; -use crate::call_lane_pool::{CallLanePool, call_lane_pool_error}; +use crate::call_lane_pool::{CallLanePool, LaneClaim, call_lane_pool_error}; use crate::memfd::MemfdSharedMemory; use crate::setup::{ copy_io_error, invalid_data, read_setup_frame, ring_error, shutdown_socket, wire_error, @@ -356,8 +356,8 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { } }; let result = association.call_on_lane(&mut lane, request); - // Fail before freeing the lane, so no other call can find it midway - // through this call. + // A failed call can leave its lane mid-exchange: fail the association + // before freeing the lane, so no other call uses it. if let Err(error) = &result { let _ = association.fail(copy_io_error(error)); } @@ -449,10 +449,11 @@ impl LocalRingAssociation { self.lane_demand.wake_broker() } - /// Writes `request` to `lane` and reads the broker's response from it. + /// Writes `request` to `lane` and reads the broker's response from it, + /// releasing the lane once the call succeeds. fn call_on_lane( &self, - lane: &mut LocalLane, + lane: &mut LaneClaim<'_, LocalLane>, request: BrokerRequest, ) -> IoResult { let request_id = request.request_id; @@ -486,15 +487,17 @@ impl LocalRingAssociation { Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } }; - // Accepting the response excludes failure recording, so no call - // succeeds once the association has failed. - self.lanes - .run_if_live(|| consumer.publish_head().map_err(ring_error)) - .map_err(call_lane_pool_error)?; - consumer.wake_producer()?; if response.request_id != request_id { return Err(invalid_data("broker answered a different request")); } + // Accepting the response with the lane's release excludes failure + // recording, so no call succeeds once the association has failed. + lane.release_if_live(|lane| { + let consumer = &mut lane.response_consumer; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer() + }) + .map_err(call_lane_pool_error)?; Ok(response) } diff --git a/litebox_broker_transport_windows_userland/src/call_lane_pool.rs b/litebox_broker_transport_windows_userland/src/call_lane_pool.rs index f10d1108fe..65c7ff64e2 100644 --- a/litebox_broker_transport_windows_userland/src/call_lane_pool.rs +++ b/litebox_broker_transport_windows_userland/src/call_lane_pool.rs @@ -7,13 +7,16 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use litebox_broker_transport::call_lane_pool::{ CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, - CallLanePoolSync, + CallLanePoolSync, LaneClaim as GenericLaneClaim, }; use crate::setup::copy_io_error; pub(crate) type CallLanePool = GenericCallLanePool; +pub(crate) type LaneClaim<'lanes, Lane> = + GenericLaneClaim<'lanes, StdCallLanePoolSync, Lane, Error>; + pub(crate) struct StdCallLanePoolSync; pub(crate) struct StdCallLanePoolMutex(Mutex); @@ -146,6 +149,28 @@ mod tests { }); } + #[test] + fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { + let lanes = CallLanePool::new(vec![0, 1]); + let mut released = lanes.claim(no_demand).unwrap(); + assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); + // Dropping a released claim leaves its lane free. + drop(released); + let mut held = lanes.claim(no_demand).unwrap(); + assert_eq!(*held, 0); + assert!(matches!( + held.release_if_live(|_| Err::<(), _>(Error::other("release failed"))), + Err(CallLanePoolError::Operation(_)) + )); + assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); + assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); + assert!(matches!( + held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), + Err(CallLanePoolError::AssociationFailed(_)) + )); + assert_eq!(*held, 0); + } + #[test] fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { let lanes = CallLanePool::new(vec![0]); diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index f8c125783b..3223b4b39c 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -32,7 +32,7 @@ use litebox_broker_transport::control_ring::{ use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; -use crate::call_lane_pool::{CallLanePool, call_lane_pool_error}; +use crate::call_lane_pool::{CallLanePool, LaneClaim, call_lane_pool_error}; use crate::control_ring::PipeLiveness; use crate::named_pipe::{TRANSFER_FRAME_TAG, WindowsNamedPipeStream}; use crate::process_image::WindowsReceivedProcessImage; @@ -277,8 +277,8 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { } }; let result = association.call_on_lane(&mut lane, request); - // Fail before freeing the lane, so no other call can find it midway - // through this call. + // A failed call can leave its lane mid-exchange: fail the association + // before freeing the lane, so no other call uses it. if let Err(error) = &result { let _ = association.fail(copy_io_error(error)); } @@ -380,10 +380,11 @@ impl LocalRingAssociation { self.lane_demand.wake_broker() } - /// Writes `request` to `lane` and reads the broker's response from it. + /// Writes `request` to `lane` and reads the broker's response from it, + /// releasing the lane once the call succeeds. fn call_on_lane( &self, - lane: &mut LocalLane, + lane: &mut LaneClaim<'_, LocalLane>, request: BrokerRequest, ) -> IoResult { let request_id = request.request_id; @@ -417,15 +418,17 @@ impl LocalRingAssociation { Err(ControlRingReadError::Decode(error)) => return Err(wire_error(error)), } }; - // Accepting the response excludes failure recording, so no call - // succeeds once the association has failed. - self.lanes - .run_if_live(|| consumer.publish_head().map_err(ring_error)) - .map_err(call_lane_pool_error)?; - consumer.wake_producer()?; if response.request_id != request_id { return Err(invalid_data("broker answered a different request")); } + // Accepting the response with the lane's release excludes failure + // recording, so no call succeeds once the association has failed. + lane.release_if_live(|lane| { + let consumer = &mut lane.response_consumer; + consumer.publish_head().map_err(ring_error)?; + consumer.wake_producer() + }) + .map_err(call_lane_pool_error)?; Ok(response) } From bbe4bd4a8e8e69f85c0a3a0ad1e3e7fb386a013b Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 08:55:41 -0700 Subject: [PATCH 04/12] Rename ControlRingLayout's lanes field to lane_count The field now matches its lane_count accessor. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/src/control_ring.rs | 32 +++++++++++--------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index 83c7c07d90..bfeea38a98 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -80,7 +80,7 @@ const DEMAND_WORDS_SIZE: usize = 8; /// Lane 0 is always in use. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ControlRingLayout { - lanes: usize, + lane_count: usize, call_slots: u64, sync_stride: usize, ready: &'static [u8], @@ -92,7 +92,7 @@ const _: () = assert!(ControlRingLayout::MULTI_LANE.memory_size() == 11_336); impl ControlRingLayout { /// One lane of 64 slots per direction, served by one broker thread. pub const SINGLE_LANE: Self = Self { - lanes: 1, + lane_count: 1, call_slots: 64, sync_stride: SYNC_WORDS_SIZE, ready: b"litebox-control-ring-ready-v1", @@ -104,7 +104,7 @@ impl ControlRingLayout { /// themselves, so neither the two directions of a call nor lanes in use at /// the same time contend for one. pub const MULTI_LANE: Self = Self { - lanes: 8, + lane_count: 8, call_slots: 1, sync_stride: CACHE_LINE_SIZE, ready: b"litebox-control-ring-lanes-ready-v1", @@ -112,7 +112,7 @@ impl ControlRingLayout { /// Returns the number of lanes. pub const fn lane_count(self) -> usize { - self.lanes + self.lane_count } /// Returns the versioned token endpoints exchange before activating a @@ -123,7 +123,11 @@ impl ControlRingLayout { /// Returns the exact size of a mapping with this layout. pub const fn memory_size(self) -> usize { - self.demand_offset() + if self.lanes > 1 { DEMAND_WORDS_SIZE } else { 0 } + if self.lane_count > 1 { + self.demand_offset() + DEMAND_WORDS_SIZE + } else { + self.demand_offset() + } } /// Returns the local-to-broker request direction of `lane`. @@ -132,7 +136,7 @@ impl ControlRingLayout { /// /// Panics if `lane` is not less than [`Self::lane_count`]. pub const fn requests(self, lane: usize) -> ControlRingDirection { - assert!(lane < self.lanes, "control-ring lane out of range"); + assert!(lane < self.lane_count, "control-ring lane out of range"); self.direction(2 * lane) } @@ -142,13 +146,13 @@ impl ControlRingLayout { /// /// Panics if `lane` is not less than [`Self::lane_count`]. pub const fn responses(self, lane: usize) -> ControlRingDirection { - assert!(lane < self.lanes, "control-ring lane out of range"); + assert!(lane < self.lane_count, "control-ring lane out of range"); self.direction(2 * lane + 1) } /// Returns the broker-to-local notification direction. pub const fn notifications(self) -> ControlRingDirection { - self.direction(2 * self.lanes) + self.direction(2 * self.lane_count) } /// Returns whether this layout permits a byte-copy range. @@ -188,7 +192,7 @@ impl ControlRingLayout { | PRODUCER_WAITING_OFFSET ); } - self.lanes > 1 + self.lane_count > 1 && matches!( relative - self.sync_size(), REQUESTED_LANES_OFFSET | DEMAND_EPOCH_OFFSET @@ -208,7 +212,7 @@ impl ControlRingLayout { /// producer and consumer epochs of every direction and, with more than one /// lane, the lane-demand epoch. pub const fn waitable_word_count(self) -> usize { - 2 * self.direction_count() + if self.lanes > 1 { 1 } else { 0 } + 2 * self.direction_count() + if self.lane_count > 1 { 1 } else { 0 } } /// Returns the position of the waitable `u32` at `offset` among the @@ -226,7 +230,7 @@ impl ControlRingLayout { _ => None, }; } - if self.lanes > 1 && relative - self.sync_size() == DEMAND_EPOCH_OFFSET { + if self.lane_count > 1 && relative - self.sync_size() == DEMAND_EPOCH_OFFSET { Some(2 * self.direction_count()) } else { None @@ -234,7 +238,7 @@ impl ControlRingLayout { } const fn direction_count(self) -> usize { - 2 * self.lanes + 1 + 2 * self.lane_count + 1 } // Layout slot counts are small constants representable by `usize`. @@ -244,7 +248,7 @@ impl ControlRingLayout { } const fn data_size(self) -> usize { - 2 * self.lanes * self.call_direction_size() + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE + 2 * self.lane_count * self.call_direction_size() + CONTROL_RING_NOTIFICATION_DIRECTION_SIZE } const fn sync_size(self) -> usize { @@ -260,7 +264,7 @@ impl ControlRingLayout { const fn direction(self, index: usize) -> ControlRingDirection { ControlRingDirection { data_offset: index * self.call_direction_size(), - slot_count: if index == 2 * self.lanes { + slot_count: if index == 2 * self.lane_count { CONTROL_RING_NOTIFICATION_SLOT_COUNT } else { self.call_slots From c57c7df4a3cf4f6a39111c76cd04c96472a91497 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 10:28:55 -0700 Subject: [PATCH 05/12] Name the local peer in HostCallLaneSource docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/src/channel.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/litebox_broker_transport/src/channel.rs b/litebox_broker_transport/src/channel.rs index 711fdb6527..01fc79882d 100644 --- a/litebox_broker_transport/src/channel.rs +++ b/litebox_broker_transport/src/channel.rs @@ -156,10 +156,10 @@ pub trait HostCallLane { /// Host-side source of the call lanes of an active association. /// -/// The peer starts using lanes as its concurrent calls grow, and this source -/// hands out each lane once the peer starts using it, so a broker serves only -/// lanes in use. The peer always uses the first lane, so the first -/// [`next_lane`](Self::next_lane) returns it without blocking unless the +/// The local peer starts using lanes as its concurrent calls grow, and this +/// source hands out each lane once the local peer starts using it, so a broker +/// serves only lanes in use. The local peer always uses the first lane, so the +/// first [`next_lane`](Self::next_lane) returns it without blocking unless the /// association has already ended. pub trait HostCallLaneSource { /// Channel-specific error type. @@ -168,11 +168,11 @@ pub trait HostCallLaneSource { /// Lane type handed out by this source. type Lane: HostCallLane; - /// Blocks until the peer starts using another lane and returns it. + /// Blocks until the local peer starts using another lane and returns it. /// /// Once every lane is handed out, this blocks until the association ends. - /// Returns `Ok(None)` when the peer closed the association cleanly. An - /// error is terminal: before returning it, the transport must terminate + /// Returns `Ok(None)` when the local peer closed the association cleanly. + /// An error is terminal: before returning it, the transport must terminate /// the association and unblock every endpoint. fn next_lane(&mut self) -> Result, Self::Error>; } From 0ad8c6772ee7e348e9ea89c97d35df5101605b82 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 10:35:04 -0700 Subject: [PATCH 06/12] Name lane counts as counts and settle on lane demand Count fields and locals in the call-lane pool and the broker's lane demand now end in _count, and the shared demand word is the demanded lane count rather than requested lanes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- .../src/call_lane_pool.rs | 18 +++++------ litebox_broker_transport/src/control_ring.rs | 30 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs index 71b1a6aa92..f934690f53 100644 --- a/litebox_broker_transport/src/call_lane_pool.rs +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -81,11 +81,11 @@ struct CallLanePoolInner { /// Each lane, or `None` while a call holds it. lanes: Vec>, /// Number of lanes demanded so far. - demanded: usize, + demanded_lane_count: usize, /// Number of claims waiting for a free lane. Freeing a lane notifies only /// if one waits, since a notification can cost a system call even with no /// waiters. - waiting: usize, + waiting_claim_count: usize, failure: Option>, } @@ -148,7 +148,7 @@ impl LaneClaim<'_, Sync, Lane, Error> { /// Returns the lane to its slot, under the admission lock. fn free(&mut self, state: &mut CallLanePoolInner) { state.lanes[self.index] = self.lane.take(); - if state.waiting > 0 { + if state.waiting_claim_count > 0 { self.lanes.lane_freed.notify_one(); } } @@ -165,8 +165,8 @@ impl CallLanePool { Self { state: Sync::mutex(CallLanePoolInner { lanes: lanes.into_iter().map(Some).collect(), - demanded: 1, - waiting: 0, + demanded_lane_count: 1, + waiting_claim_count: 0, failure: None, }), lane_freed: Sync::condvar(), @@ -190,13 +190,13 @@ impl CallLanePool { if let Some(index) = state.lanes.iter().position(Option::is_some) { break index; } - state.waiting += 1; + state.waiting_claim_count += 1; state = self.lane_freed.wait(state); - state.waiting -= 1; + state.waiting_claim_count -= 1; }; - if index >= state.demanded { + if index >= state.demanded_lane_count { demand(index + 1).map_err(CallLanePoolError::Operation)?; - state.demanded = index + 1; + state.demanded_lane_count = index + 1; } Ok(LaneClaim { lanes: self, diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index bfeea38a98..8347e95571 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -58,8 +58,8 @@ const SYNC_WORDS_SIZE: usize = 24; /// Cache-line size assumed when separating words that different threads poll. const CACHE_LINE_SIZE: usize = 64; /// Number of lanes the local peer asks the broker to serve. -const REQUESTED_LANES_OFFSET: usize = 0; -/// Incremented whenever the local peer raises its requested lane count. +const DEMANDED_LANE_COUNT_OFFSET: usize = 0; +/// Incremented whenever the local peer raises its demanded lane count. const DEMAND_EPOCH_OFFSET: usize = 4; const DEMAND_WORDS_SIZE: usize = 8; @@ -195,7 +195,7 @@ impl ControlRingLayout { self.lane_count > 1 && matches!( relative - self.sync_size(), - REQUESTED_LANES_OFFSET | DEMAND_EPOCH_OFFSET + DEMANDED_LANE_COUNT_OFFSET | DEMAND_EPOCH_OFFSET ) } @@ -602,7 +602,7 @@ impl ControlRing { .collect(), lane_demand: (layout.lane_count() > 1).then(|| BrokerLaneDemand { ring: Arc::clone(&ring), - started: 0, + started_lane_count: 0, }), notification_producer: ControlRingProducer::new(ring, layout.notifications()), } @@ -715,7 +715,7 @@ impl LocalLaneDemand { let demand = layout.demand_offset(); self.ring .memory - .store_u32_release(demand + REQUESTED_LANES_OFFSET, lanes.to_le())?; + .store_u32_release(demand + DEMANDED_LANE_COUNT_OFFSET, lanes.to_le())?; self.ring .memory .increment_u32_release(demand + DEMAND_EPOCH_OFFSET)?; @@ -754,7 +754,7 @@ pub enum ControlRingDemandStatus { /// lanes and never takes a lane back. pub struct BrokerLaneDemand { ring: Arc>, - started: usize, + started_lane_count: usize, } impl BrokerLaneDemand { @@ -766,17 +766,17 @@ impl BrokerLaneDemand { .ring .memory .load_u32_acquire(demand + DEMAND_EPOCH_OFFSET)?; - let requested = u32::from_le( + let demanded_lane_count = u32::from_le( self.ring .memory - .load_u32_acquire(demand + REQUESTED_LANES_OFFSET)?, + .load_u32_acquire(demand + DEMANDED_LANE_COUNT_OFFSET)?, ); - let demanded = usize::try_from(requested) + let demanded_lane_count = usize::try_from(demanded_lane_count) .unwrap_or(usize::MAX) .clamp(1, layout.lane_count()); - if self.started < demanded { - let lane = self.started; - self.started += 1; + if self.started_lane_count < demanded_lane_count { + let lane = self.started_lane_count; + self.started_lane_count += 1; return Ok(ControlRingDemandStatus::Lane(lane)); } Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) @@ -1303,7 +1303,7 @@ mod tests { assert!(!multi_lane.permits_u32(padding)); assert!(!multi_lane.permits_u64(padding)); let demand = multi_lane.demand_offset(); - assert!(multi_lane.permits_u32(demand + REQUESTED_LANES_OFFSET)); + assert!(multi_lane.permits_u32(demand + DEMANDED_LANE_COUNT_OFFSET)); assert!(multi_lane.permits_u32(demand + DEMAND_EPOCH_OFFSET)); assert!(!multi_lane.permits_u64(demand)); assert!(!multi_lane.permits_byte_range(demand, 1)); @@ -1533,7 +1533,7 @@ mod tests { }; let mut broker = BrokerLaneDemand { ring: Arc::clone(&ring), - started: 0, + started_lane_count: 0, }; assert_eq!(broker.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); let Ok(ControlRingDemandStatus::Satisfied { wait_epoch }) = broker.try_next_lane() else { @@ -1561,7 +1561,7 @@ mod tests { )); ring.memory .store_u32_release( - ring.layout.demand_offset() + REQUESTED_LANES_OFFSET, + ring.layout.demand_offset() + DEMANDED_LANE_COUNT_OFFSET, u32::MAX.to_le(), ) .unwrap(); From af0e3faf26558a227fd982e96ec8bca2c573211c Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 10:55:41 -0700 Subject: [PATCH 07/12] Name broker-side control-ring types Host like the other transport types BrokerControlRingEndpoints, BrokerControlRingLane, BrokerLaneDemand and into_broker become HostControlRingEndpoints, HostControlRingLane, HostLaneDemand and into_host, matching the Local/Host pairing the channel traits and platform transports use. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_local_vm_userland/src/lib.rs | 2 +- litebox_broker_transport/src/control_ring.rs | 34 +++++++++---------- .../src/memfd.rs | 2 +- .../src/unix_socket/host.rs | 14 ++++---- .../src/unix_socket/local.rs | 6 ++-- .../src/host.rs | 14 ++++---- .../src/named_pipe.rs | 10 +++--- .../src/shared_memory.rs | 12 +++---- litebox_broker_vm_kernel/src/lib.rs | 8 ++--- 9 files changed, 49 insertions(+), 53 deletions(-) diff --git a/litebox_broker_local_vm_userland/src/lib.rs b/litebox_broker_local_vm_userland/src/lib.rs index 7eeded7db1..27acb5df26 100644 --- a/litebox_broker_local_vm_userland/src/lib.rs +++ b/litebox_broker_local_vm_userland/src/lib.rs @@ -762,7 +762,7 @@ mod tests { }; let producer = ControlRing::new(ring(), LAYOUT) .expect("ring") - .into_broker() + .into_host() .notification_producer; (notifications, producer) } diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index 8347e95571..31cd43db78 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -478,7 +478,7 @@ pub struct LocalControlRingLane { } /// The broker peer's ends of one call lane. -pub struct BrokerControlRingLane { +pub struct HostControlRingLane { /// Local-to-broker request consumer. pub request_consumer: ControlRingConsumer, /// Broker-to-local response producer. @@ -496,11 +496,11 @@ pub struct LocalControlRingEndpoints { } /// Role-bound association ring endpoints owned by the broker peer. -pub struct BrokerControlRingEndpoints { +pub struct HostControlRingEndpoints { /// Call lanes, in lane order. - pub lanes: Vec>, + pub lanes: Vec>, /// Lane-demand observer, present when the layout has more than one lane. - pub lane_demand: Option>, + pub lane_demand: Option>, /// Broker-to-local notification producer. pub notification_producer: ControlRingProducer, } @@ -521,7 +521,7 @@ impl LocalControlRingEndpoints { } } -impl BrokerControlRingEndpoints { +impl HostControlRingEndpoints { /// Returns handles that interrupt every wait these endpoints offer. pub fn wake_handles(&self) -> Vec> { self.lanes @@ -533,7 +533,7 @@ impl BrokerControlRingEndpoints { ] }) .chain([self.notification_producer.wake_handle()]) - .chain(self.lane_demand.as_ref().map(BrokerLaneDemand::wake_handle)) + .chain(self.lane_demand.as_ref().map(HostLaneDemand::wake_handle)) .collect() } } @@ -584,12 +584,12 @@ impl ControlRing { } /// Consumes the mapping into endpoints owned by the broker peer. - pub fn into_broker(self) -> BrokerControlRingEndpoints { + pub fn into_host(self) -> HostControlRingEndpoints { let layout = self.layout; let ring = Arc::new(self); - BrokerControlRingEndpoints { + HostControlRingEndpoints { lanes: (0..layout.lane_count()) - .map(|lane| BrokerControlRingLane { + .map(|lane| HostControlRingLane { request_consumer: ControlRingConsumer::new( Arc::clone(&ring), layout.requests(lane), @@ -600,7 +600,7 @@ impl ControlRing { ), }) .collect(), - lane_demand: (layout.lane_count() > 1).then(|| BrokerLaneDemand { + lane_demand: (layout.lane_count() > 1).then(|| HostLaneDemand { ring: Arc::clone(&ring), started_lane_count: 0, }), @@ -735,7 +735,7 @@ impl LocalLaneDemand { } } -/// Result of polling a [`BrokerLaneDemand`]. +/// Result of polling a [`HostLaneDemand`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ControlRingDemandStatus { /// The local peer demands this lane, which the broker has yet to serve. @@ -752,12 +752,12 @@ pub enum ControlRingDemandStatus { /// It hands out each lane, in lane order, once the local peer demands it. /// The local peer can publish any count; the broker clamps it to the layout's /// lanes and never takes a lane back. -pub struct BrokerLaneDemand { +pub struct HostLaneDemand { ring: Arc>, started_lane_count: usize, } -impl BrokerLaneDemand { +impl HostLaneDemand { /// Returns the next demanded lane the broker has yet to serve, if any. pub fn try_next_lane(&mut self) -> Result { let layout = self.ring.layout; @@ -791,7 +791,7 @@ impl BrokerLaneDemand { } } -impl BrokerLaneDemand { +impl HostLaneDemand { /// Waits for new demand after [`ControlRingDemandStatus::Satisfied`]. /// /// The caller must poll again after this possibly spurious wakeup. @@ -1436,7 +1436,7 @@ mod tests { ); assert_eq!(local.wake_handles().len(), 2 * layout.lane_count() + 1); - let broker = test_ring_with(layout).into_broker(); + let broker = test_ring_with(layout).into_host(); assert_eq!(broker.lanes.len(), layout.lane_count()); assert_eq!(broker.lane_demand.is_some(), layout.lane_count() > 1); for (index, lane) in broker.lanes.iter().enumerate() { @@ -1478,7 +1478,7 @@ mod tests { layout.responses(lane), ), }; - let broker = BrokerControlRingLane { + let broker = HostControlRingLane { request_consumer: ControlRingConsumer::new( Arc::clone(&ring), layout.requests(lane), @@ -1531,7 +1531,7 @@ mod tests { let local = LocalLaneDemand { ring: Arc::clone(&ring), }; - let mut broker = BrokerLaneDemand { + let mut broker = HostLaneDemand { ring: Arc::clone(&ring), started_lane_count: 0, }; diff --git a/litebox_broker_transport_linux_userland/src/memfd.rs b/litebox_broker_transport_linux_userland/src/memfd.rs index 7b37bc3334..ee44b2e4a4 100644 --- a/litebox_broker_transport_linux_userland/src/memfd.rs +++ b/litebox_broker_transport_linux_userland/src/memfd.rs @@ -1090,7 +1090,7 @@ mod tests { .request_producer; let mut consumer = ControlRing::new(broker_memory, LAYOUT) .unwrap() - .into_broker() + .into_host() .lanes .swap_remove(0) .request_consumer; diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index cc1dde42cd..7a1d797734 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -28,9 +28,9 @@ use litebox_broker_transport::channel::{ HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, BrokerControlRingLane, BrokerLaneDemand, ControlRing, - ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, - ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, + ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + HostControlRingEndpoints, HostControlRingLane, HostLaneDemand, }; use crate::memfd::{MemfdProcessImage, MemfdSharedMemory}; @@ -100,14 +100,14 @@ pub struct UnixStreamHostSetupChannel { negotiated: bool, } -type BrokerLane = BrokerControlRingLane; +type BrokerLane = HostControlRingLane; /// Source of the call lanes of an active host control-ring association, which /// hands out each lane once the runner starts using it. pub struct UnixControlRingHostCallLaneSource { /// Each lane, or `None` once handed out. lanes: Vec>, - demand: BrokerLaneDemand, + demand: HostLaneDemand, association: Arc, } @@ -202,9 +202,9 @@ impl UnixStreamHostSetupChannel { )); } let ready_token = ring.layout().ready_token(); - let endpoints = ring.into_broker(); + let endpoints = ring.into_host(); let wakes = endpoints.wake_handles(); - let BrokerControlRingEndpoints { + let HostControlRingEndpoints { lanes, lane_demand: Some(demand), notification_producer, diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 71a741041b..83fd8e4b36 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -552,7 +552,7 @@ mod control_ring_tests { use litebox_broker_protocol::{ObjectHandle, RequestId}; use litebox_broker_transport::channel::{LocalCallChannel, LocalSetupChannel}; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, ControlRingDemandStatus, ControlRingProducer, + ControlRingDemandStatus, ControlRingProducer, HostControlRingEndpoints, }; use rustix::fs::{OFlags, fcntl_getfl}; use std::io::{Read, Write}; @@ -564,7 +564,7 @@ mod control_ring_tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; - type Broker = BrokerControlRingEndpoints; + type Broker = HostControlRingEndpoints; const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; @@ -664,7 +664,7 @@ mod control_ring_tests { channel, notifications, shutdown, - broker_ring.into_broker(), + broker_ring.into_host(), peer_stream, ) } diff --git a/litebox_broker_transport_windows_userland/src/host.rs b/litebox_broker_transport_windows_userland/src/host.rs index 523ab05160..0432432a15 100644 --- a/litebox_broker_transport_windows_userland/src/host.rs +++ b/litebox_broker_transport_windows_userland/src/host.rs @@ -21,9 +21,9 @@ use litebox_broker_transport::channel::{ HostReceive, HostSetupChannel, PeerCredential, }; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, BrokerControlRingLane, BrokerLaneDemand, ControlRing, - ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, ControlRingReadError, - ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingProducer, + ControlRingReadError, ControlRingReadStatus, ControlRingWakeHandle, ControlRingWriteStatus, + HostControlRingEndpoints, HostControlRingLane, HostLaneDemand, }; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::System::Pipes::GetNamedPipeClientProcessId; @@ -46,14 +46,14 @@ pub struct WindowsNamedPipeHostSetupChannel { negotiated: bool, } -type BrokerLane = BrokerControlRingLane; +type BrokerLane = HostControlRingLane; /// Source of the call lanes of an active host control-ring association, which /// hands out each lane once the runner starts using it. pub struct WindowsControlRingHostCallLaneSource { /// Each lane, or `None` once handed out. lanes: Vec>, - demand: BrokerLaneDemand, + demand: HostLaneDemand, association: Arc, } @@ -231,9 +231,9 @@ fn activate_host( )); } let ready_token = ring.layout().ready_token(); - let endpoints = ring.into_broker(); + let endpoints = ring.into_host(); let wakes = endpoints.wake_handles(); - let BrokerControlRingEndpoints { + let HostControlRingEndpoints { lanes, lane_demand: Some(demand), notification_producer, diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index 1b0959cf50..fa4a1a722d 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -162,9 +162,9 @@ mod tests { LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, ControlRing, ControlRingConsumer, ControlRingDemandStatus, - ControlRingLayout, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, - LocalControlRingEndpoints, + ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingLayout, + ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, + HostControlRingEndpoints, LocalControlRingEndpoints, }; use litebox_broker_transport::shared_memory::SharedMemory as _; use windows_sys::Win32::System::Threading::GetCurrentProcessId; @@ -172,7 +172,7 @@ mod tests { type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; type Local = LocalControlRingEndpoints; - type Broker = BrokerControlRingEndpoints; + type Broker = HostControlRingEndpoints; const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); const LAYOUT: ControlRingLayout = ControlRingLayout::MULTI_LANE; @@ -324,7 +324,7 @@ mod tests { }); local.into_active(local_ring).unwrap() }); - (calls, notifications, broker_ring.into_broker(), host) + (calls, notifications, broker_ring.into_host(), host) } fn demand_lanes(local: &Local, lanes: usize) { diff --git a/litebox_broker_transport_windows_userland/src/shared_memory.rs b/litebox_broker_transport_windows_userland/src/shared_memory.rs index 8647f0864a..9ea2a01dea 100644 --- a/litebox_broker_transport_windows_userland/src/shared_memory.rs +++ b/litebox_broker_transport_windows_userland/src/shared_memory.rs @@ -674,9 +674,7 @@ mod tests { let local_memory = WindowsSharedMemory::create_control_ring(LAYOUT).unwrap(); let broker_memory = local_memory.try_clone_view().unwrap(); let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory, LAYOUT) - .unwrap() - .into_broker(); + let mut broker = ControlRing::new(broker_memory, LAYOUT).unwrap().into_host(); assert_eq!( local.lanes[0] @@ -706,7 +704,7 @@ mod tests { .unwrap(); let mut broker = ControlRing::new(broker_memory, LAYOUT) .unwrap() - .into_broker() + .into_host() .lanes .pop() .unwrap(); @@ -746,7 +744,7 @@ mod tests { .unwrap(); let mut broker_demand = ControlRing::new(broker_memory, LAYOUT) .unwrap() - .into_broker() + .into_host() .lane_demand .unwrap(); @@ -830,9 +828,7 @@ mod tests { unsafe { WindowsSharedMemory::control_ring_from_transferred(transfer, LAYOUT) } .unwrap(); let mut local = ControlRing::new(local_memory, LAYOUT).unwrap().into_local(); - let mut broker = ControlRing::new(broker_memory, LAYOUT) - .unwrap() - .into_broker(); + let mut broker = ControlRing::new(broker_memory, LAYOUT).unwrap().into_host(); assert_eq!( local.lanes[0] diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index b32a78f74c..fcc0d6c47c 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -80,8 +80,8 @@ use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER use litebox_broker_protocol::wire; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::control_ring::{ - BrokerControlRingEndpoints, ControlRing, ControlRingConsumer, ControlRingLayout, - ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingProducer, + ControlRingReadStatus, ControlRingWriteStatus, HostControlRingEndpoints, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool}; use litebox_common_vm_abi::{Status, UserRange, WireFrame}; @@ -572,11 +572,11 @@ impl Association { return Err(Status::Denied); } let words = ring.memory().clone(); - let BrokerControlRingEndpoints { + let HostControlRingEndpoints { lanes, lane_demand: _, notification_producer, - } = ring.into_broker(); + } = ring.into_host(); let [lane] = <[_; 1]>::try_from(lanes) .unwrap_or_else(|_| unreachable!("a single-lane layout has one lane")); self.readiness.activate(notification_producer); From 366788261bd0ac2f577cee8c6bfddf710a2af7f0 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 11:06:09 -0700 Subject: [PATCH 08/12] Make the call-lane pool a std-only LocalCallLanePool Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/Cargo.toml | 4 + .../src/call_lane_pool.rs | 290 ++++++++++++------ litebox_broker_transport/src/lib.rs | 10 +- .../Cargo.toml | 2 +- .../src/call_lane_pool.rs | 197 ------------ .../src/lib.rs | 1 - .../src/unix_socket/local.rs | 56 ++-- .../Cargo.toml | 2 +- .../src/call_lane_pool.rs | 197 ------------ .../src/lib.rs | 1 - .../src/local.rs | 45 ++- 11 files changed, 242 insertions(+), 563 deletions(-) delete mode 100644 litebox_broker_transport_linux_userland/src/call_lane_pool.rs delete mode 100644 litebox_broker_transport_windows_userland/src/call_lane_pool.rs diff --git a/litebox_broker_transport/Cargo.toml b/litebox_broker_transport/Cargo.toml index 94efa2df8a..6d6e123580 100644 --- a/litebox_broker_transport/Cargo.toml +++ b/litebox_broker_transport/Cargo.toml @@ -3,6 +3,10 @@ name = "litebox_broker_transport" version = "0.1.0" edition = "2024" +[features] +# Local call-lane pool for hosted local peers. +std = [] + [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs index f934690f53..2548a823b1 100644 --- a/litebox_broker_transport/src/call_lane_pool.rs +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -1,68 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Portable call-lane admission over caller-supplied synchronization. +//! Local-side admission of concurrent calls to call lanes, for hosted peers. -use alloc::sync::Arc; +use alloc::string::ToString; use alloc::vec::Vec; use core::ops::{Deref, DerefMut}; - -/// A mutex usable by [`CallLanePool`]. -pub trait CallLanePoolMutex { - /// Guard granting access to the protected value. - type Guard<'a>: Deref + DerefMut - where - Self: 'a, - T: 'a; - - /// Locks the mutex. - fn lock(&self) -> Self::Guard<'_>; -} - -/// A condition variable paired with a [`CallLanePoolMutex`]. -pub trait CallLanePoolCondvar { - /// Mutex type whose guard this condition variable waits on. - type Mutex: CallLanePoolMutex; - - /// Atomically releases the guard and waits for a notification. - fn wait<'a>( - &self, - guard: >::Guard<'a>, - ) -> >::Guard<'a> - where - Self::Mutex: 'a, - T: 'a; - - /// Wakes one waiter. - fn notify_one(&self); - - /// Wakes all waiters. - fn notify_all(&self); -} - -/// Supplies synchronization primitives for [`CallLanePool`]. -pub trait CallLanePoolSync { - /// Mutex protecting a value of type `T`. - type Mutex: CallLanePoolMutex; - - /// Condition variable paired with [`Self::Mutex`]. - type Condvar: CallLanePoolCondvar>; - - /// Creates a mutex protecting `value`. - fn mutex(value: T) -> Self::Mutex; - - /// Creates a condition variable. - fn condvar() -> Self::Condvar; -} - -/// Failure from call-lane coordination. -#[derive(Debug)] -pub enum CallLanePoolError { - /// The association had already failed. - AssociationFailed(Arc), - /// The guarded operation failed. - Operation(Error), -} +use std::io::Error; +use std::sync::{Condvar, Mutex, MutexGuard}; /// Admission of concurrent calls to a fixed set of call lanes. /// @@ -71,13 +16,13 @@ pub enum CallLanePoolError { /// lowest free lane, so an association uses only as many lanes as it has /// concurrent calls; callers beyond the lane count wait for a lane to free up. /// Lane 0 is always demanded; before a call first claims any higher lane, the -/// claim raises the demand so the peer starts serving that lane. -pub struct CallLanePool { - state: Sync::Mutex>, - lane_freed: Sync::Condvar>, +/// claim raises the demand so the broker starts serving that lane. +pub struct LocalCallLanePool { + state: Mutex>, + lane_freed: Condvar, } -struct CallLanePoolInner { +struct LocalCallLanePoolInner { /// Each lane, or `None` while a call holds it. lanes: Vec>, /// Number of lanes demanded so far. @@ -86,18 +31,28 @@ struct CallLanePoolInner { /// if one waits, since a notification can cost a system call even with no /// waiters. waiting_claim_count: usize, - failure: Option>, + failure: Option, +} + +impl LocalCallLanePoolInner { + /// Fails with a copy of the association's failure, if one was recorded. + fn check_live(&self) -> Result<(), Error> { + match &self.failure { + Some(failure) => Err(copy_error(failure)), + None => Ok(()), + } + } } /// A lane held by one call; releasing or dropping the claim frees the lane. -pub struct LaneClaim<'lanes, Sync: CallLanePoolSync, Lane, Error> { - lanes: &'lanes CallLanePool, +pub struct LaneClaim<'lanes, Lane> { + lanes: &'lanes LocalCallLanePool, index: usize, /// The lane, or `None` once released. lane: Option, } -impl Deref for LaneClaim<'_, Sync, Lane, Error> { +impl Deref for LaneClaim<'_, Lane> { type Target = Lane; fn deref(&self) -> &Lane { @@ -107,7 +62,7 @@ impl Deref for LaneClaim<'_, Sync, Lane, Er } } -impl DerefMut for LaneClaim<'_, Sync, Lane, Error> { +impl DerefMut for LaneClaim<'_, Lane> { fn deref_mut(&mut self) -> &mut Lane { self.lane .as_mut() @@ -115,16 +70,16 @@ impl DerefMut for LaneClaim<'_, Sync, Lane, } } -impl Drop for LaneClaim<'_, Sync, Lane, Error> { +impl Drop for LaneClaim<'_, Lane> { fn drop(&mut self) { if self.lane.is_some() { let lanes = self.lanes; - self.free(&mut lanes.state.lock()); + self.free(&mut lanes.lock()); } } } -impl LaneClaim<'_, Sync, Lane, Error> { +impl LaneClaim<'_, Lane> { /// Runs `operation` on the lane and frees the lane, both while excluding /// failure recording, so a call completes only while the association is /// live. @@ -134,19 +89,17 @@ impl LaneClaim<'_, Sync, Lane, Error> { pub fn release_if_live( &mut self, operation: impl FnOnce(&mut Lane) -> Result, - ) -> Result> { + ) -> Result { let lanes = self.lanes; - let mut state = lanes.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); - } - let value = operation(self).map_err(CallLanePoolError::Operation)?; + let mut state = lanes.lock(); + state.check_live()?; + let value = operation(self)?; self.free(&mut state); Ok(value) } /// Returns the lane to its slot, under the admission lock. - fn free(&mut self, state: &mut CallLanePoolInner) { + fn free(&mut self, state: &mut LocalCallLanePoolInner) { state.lanes[self.index] = self.lane.take(); if state.waiting_claim_count > 0 { self.lanes.lane_freed.notify_one(); @@ -154,7 +107,7 @@ impl LaneClaim<'_, Sync, Lane, Error> { } } -impl CallLanePool { +impl LocalCallLanePool { /// Creates live admission over `lanes`, of which only lane 0 is demanded. /// /// # Panics @@ -163,13 +116,13 @@ impl CallLanePool { pub fn new(lanes: Vec) -> Self { assert!(!lanes.is_empty(), "call lanes need at least one lane"); Self { - state: Sync::mutex(CallLanePoolInner { + state: Mutex::new(LocalCallLanePoolInner { lanes: lanes.into_iter().map(Some).collect(), demanded_lane_count: 1, waiting_claim_count: 0, failure: None, }), - lane_freed: Sync::condvar(), + lane_freed: Condvar::new(), } } @@ -181,21 +134,19 @@ impl CallLanePool { pub fn claim( &self, demand: impl FnOnce(usize) -> Result<(), Error>, - ) -> Result, CallLanePoolError> { - let mut state = self.state.lock(); + ) -> Result, Error> { + let mut state = self.lock(); let index = loop { - if let Some(error) = state.failure.as_ref() { - return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); - } + state.check_live()?; if let Some(index) = state.lanes.iter().position(Option::is_some) { break index; } state.waiting_claim_count += 1; - state = self.lane_freed.wait(state); + state = self.wait_for_free_lane(state); state.waiting_claim_count -= 1; }; if index >= state.demanded_lane_count { - demand(index + 1).map_err(CallLanePoolError::Operation)?; + demand(index + 1)?; state.demanded_lane_count = index + 1; } Ok(LaneClaim { @@ -208,8 +159,8 @@ impl CallLanePool { /// Records the first terminal failure and wakes every waiting claim. /// /// Returns whether this call recorded the first failure. - pub fn record_failure(&self, error: Arc) -> bool { - let mut state = self.state.lock(); + pub fn record_failure(&self, error: Error) -> bool { + let mut state = self.lock(); if state.failure.is_some() { return false; } @@ -218,20 +169,155 @@ impl CallLanePool { true } - /// Returns the association's terminal failure, if one was recorded. - pub fn current_failure(&self) -> Option> { - self.state.lock().failure.as_ref().map(Arc::clone) + /// Returns a copy of the association's terminal failure, if one was + /// recorded. + pub fn current_failure(&self) -> Option { + self.lock().failure.as_ref().map(copy_error) } /// Runs an operation while excluding failure recording. - pub fn run_if_live( + pub fn run_if_live(&self, operation: impl FnOnce() -> Result) -> Result { + let state = self.lock(); + state.check_live()?; + operation() + } + + fn lock(&self) -> MutexGuard<'_, LocalCallLanePoolInner> { + self.state.lock().expect("broker call-lanes mutex poisoned") + } + + fn wait_for_free_lane<'a>( &self, - operation: impl FnOnce() -> Result, - ) -> Result> { - let state = self.state.lock(); - if let Some(error) = state.failure.as_ref() { - return Err(CallLanePoolError::AssociationFailed(Arc::clone(error))); - } - operation().map_err(CallLanePoolError::Operation) + state: MutexGuard<'a, LocalCallLanePoolInner>, + ) -> MutexGuard<'a, LocalCallLanePoolInner> { + self.lane_freed + .wait(state) + .expect("broker call-lanes mutex poisoned") + } +} + +/// Copies `error`, since [`Error`] is not `Clone`. +fn copy_error(error: &Error) -> Error { + match error.raw_os_error() { + Some(code) => Error::from_raw_os_error(code), + None => Error::new(error.kind(), error.to_string()), + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + use std::io::{Error, ErrorKind}; + use std::sync::mpsc; + use std::time::Duration; + + use super::{LocalCallLanePool, Vec}; + + fn no_demand(_: usize) -> Result<(), Error> { + unreachable!("lane 0 needs no demand") + } + + #[test] + fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { + let lanes = LocalCallLanePool::new(vec![0, 1, 2]); + let mut demands = Vec::new(); + let first = lanes.claim(no_demand).unwrap(); + let second = lanes + .claim(|count| { + demands.push(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*first, *second), (0, 1)); + drop(first); + let reused = lanes.claim(no_demand).unwrap(); + assert_eq!(*reused, 0); + drop((reused, second)); + let again = lanes.claim(no_demand).unwrap(); + let next = lanes + .claim(|_| panic!("lane 1 is already demanded")) + .unwrap(); + assert_eq!((*again, *next), (0, 1)); + assert_eq!(demands, [2]); + } + + #[test] + fn a_failed_demand_leaves_the_lane_free_and_undemanded() { + let lanes = LocalCallLanePool::new(vec![0, 1]); + let _first = lanes.claim(no_demand).unwrap(); + assert!(matches!( + lanes.claim(|_| Err(Error::from(ErrorKind::Other))), + Err(error) if error.kind() == ErrorKind::Other + )); + let mut demanded = None; + let second = lanes + .claim(|count| { + demanded = Some(count); + Ok(()) + }) + .unwrap(); + assert_eq!((*second, demanded), (1, Some(2))); + } + + #[test] + fn claims_wait_while_every_lane_is_held() { + let lanes = LocalCallLanePool::new(vec![0]); + let held = lanes.claim(no_demand).unwrap(); + let (claimed, claim) = mpsc::channel(); + std::thread::scope(|scope| { + scope.spawn(|| { + let lane = lanes.claim(no_demand).map(|lane| *lane); + claimed.send(lane.is_ok()).unwrap(); + }); + assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); + drop(held); + assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); + }); + } + + #[test] + fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { + let lanes = LocalCallLanePool::new(vec![0, 1]); + let mut released = lanes.claim(no_demand).unwrap(); + assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); + // Dropping a released claim leaves its lane free. + drop(released); + let mut held = lanes.claim(no_demand).unwrap(); + assert_eq!(*held, 0); + assert!(matches!( + held.release_if_live(|_| Err::<(), _>(Error::from(ErrorKind::Other))), + Err(error) if error.kind() == ErrorKind::Other + )); + assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); + assert!(lanes.record_failure(Error::from(ErrorKind::BrokenPipe))); + assert!(matches!( + held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); + assert_eq!(*held, 0); + } + + #[test] + fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { + let lanes = LocalCallLanePool::new(vec![0]); + let _held = lanes.claim(no_demand).unwrap(); + std::thread::scope(|scope| { + let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); + std::thread::sleep(Duration::from_millis(10)); + assert!(lanes.record_failure(Error::from(ErrorKind::BrokenPipe))); + assert!(matches!( + waiter.join().unwrap(), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); + }); + assert!(!lanes.record_failure(Error::from(ErrorKind::Other))); + assert_eq!( + lanes.current_failure().unwrap().kind(), + ErrorKind::BrokenPipe + ); + assert!(matches!( + lanes.run_if_live(|| Ok(())), + Err(error) if error.kind() == ErrorKind::BrokenPipe + )); } } diff --git a/litebox_broker_transport/src/lib.rs b/litebox_broker_transport/src/lib.rs index 2d0259470f..a99c052b62 100644 --- a/litebox_broker_transport/src/lib.rs +++ b/litebox_broker_transport/src/lib.rs @@ -9,9 +9,10 @@ //! shared-buffer pool built on them, and the portable shared control-ring state //! machines. //! -//! Nothing here is tied to an operating system or to hosted userland. The crate -//! is unconditionally `no_std`, so a kernel deployment can use the same -//! interfaces and control rings. Concrete association bindings, such as the +//! Nothing here is tied to an operating system or to hosted userland, except +//! the local call-lane pool behind the opt-in `std` feature. The crate is +//! `no_std` without it, so a kernel deployment can use the same interfaces and +//! control rings. Concrete association bindings, such as the //! Unix-domain-socket and memfd endpoints in //! `litebox_broker_transport_linux_userland`, live in separate crates, and a //! deployment may implement the channel traits directly instead of using the @@ -25,9 +26,10 @@ extern crate alloc; -#[cfg(test)] +#[cfg(any(test, feature = "std"))] extern crate std; +#[cfg(feature = "std")] pub mod call_lane_pool; pub mod channel; pub mod control_ring; diff --git a/litebox_broker_transport_linux_userland/Cargo.toml b/litebox_broker_transport_linux_userland/Cargo.toml index 382b1ac501..eda1665ec4 100644 --- a/litebox_broker_transport_linux_userland/Cargo.toml +++ b/litebox_broker_transport_linux_userland/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } -litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0" } +litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0", features = ["std"] } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/litebox_broker_transport_linux_userland/src/call_lane_pool.rs b/litebox_broker_transport_linux_userland/src/call_lane_pool.rs deleted file mode 100644 index 65c7ff64e2..0000000000 --- a/litebox_broker_transport_linux_userland/src/call_lane_pool.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::call_lane_pool::{ - CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, - CallLanePoolSync, LaneClaim as GenericLaneClaim, -}; - -use crate::setup::copy_io_error; - -pub(crate) type CallLanePool = GenericCallLanePool; - -pub(crate) type LaneClaim<'lanes, Lane> = - GenericLaneClaim<'lanes, StdCallLanePoolSync, Lane, Error>; - -pub(crate) struct StdCallLanePoolSync; - -pub(crate) struct StdCallLanePoolMutex(Mutex); - -pub(crate) struct StdCallLanePoolCondvar(Condvar, PhantomData); - -impl CallLanePoolMutex for StdCallLanePoolMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker call-lanes mutex poisoned") - } -} - -impl CallLanePoolCondvar for StdCallLanePoolCondvar { - type Mutex = StdCallLanePoolMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0 - .wait(guard) - .expect("broker call-lanes mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl CallLanePoolSync for StdCallLanePoolSync { - type Mutex = StdCallLanePoolMutex; - type Condvar = StdCallLanePoolCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdCallLanePoolMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdCallLanePoolCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn call_lane_pool_error(error: CallLanePoolError) -> Error { - match error { - CallLanePoolError::AssociationFailed(error) => copy_io_error(&error), - CallLanePoolError::Operation(error) => error, - } -} - -#[cfg(test)] -mod tests { - use std::io::{Error, ErrorKind}; - use std::sync::Arc; - use std::sync::mpsc; - use std::time::Duration; - - use litebox_broker_transport::call_lane_pool::CallLanePoolError; - - use super::CallLanePool; - - fn no_demand(_: usize) -> Result<(), Error> { - unreachable!("lane 0 needs no demand") - } - - #[test] - fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { - let lanes = CallLanePool::new(vec![0, 1, 2]); - let mut demands = Vec::new(); - let first = lanes.claim(no_demand).unwrap(); - let second = lanes - .claim(|count| { - demands.push(count); - Ok(()) - }) - .unwrap(); - assert_eq!((*first, *second), (0, 1)); - drop(first); - let reused = lanes.claim(no_demand).unwrap(); - assert_eq!(*reused, 0); - drop((reused, second)); - let again = lanes.claim(no_demand).unwrap(); - let next = lanes - .claim(|_| panic!("lane 1 is already demanded")) - .unwrap(); - assert_eq!((*again, *next), (0, 1)); - assert_eq!(demands, [2]); - } - - #[test] - fn a_failed_demand_leaves_the_lane_free_and_undemanded() { - let lanes = CallLanePool::new(vec![0, 1]); - let _first = lanes.claim(no_demand).unwrap(); - assert!(matches!( - lanes.claim(|_| Err(Error::other("demand failed"))), - Err(CallLanePoolError::Operation(_)) - )); - let mut demanded = None; - let second = lanes - .claim(|count| { - demanded = Some(count); - Ok(()) - }) - .unwrap(); - assert_eq!((*second, demanded), (1, Some(2))); - } - - #[test] - fn claims_wait_while_every_lane_is_held() { - let lanes = CallLanePool::new(vec![0]); - let held = lanes.claim(no_demand).unwrap(); - let (claimed, claim) = mpsc::channel(); - std::thread::scope(|scope| { - scope.spawn(|| { - let lane = lanes.claim(no_demand).map(|lane| *lane); - claimed.send(lane.is_ok()).unwrap(); - }); - assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); - drop(held); - assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); - }); - } - - #[test] - fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { - let lanes = CallLanePool::new(vec![0, 1]); - let mut released = lanes.claim(no_demand).unwrap(); - assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); - // Dropping a released claim leaves its lane free. - drop(released); - let mut held = lanes.claim(no_demand).unwrap(); - assert_eq!(*held, 0); - assert!(matches!( - held.release_if_live(|_| Err::<(), _>(Error::other("release failed"))), - Err(CallLanePoolError::Operation(_)) - )); - assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); - assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); - assert!(matches!( - held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), - Err(CallLanePoolError::AssociationFailed(_)) - )); - assert_eq!(*held, 0); - } - - #[test] - fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { - let lanes = CallLanePool::new(vec![0]); - let _held = lanes.claim(no_demand).unwrap(); - std::thread::scope(|scope| { - let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); - std::thread::sleep(Duration::from_millis(10)); - assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); - assert!(matches!( - waiter.join().unwrap(), - Err(CallLanePoolError::AssociationFailed(_)) - )); - }); - assert!(!lanes.record_failure(Arc::new(Error::from(ErrorKind::Other)))); - assert_eq!( - lanes.current_failure().unwrap().kind(), - ErrorKind::BrokenPipe - ); - assert!(matches!( - lanes.run_if_live(|| Ok(())), - Err(CallLanePoolError::AssociationFailed(_)) - )); - } -} diff --git a/litebox_broker_transport_linux_userland/src/lib.rs b/litebox_broker_transport_linux_userland/src/lib.rs index e8a9a36eae..8d5de7b14f 100644 --- a/litebox_broker_transport_linux_userland/src/lib.rs +++ b/litebox_broker_transport_linux_userland/src/lib.rs @@ -19,7 +19,6 @@ #![cfg(target_os = "linux")] -mod call_lane_pool; mod setup; pub mod existing_runner; diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 83fd8e4b36..b1f017e247 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -30,7 +30,7 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; -use litebox_broker_transport::call_lane_pool::CallLanePoolError; +use litebox_broker_transport::call_lane_pool::{LaneClaim, LocalCallLanePool}; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; @@ -40,7 +40,6 @@ use litebox_broker_transport::control_ring::{ LocalControlRingEndpoints, LocalControlRingLane, LocalLaneDemand, }; -use crate::call_lane_pool::{CallLanePool, LaneClaim, call_lane_pool_error}; use crate::memfd::MemfdSharedMemory; use crate::setup::{ copy_io_error, invalid_data, read_setup_frame, ring_error, shutdown_socket, wire_error, @@ -73,7 +72,7 @@ type LocalLane = LocalControlRingLane; /// call lanes and their demand, the setup socket used for liveness and /// teardown, and the wake handles of every ring wait. struct LocalRingAssociation { - lanes: CallLanePool, + lanes: LocalCallLanePool, lane_demand: LocalLaneDemand, control_stream: UnixStream, /// Whether the call channel was dropped, which ends the association as a @@ -206,7 +205,7 @@ impl UnixStreamLocalSetupChannel { let shutdown_stream = setup_stream.try_clone()?; let association = Arc::new(LocalRingAssociation { - lanes: CallLanePool::new(lanes), + lanes: LocalCallLanePool::new(lanes), lane_demand, control_stream: shutdown_stream, closed: AtomicBool::new(false), @@ -348,8 +347,7 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { .claim(|lanes| association.demand_lanes(lanes)) { Ok(lane) => lane, - Err(CallLanePoolError::AssociationFailed(error)) => return Err(copy_io_error(&error)), - Err(CallLanePoolError::Operation(error)) => { + Err(error) => { let result = Err(copy_io_error(&error)); let _ = association.fail(error); return result; @@ -372,7 +370,7 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { if let Some(error) = self.association.lanes.current_failure() { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } match self.consumer.try_read(decode_notification) { Ok(ControlRingReadStatus::Message(notification)) => { @@ -380,13 +378,13 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { .association .acknowledge_notification(&mut self.consumer) { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { if let Some(error) = self.association.lanes.current_failure() { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); @@ -414,11 +412,11 @@ impl LocalNotificationChannel for UnixControlRingLocalNotificationChannel { impl LocalRingAssociation { /// Ends notification receipt once the association has ended with /// `failure`: cleanly if the local side closed it. - fn end_notifications(&self, failure: &Error) -> IoResult> { + fn end_notifications(&self, failure: Error) -> IoResult> { if self.closed.load(Ordering::Acquire) { Ok(None) } else { - Err(copy_io_error(failure)) + Err(failure) } } @@ -426,15 +424,12 @@ impl LocalRingAssociation { &self, consumer: &mut ControlRingConsumer, ) -> IoResult<()> { - let result = self - .lanes - .run_if_live(|| { - consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - }) - .map_err(call_lane_pool_error); + let result = self.lanes.run_if_live(|| { + consumer + .publish_head() + .map_err(ring_error) + .and_then(|()| consumer.wake_producer()) + }); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -462,8 +457,7 @@ impl LocalRingAssociation { loop { let write_status = self .lanes - .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) - .map_err(call_lane_pool_error)?; + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error))?; match write_status { ControlRingWriteStatus::Written => break, ControlRingWriteStatus::Full { wait_epoch } => { @@ -479,7 +473,7 @@ impl LocalRingAssociation { Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { if let Some(error) = self.lanes.current_failure() { - return Err(copy_io_error(&error)); + return Err(error); } consumer.wait_for_message(wait_epoch)?; } @@ -496,13 +490,12 @@ impl LocalRingAssociation { let consumer = &mut lane.response_consumer; consumer.publish_head().map_err(ring_error)?; consumer.wake_producer() - }) - .map_err(call_lane_pool_error)?; + })?; Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - let first_failure = self.lanes.record_failure(Arc::new(error)); + let first_failure = self.lanes.record_failure(error); let wakes = self .wakes .iter() @@ -1039,7 +1032,7 @@ mod control_ring_tests { #[test] fn failure_recording_waits_for_in_progress_publication() { - let pending = Arc::new(CallLanePool::new(vec![()])); + let pending = Arc::new(LocalCallLanePool::new(vec![()])); let publication_state = Arc::new(AtomicUsize::new(0)); let (publication_started, wait_for_publication) = std::sync::mpsc::sync_channel(0); let (release_publication, publication_released) = std::sync::mpsc::sync_channel(0); @@ -1064,10 +1057,8 @@ mod control_ring_tests { let failure_state = Arc::clone(&publication_state); let failure = thread::spawn(move || { failure_started.send(()).unwrap(); - failure_pending.record_failure(Arc::new(Error::new( - ErrorKind::ConnectionAborted, - "test failure", - ))); + failure_pending + .record_failure(Error::new(ErrorKind::ConnectionAborted, "test failure")); assert_eq!(failure_state.load(Ordering::Acquire), 2); failure_recorded.send(()).unwrap(); }); @@ -1085,8 +1076,7 @@ mod control_ring_tests { failure.join().unwrap(); assert!(matches!( pending.claim(|_| Ok(())), - Err(CallLanePoolError::AssociationFailed(error)) - if error.kind() == ErrorKind::ConnectionAborted + Err(error) if error.kind() == ErrorKind::ConnectionAborted )); } } diff --git a/litebox_broker_transport_windows_userland/Cargo.toml b/litebox_broker_transport_windows_userland/Cargo.toml index 0cbe83e836..430ad4259c 100644 --- a/litebox_broker_transport_windows_userland/Cargo.toml +++ b/litebox_broker_transport_windows_userland/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } -litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0" } +litebox_broker_transport = { path = "../litebox_broker_transport", version = "0.1.0", features = ["std"] } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(windows)'.dependencies] diff --git a/litebox_broker_transport_windows_userland/src/call_lane_pool.rs b/litebox_broker_transport_windows_userland/src/call_lane_pool.rs deleted file mode 100644 index 65c7ff64e2..0000000000 --- a/litebox_broker_transport_windows_userland/src/call_lane_pool.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use std::io::Error; -use std::marker::PhantomData; -use std::sync::{Condvar, Mutex, MutexGuard}; - -use litebox_broker_transport::call_lane_pool::{ - CallLanePool as GenericCallLanePool, CallLanePoolCondvar, CallLanePoolError, CallLanePoolMutex, - CallLanePoolSync, LaneClaim as GenericLaneClaim, -}; - -use crate::setup::copy_io_error; - -pub(crate) type CallLanePool = GenericCallLanePool; - -pub(crate) type LaneClaim<'lanes, Lane> = - GenericLaneClaim<'lanes, StdCallLanePoolSync, Lane, Error>; - -pub(crate) struct StdCallLanePoolSync; - -pub(crate) struct StdCallLanePoolMutex(Mutex); - -pub(crate) struct StdCallLanePoolCondvar(Condvar, PhantomData); - -impl CallLanePoolMutex for StdCallLanePoolMutex { - type Guard<'a> - = MutexGuard<'a, T> - where - T: 'a; - - fn lock(&self) -> Self::Guard<'_> { - self.0.lock().expect("broker call-lanes mutex poisoned") - } -} - -impl CallLanePoolCondvar for StdCallLanePoolCondvar { - type Mutex = StdCallLanePoolMutex; - - fn wait<'a>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> - where - Self::Mutex: 'a, - T: 'a, - { - self.0 - .wait(guard) - .expect("broker call-lanes mutex poisoned") - } - - fn notify_one(&self) { - self.0.notify_one(); - } - - fn notify_all(&self) { - self.0.notify_all(); - } -} - -impl CallLanePoolSync for StdCallLanePoolSync { - type Mutex = StdCallLanePoolMutex; - type Condvar = StdCallLanePoolCondvar; - - fn mutex(value: T) -> Self::Mutex { - StdCallLanePoolMutex(Mutex::new(value)) - } - - fn condvar() -> Self::Condvar { - StdCallLanePoolCondvar(Condvar::new(), PhantomData) - } -} - -pub(crate) fn call_lane_pool_error(error: CallLanePoolError) -> Error { - match error { - CallLanePoolError::AssociationFailed(error) => copy_io_error(&error), - CallLanePoolError::Operation(error) => error, - } -} - -#[cfg(test)] -mod tests { - use std::io::{Error, ErrorKind}; - use std::sync::Arc; - use std::sync::mpsc; - use std::time::Duration; - - use litebox_broker_transport::call_lane_pool::CallLanePoolError; - - use super::CallLanePool; - - fn no_demand(_: usize) -> Result<(), Error> { - unreachable!("lane 0 needs no demand") - } - - #[test] - fn calls_claim_the_lowest_free_lane_and_demand_each_new_lane_once() { - let lanes = CallLanePool::new(vec![0, 1, 2]); - let mut demands = Vec::new(); - let first = lanes.claim(no_demand).unwrap(); - let second = lanes - .claim(|count| { - demands.push(count); - Ok(()) - }) - .unwrap(); - assert_eq!((*first, *second), (0, 1)); - drop(first); - let reused = lanes.claim(no_demand).unwrap(); - assert_eq!(*reused, 0); - drop((reused, second)); - let again = lanes.claim(no_demand).unwrap(); - let next = lanes - .claim(|_| panic!("lane 1 is already demanded")) - .unwrap(); - assert_eq!((*again, *next), (0, 1)); - assert_eq!(demands, [2]); - } - - #[test] - fn a_failed_demand_leaves_the_lane_free_and_undemanded() { - let lanes = CallLanePool::new(vec![0, 1]); - let _first = lanes.claim(no_demand).unwrap(); - assert!(matches!( - lanes.claim(|_| Err(Error::other("demand failed"))), - Err(CallLanePoolError::Operation(_)) - )); - let mut demanded = None; - let second = lanes - .claim(|count| { - demanded = Some(count); - Ok(()) - }) - .unwrap(); - assert_eq!((*second, demanded), (1, Some(2))); - } - - #[test] - fn claims_wait_while_every_lane_is_held() { - let lanes = CallLanePool::new(vec![0]); - let held = lanes.claim(no_demand).unwrap(); - let (claimed, claim) = mpsc::channel(); - std::thread::scope(|scope| { - scope.spawn(|| { - let lane = lanes.claim(no_demand).map(|lane| *lane); - claimed.send(lane.is_ok()).unwrap(); - }); - assert!(claim.recv_timeout(Duration::from_millis(50)).is_err()); - drop(held); - assert!(claim.recv_timeout(Duration::from_secs(10)).unwrap()); - }); - } - - #[test] - fn a_release_frees_the_lane_only_if_live_and_its_operation_succeeds() { - let lanes = CallLanePool::new(vec![0, 1]); - let mut released = lanes.claim(no_demand).unwrap(); - assert_eq!(released.release_if_live(|lane| Ok(*lane)).unwrap(), 0); - // Dropping a released claim leaves its lane free. - drop(released); - let mut held = lanes.claim(no_demand).unwrap(); - assert_eq!(*held, 0); - assert!(matches!( - held.release_if_live(|_| Err::<(), _>(Error::other("release failed"))), - Err(CallLanePoolError::Operation(_)) - )); - assert_eq!(*lanes.claim(|_| Ok(())).unwrap(), 1); - assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); - assert!(matches!( - held.release_if_live(|_| -> Result<(), Error> { unreachable!() }), - Err(CallLanePoolError::AssociationFailed(_)) - )); - assert_eq!(*held, 0); - } - - #[test] - fn a_failure_wakes_waiting_claims_and_rejects_new_ones() { - let lanes = CallLanePool::new(vec![0]); - let _held = lanes.claim(no_demand).unwrap(); - std::thread::scope(|scope| { - let waiter = scope.spawn(|| lanes.claim(no_demand).map(|lane| *lane)); - std::thread::sleep(Duration::from_millis(10)); - assert!(lanes.record_failure(Arc::new(Error::from(ErrorKind::BrokenPipe)))); - assert!(matches!( - waiter.join().unwrap(), - Err(CallLanePoolError::AssociationFailed(_)) - )); - }); - assert!(!lanes.record_failure(Arc::new(Error::from(ErrorKind::Other)))); - assert_eq!( - lanes.current_failure().unwrap().kind(), - ErrorKind::BrokenPipe - ); - assert!(matches!( - lanes.run_if_live(|| Ok(())), - Err(CallLanePoolError::AssociationFailed(_)) - )); - } -} diff --git a/litebox_broker_transport_windows_userland/src/lib.rs b/litebox_broker_transport_windows_userland/src/lib.rs index ac8e9a6c6c..18f14354ce 100644 --- a/litebox_broker_transport_windows_userland/src/lib.rs +++ b/litebox_broker_transport_windows_userland/src/lib.rs @@ -8,7 +8,6 @@ #![cfg(all(windows, target_arch = "x86_64"))] -mod call_lane_pool; pub mod control_ring; pub mod existing_runner; mod host; diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 3223b4b39c..dd8c3488a1 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -20,7 +20,7 @@ use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, encode_request, }; -use litebox_broker_transport::call_lane_pool::CallLanePoolError; +use litebox_broker_transport::call_lane_pool::{LaneClaim, LocalCallLanePool}; use litebox_broker_transport::channel::{ LocalCallChannel, LocalNotificationChannel, LocalSetupChannel, }; @@ -32,7 +32,6 @@ use litebox_broker_transport::control_ring::{ use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; -use crate::call_lane_pool::{CallLanePool, LaneClaim, call_lane_pool_error}; use crate::control_ring::PipeLiveness; use crate::named_pipe::{TRANSFER_FRAME_TAG, WindowsNamedPipeStream}; use crate::process_image::WindowsReceivedProcessImage; @@ -68,7 +67,7 @@ type LocalLane = LocalControlRingLane; /// call lanes and their demand, the pipe liveness used for teardown, and the /// wake handles of every ring wait. struct LocalRingAssociation { - lanes: CallLanePool, + lanes: LocalCallLanePool, lane_demand: LocalLaneDemand, /// Whether the call channel was dropped, which ends the association as a /// local close rather than a failure. @@ -238,7 +237,7 @@ fn activate_local( let stream = Arc::new(stream); let association = Arc::new(LocalRingAssociation { - lanes: CallLanePool::new(lanes), + lanes: LocalCallLanePool::new(lanes), lane_demand, closed: AtomicBool::new(false), liveness: PipeLiveness::new(Arc::clone(&stream), false)?, @@ -269,8 +268,7 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { .claim(|lanes| association.demand_lanes(lanes)) { Ok(lane) => lane, - Err(CallLanePoolError::AssociationFailed(error)) => return Err(copy_io_error(&error)), - Err(CallLanePoolError::Operation(error)) => { + Err(error) => { let result = Err(copy_io_error(&error)); let _ = association.fail(error); return result; @@ -303,7 +301,7 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { fn recv_notification(&mut self) -> IoResult> { loop { if let Some(error) = self.association.lanes.current_failure() { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } match self.consumer.try_read(decode_notification) { Ok(ControlRingReadStatus::Message(notification)) => { @@ -311,13 +309,13 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { .association .acknowledge_notification(&mut self.consumer) { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } return Ok(Some(notification)); } Ok(ControlRingReadStatus::Empty { wait_epoch }) => { if let Some(error) = self.association.lanes.current_failure() { - return self.association.end_notifications(&error); + return self.association.end_notifications(error); } if let Err(error) = self.consumer.wait_for_message(wait_epoch) { let result = Err(copy_io_error(&error)); @@ -345,11 +343,11 @@ impl LocalNotificationChannel for WindowsControlRingLocalNotificationChannel { impl LocalRingAssociation { /// Ends notification receipt once the association has ended with /// `failure`: cleanly if the local side closed it. - fn end_notifications(&self, failure: &Error) -> IoResult> { + fn end_notifications(&self, failure: Error) -> IoResult> { if self.closed.load(Ordering::Acquire) { Ok(None) } else { - Err(copy_io_error(failure)) + Err(failure) } } @@ -357,15 +355,12 @@ impl LocalRingAssociation { &self, consumer: &mut ControlRingConsumer, ) -> IoResult<()> { - let result = self - .lanes - .run_if_live(|| { - consumer - .publish_head() - .map_err(ring_error) - .and_then(|()| consumer.wake_producer()) - }) - .map_err(call_lane_pool_error); + let result = self.lanes.run_if_live(|| { + consumer + .publish_head() + .map_err(ring_error) + .and_then(|()| consumer.wake_producer()) + }); if let Err(error) = result { let result = Err(copy_io_error(&error)); let _ = self.fail(error); @@ -393,8 +388,7 @@ impl LocalRingAssociation { loop { let write_status = self .lanes - .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error)) - .map_err(call_lane_pool_error)?; + .run_if_live(|| producer.try_write(&request_frame).map_err(ring_error))?; match write_status { ControlRingWriteStatus::Written => break, ControlRingWriteStatus::Full { wait_epoch } => { @@ -410,7 +404,7 @@ impl LocalRingAssociation { Ok(ControlRingReadStatus::Message(response)) => break response, Ok(ControlRingReadStatus::Empty { wait_epoch }) => { if let Some(error) = self.lanes.current_failure() { - return Err(copy_io_error(&error)); + return Err(error); } consumer.wait_for_message(wait_epoch)?; } @@ -427,13 +421,12 @@ impl LocalRingAssociation { let consumer = &mut lane.response_consumer; consumer.publish_head().map_err(ring_error)?; consumer.wake_producer() - }) - .map_err(call_lane_pool_error)?; + })?; Ok(response) } fn fail(&self, error: Error) -> IoResult<()> { - self.lanes.record_failure(Arc::new(error)); + self.lanes.record_failure(error); self.wakes .iter() .map(ControlRingWakeHandle::interrupt_wait) From 62d7671f0e70b94285ea7c5cc1c44cf5462e6adc Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 22 Oct 2026 12:45:17 -0700 Subject: [PATCH 09/12] Trim the call-lane pool's module and feature docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/Cargo.toml | 2 +- litebox_broker_transport/src/call_lane_pool.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litebox_broker_transport/Cargo.toml b/litebox_broker_transport/Cargo.toml index 6d6e123580..63739c585e 100644 --- a/litebox_broker_transport/Cargo.toml +++ b/litebox_broker_transport/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [features] -# Local call-lane pool for hosted local peers. +# Local call-lane pool. std = [] [dependencies] diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs index 2548a823b1..b7ccdf9e1b 100644 --- a/litebox_broker_transport/src/call_lane_pool.rs +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Local-side admission of concurrent calls to call lanes, for hosted peers. +//! Local-side admission of concurrent calls to call lanes. use alloc::string::ToString; use alloc::vec::Vec; From d30d8bab499e92466363f4a76c1c13966e53dc56 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sun, 25 Oct 2026 07:17:48 -0700 Subject: [PATCH 10/12] Rename LocalLaneDemand::wake_broker to wake_host Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/src/control_ring.rs | 2 +- litebox_broker_transport_linux_userland/src/unix_socket/host.rs | 2 +- .../src/unix_socket/local.rs | 2 +- litebox_broker_transport_windows_userland/src/local.rs | 2 +- litebox_broker_transport_windows_userland/src/named_pipe.rs | 2 +- litebox_broker_transport_windows_userland/src/shared_memory.rs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litebox_broker_transport/src/control_ring.rs b/litebox_broker_transport/src/control_ring.rs index 31cd43db78..610d96fc23 100644 --- a/litebox_broker_transport/src/control_ring.rs +++ b/litebox_broker_transport/src/control_ring.rs @@ -728,7 +728,7 @@ impl LocalLaneDemand { /// /// The wake is unconditional: demand rises at most once per lane, so /// there is nothing to gain from an announcement word. - pub fn wake_broker(&self) -> Result<(), Memory::Error> { + pub fn wake_host(&self) -> Result<(), Memory::Error> { self.ring .memory .wake_one(self.ring.layout.demand_offset() + DEMAND_EPOCH_OFFSET) diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 7a1d797734..b083764300 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -729,7 +729,7 @@ mod tests { fn demand_lanes(local: &Local, lanes: usize) { let demand = local.lane_demand.as_ref().unwrap(); demand.request(lanes).unwrap(); - demand.wake_broker().unwrap(); + demand.wake_host().unwrap(); } fn notification_channel_pair() -> ( diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index b1f017e247..fa267d0f7f 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -441,7 +441,7 @@ impl LocalRingAssociation { /// Raises the lane demand to `lanes` and wakes the broker to serve them. fn demand_lanes(&self, lanes: usize) -> IoResult<()> { self.lane_demand.request(lanes).map_err(ring_error)?; - self.lane_demand.wake_broker() + self.lane_demand.wake_host() } /// Writes `request` to `lane` and reads the broker's response from it, diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index dd8c3488a1..a566282b4b 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -372,7 +372,7 @@ impl LocalRingAssociation { /// Raises the lane demand to `lanes` and wakes the broker to serve them. fn demand_lanes(&self, lanes: usize) -> IoResult<()> { self.lane_demand.request(lanes).map_err(ring_error)?; - self.lane_demand.wake_broker() + self.lane_demand.wake_host() } /// Writes `request` to `lane` and reads the broker's response from it, diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index fa4a1a722d..ec2f349b95 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -330,7 +330,7 @@ mod tests { fn demand_lanes(local: &Local, lanes: usize) { let demand = local.lane_demand.as_ref().unwrap(); demand.request(lanes).unwrap(); - demand.wake_broker().unwrap(); + demand.wake_host().unwrap(); } fn read_request(consumer: &mut Consumer) -> BrokerRequest { diff --git a/litebox_broker_transport_windows_userland/src/shared_memory.rs b/litebox_broker_transport_windows_userland/src/shared_memory.rs index 9ea2a01dea..4e196db31e 100644 --- a/litebox_broker_transport_windows_userland/src/shared_memory.rs +++ b/litebox_broker_transport_windows_userland/src/shared_memory.rs @@ -754,7 +754,7 @@ mod tests { ); let demander = std::thread::spawn(move || { local_demand.request(2).unwrap(); - local_demand.wake_broker().unwrap(); + local_demand.wake_host().unwrap(); }); loop { match broker_demand.try_next_lane().unwrap() { From 4261328416bd754395f746e65d1a7beb52dc3aa0 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sun, 25 Oct 2026 07:17:48 -0700 Subject: [PATCH 11/12] Document the call-lane pool as the association's failure record Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- litebox_broker_transport/src/call_lane_pool.rs | 11 +++++++++-- litebox_broker_transport_linux_userland/src/setup.rs | 4 ++-- .../src/unix_socket/local.rs | 2 ++ .../src/local.rs | 2 ++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/litebox_broker_transport/src/call_lane_pool.rs b/litebox_broker_transport/src/call_lane_pool.rs index b7ccdf9e1b..77554c1ffd 100644 --- a/litebox_broker_transport/src/call_lane_pool.rs +++ b/litebox_broker_transport/src/call_lane_pool.rs @@ -17,6 +17,12 @@ use std::sync::{Condvar, Mutex, MutexGuard}; /// concurrent calls; callers beyond the lane count wait for a lane to free up. /// Lane 0 is always demanded; before a call first claims any higher lane, the /// claim raises the demand so the broker starts serving that lane. +/// +/// The pool also records the failure that ended the association, since claims +/// must check it and wait under the same lock. Other users of the association, +/// such as its notification receiver, check that record through +/// [`current_failure`](Self::current_failure) and +/// [`run_if_live`](Self::run_if_live). pub struct LocalCallLanePool { state: Mutex>, lane_freed: Condvar, @@ -156,7 +162,8 @@ impl LocalCallLanePool { }) } - /// Records the first terminal failure and wakes every waiting claim. + /// Records the first failure, which ends the association, and wakes every + /// waiting claim. /// /// Returns whether this call recorded the first failure. pub fn record_failure(&self, error: Error) -> bool { @@ -169,7 +176,7 @@ impl LocalCallLanePool { true } - /// Returns a copy of the association's terminal failure, if one was + /// Returns a copy of the failure that ended the association, if one was /// recorded. pub fn current_failure(&self) -> Option { self.lock().failure.as_ref().map(copy_error) diff --git a/litebox_broker_transport_linux_userland/src/setup.rs b/litebox_broker_transport_linux_userland/src/setup.rs index b08588185e..93739788d4 100644 --- a/litebox_broker_transport_linux_userland/src/setup.rs +++ b/litebox_broker_transport_linux_userland/src/setup.rs @@ -101,8 +101,8 @@ pub(crate) fn wire_error(error: WireError) -> Error { ) } -/// Clones an error so one recorded terminal failure can be reported to every -/// waiter of an association. +/// Clones an error so the recorded failure that ended an association can be +/// reported to every waiter of the association. pub(crate) fn copy_io_error(error: &Error) -> Error { match error.raw_os_error() { Some(code) => Error::from_raw_os_error(code), diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index fa267d0f7f..b64e43c143 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -72,6 +72,8 @@ type LocalLane = LocalControlRingLane; /// call lanes and their demand, the setup socket used for liveness and /// teardown, and the wake handles of every ring wait. struct LocalRingAssociation { + /// The call lanes, which also record the failure that ended the + /// association. lanes: LocalCallLanePool, lane_demand: LocalLaneDemand, control_stream: UnixStream, diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index a566282b4b..f4b9820437 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -67,6 +67,8 @@ type LocalLane = LocalControlRingLane; /// call lanes and their demand, the pipe liveness used for teardown, and the /// wake handles of every ring wait. struct LocalRingAssociation { + /// The call lanes, which also record the failure that ended the + /// association. lanes: LocalCallLanePool, lane_demand: LocalLaneDemand, /// Whether the call channel was dropped, which ends the association as a From c368b7d670436201c196f849ebf87003ade46378 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sun, 25 Oct 2026 07:17:48 -0700 Subject: [PATCH 12/12] Drop Windows lane tests that repeat other coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e8b6cad-28c2-46d9-aef5-95cf4a401fb7 --- .../src/named_pipe.rs | 74 +------------------ 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index ec2f349b95..493c3fef98 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -162,9 +162,9 @@ mod tests { LocalNotificationChannel, LocalSetupChannel, }; use litebox_broker_transport::control_ring::{ - ControlRing, ControlRingConsumer, ControlRingDemandStatus, ControlRingLayout, - ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, - HostControlRingEndpoints, LocalControlRingEndpoints, + ControlRing, ControlRingConsumer, ControlRingLayout, ControlRingProducer, + ControlRingReadStatus, ControlRingWriteStatus, HostControlRingEndpoints, + LocalControlRingEndpoints, }; use litebox_broker_transport::shared_memory::SharedMemory as _; use windows_sys::Win32::System::Threading::GetCurrentProcessId; @@ -776,74 +776,6 @@ mod tests { assert!(next.join().unwrap().unwrap().is_none()); } - #[test] - fn concurrent_local_calls_use_separate_lanes() { - let (calls, _notifications, mut broker, _peer) = activate_local("local-concurrent"); - let calls = Arc::new(calls); - let callers = [3, 7].map(|id| { - let calls = Arc::clone(&calls); - thread::spawn(move || calls.call(request(id))) - }); - let demand = broker.lane_demand.as_mut().unwrap(); - assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); - loop { - match demand.try_next_lane().unwrap() { - ControlRingDemandStatus::Lane(lane) => { - assert_eq!(lane, 1); - break; - } - ControlRingDemandStatus::Satisfied { wait_epoch } => { - demand.wait_for_demand(wait_epoch).unwrap(); - } - } - } - let first = read_request(&mut broker.lanes[0].request_consumer); - let second = read_request(&mut broker.lanes[1].request_consumer); - - // The later lane answers first; neither call waits for the other. - write_payload( - &mut broker.lanes[1].response_producer, - &encode_response(response(second.request_id)), - ); - write_payload( - &mut broker.lanes[0].response_producer, - &encode_response(response(first.request_id)), - ); - for caller in callers { - assert!(caller.join().unwrap().is_ok()); - } - assert!(matches!( - demand.try_next_lane(), - Ok(ControlRingDemandStatus::Satisfied { .. }) - )); - } - - #[test] - fn sequential_local_calls_reuse_the_first_lane() { - let (calls, _notifications, mut broker, _peer) = activate_local("local-sequential"); - let calls = Arc::new(calls); - for id in 1..=3 { - let caller = { - let calls = Arc::clone(&calls); - thread::spawn(move || calls.call(request(id))) - }; - let lane = &mut broker.lanes[0]; - let request = read_request(&mut lane.request_consumer); - assert_eq!(request.request_id, RequestId(id)); - write_payload( - &mut lane.response_producer, - &encode_response(response(request.request_id)), - ); - assert_eq!(caller.join().unwrap().unwrap().request_id, RequestId(id)); - } - let demand = broker.lane_demand.as_mut().unwrap(); - assert_eq!(demand.try_next_lane(), Ok(ControlRingDemandStatus::Lane(0))); - assert!(matches!( - demand.try_next_lane(), - Ok(ControlRingDemandStatus::Satisfied { .. }) - )); - } - #[test] fn mismatched_and_malformed_responses_fail_closed() { for payload_kind in 0..3 {