Skip to content

Fix races in Linux file descriptor duplication #1181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Weiteng Chen (CvvT) merged 4 commits into from
Sep 2, 2026
Merged

Fix races in Linux file descriptor duplication #1181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
896c2ef
Fix races in Linux fd duplication
CvvT Aug 17, 2026
cecc5cf
fix rebase
CvvT Aug 17, 2026
a14c722
revert max_fd
CvvT Sep 1, 2026
f70aaee
change max_fd to be exclusive
CvvT Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion litebox_shim_linux/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ impl<Platform: ShimPlatform> LinuxShim<Platform> {
} = task;

let files = syscalls::file::FilesState::new(fs);
files.set_max_fd(syscalls::process::RLIMIT_NOFILE_CUR - 1);
files.set_max_fd(syscalls::process::RLIMIT_NOFILE_CUR);
let files = Arc::new(files);
let credentials = Arc::new(syscalls::process::Credentials {
uid,
Expand Down
152 changes: 100 additions & 52 deletions litebox_shim_linux/src/syscalls/file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ pub(crate) struct FilesState<Platform: ShimPlatform> {
pub(crate) fs: alloc::sync::Arc<LinuxFS<Platform>>,
pub(crate) raw_descriptor_store:
litebox::sync::RwLock<Platform, litebox::fd::RawDescriptorStorage>,
/// Exclusive upper bound for raw file descriptor values.
max_fd: AtomicUsize,
}

Expand Down Expand Up @@ -124,14 +125,44 @@ impl<Platform: ShimPlatform> FilesState<Platform> {
rds: &mut litebox::fd::RawDescriptorStorage,
typed_fd: TypedFd<Subsystem>,
) -> Result<usize, TypedFd<Subsystem>> {
// XXX(jb): should we try to somehow enforce that it is set at the smallest
// available/unassigned FD number?
let raw_fd = rds.fd_into_raw_integer(typed_fd);
let max_fd = self.max_fd.load(Ordering::Relaxed);
if raw_fd > max_fd {
let orig = rds.fd_consume_raw_integer::<Subsystem>(raw_fd).unwrap();
return Err(alloc::sync::Arc::into_inner(orig).unwrap());
self.insert_raw_fd_at_or_above_locked(rds, typed_fd, 0, max_fd)
}

fn insert_raw_fd_at_or_above<Subsystem: FdEnabledSubsystem>(
&self,
typed_fd: TypedFd<Subsystem>,
min_fd: usize,
max_fd: usize,
) -> Result<usize, TypedFd<Subsystem>> {
let mut rds = self.raw_descriptor_store.write();
self.insert_raw_fd_at_or_above_locked(&mut rds, typed_fd, min_fd, max_fd)
}

fn insert_raw_fd_at_or_above_locked<Subsystem: FdEnabledSubsystem>(
&self,
rds: &mut litebox::fd::RawDescriptorStorage,
typed_fd: TypedFd<Subsystem>,
min_fd: usize,
max_fd: usize,
) -> Result<usize, TypedFd<Subsystem>> {
if min_fd >= max_fd {
return Err(typed_fd);
}

// XXX: Can clean+speed this up by exposing a new method at RawDescriptorStorage
let mut raw_fd = min_fd;
for occupied_raw_fd in rds.iter_alive().skip_while(|&fd| fd < min_fd) {
if occupied_raw_fd != raw_fd {
break;
}
raw_fd += 1;
}
Comment thread
jaybosamiya-ms marked this conversation as resolved.
if raw_fd >= max_fd {
return Err(typed_fd);
}
let success = rds.fd_into_specific_raw_integer(typed_fd, raw_fd);
assert!(success);
Ok(raw_fd)
}
}
Expand Down Expand Up @@ -779,6 +810,15 @@ impl<Platform: ShimPlatform> Task<Platform> {
self.do_close_and_replace::<LinuxFS<Platform>>(raw_fd, None)
}

fn remove_and_drop_descriptor<S: FdEnabledSubsystem>(&self, fd: &TypedFd<S>) {
let entry = {
let mut dt = self.global.litebox.descriptor_table_mut();
dt.remove(fd)
};
// do not hold any locks while dropping the entry
drop(entry);
}

/// Close the file at `raw_fd` and optionally place a new file in the same slot.
///
/// This function ensure `close` and `insert` are done atomically.
Expand Down Expand Up @@ -856,30 +896,15 @@ impl<Platform: ShimPlatform> Task<Platform> {
ConsumedFd::Network(fd) => self.global.close_socket(&self.wait_cx(), fd),
ConsumedFd::Pipes(fd) => self.global.close_linux_pipe(&fd),
ConsumedFd::Eventfd(fd) => {
let entry = {
let mut dt = self.global.litebox.descriptor_table_mut();
dt.remove(&fd)
};
// do not hold any locks while dropping the entry
drop(entry);
self.remove_and_drop_descriptor(&fd);
Ok(())
}
ConsumedFd::Epoll(fd) => {
let entry = {
let mut dt = self.global.litebox.descriptor_table_mut();
dt.remove(&fd)
};
// do not hold any locks while dropping the entry
drop(entry);
self.remove_and_drop_descriptor(&fd);
Ok(())
}
ConsumedFd::Unix(fd) => {
let entry = {
let mut dt = self.global.litebox.descriptor_table_mut();
dt.remove(&fd)
};
// do not hold any locks while dropping the entry
drop(entry);
self.remove_and_drop_descriptor(&fd);
Ok(())
}
}
Expand Down Expand Up @@ -2434,17 +2459,17 @@ impl<Platform: ShimPlatform> Task<Platform> {
fd: &TypedFd<S>,
close_on_exec: bool,
target: DupFdRequest,
close_typed_fd: impl FnOnce(TypedFd<S>),
) -> Result<usize, DupFdError> {
let max_fd = task
.process()
.limits
.get_rlimit_cur(litebox_common_linux::RlimitResource::NOFILE);
let max_fd = files.max_fd.load(Ordering::Relaxed);
match target {
DupFdRequest::Exact(target) if target >= max_fd => {
DupFdRequest::Exact(target) | DupFdRequest::LowestAtOrAbove(target)
if target >= max_fd =>
{
return Err(DupFdError::TargetFdExceedsLimit);
}
DupFdRequest::LowestAtOrAbove(min_fd) if min_fd >= max_fd => {
return Err(DupFdError::TargetFdExceedsLimit);
DupFdRequest::LowestAvailable if max_fd == 0 => {
return Err(DupFdError::TooManyFiles);
}
_ => {}
}
Expand All @@ -2463,27 +2488,24 @@ impl<Platform: ShimPlatform> Task<Platform> {
target
}
DupFdRequest::LowestAvailable => {
let rds = &mut *files.raw_descriptor_store.write();
rds.fd_into_raw_integer(fd)
match files.insert_raw_fd_at_or_above(fd, 0, max_fd) {
Ok(fd) => fd,
Err(fd) => {
close_typed_fd(fd);
return Err(DupFdError::TooManyFiles);
}
}
}
DupFdRequest::LowestAtOrAbove(min_fd) => {
let rds = &mut *files.raw_descriptor_store.write();
let mut raw_fd = min_fd;
for occupied_raw_fd in rds.iter_alive().skip_while(|&fd| fd < min_fd) {
if occupied_raw_fd != raw_fd {
break;
match files.insert_raw_fd_at_or_above(fd, min_fd, max_fd) {
Ok(fd) => fd,
Err(fd) => {
close_typed_fd(fd);
return Err(DupFdError::TooManyFiles);
}
raw_fd += 1;
}
let success = rds.fd_into_specific_raw_integer(fd, raw_fd);
assert!(success);
raw_fd
}
};
if new_fd >= max_fd {
let _ = task.do_close(new_fd);
return Err(DupFdError::TooManyFiles);
}
Ok(new_fd)
}

Expand All @@ -2492,12 +2514,38 @@ impl<Platform: ShimPlatform> Task<Platform> {
files
.run_on_raw_fd(
file,
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| dup(self, &files, fd, close_on_exec, target),
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
let _ = files.fs.close(&fd);
})
},
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
let _ = self
.global
.close_socket(&self.wait_cx(), alloc::sync::Arc::new(fd));
})
},
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
let _ = self.global.close_linux_pipe(&fd);
})
},
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
self.remove_and_drop_descriptor(&fd);
})
},
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
self.remove_and_drop_descriptor(&fd);
})
},
|fd| {
dup(self, &files, fd, close_on_exec, target, |fd| {
self.remove_and_drop_descriptor(&fd);
})
},
)
.map_err(|_| DupFdError::BadFd)?
}
Expand Down
3 changes: 1 addition & 2 deletions litebox_shim_linux/src/syscalls/process.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -820,8 +820,7 @@ impl<Platform: ShimPlatform> Task<Platform> {
}
match resource {
litebox_common_linux::RlimitResource::NOFILE => {
let new_max_fd = new_limit.rlim_cur.saturating_sub(1);
self.files.borrow().set_max_fd(new_max_fd);
self.files.borrow().set_max_fd(new_limit.rlim_cur);
}
_ => unimplemented!("Unsupported resource for set_rlimit: {:?}", resource),
}
Expand Down
2 changes: 1 addition & 1 deletion litebox_shim_linux/src/syscalls/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ fn test_fcntl() {
#[test]
fn test_pipe2_race_with_concurrent_close() {
let task = init_platform(None);
task.files.borrow().set_max_fd(3);
task.files.borrow().set_max_fd(4);

let stop = alloc::sync::Arc::new(core::sync::atomic::AtomicBool::new(false));
let stop_closer = stop.clone();
Expand Down