From fc658215a048f4c07faf4a289c1961e79595358d Mon Sep 17 00:00:00 2001 From: Dmitry Polonsky Date: Sat, 26 Sep 2026 08:39:08 +0300 Subject: [PATCH] DAOS-19898 dfuse: Add an option to allow idmapped mounts Container runtimes give a container that runs in a user namespace, such as a Kubernetes pod with "hostUsers: false", access to a host directory through an idmapped mount of it. Since Linux 6.12 FUSE filesystems support idmapped mounts, but only if the server sets FUSE_ALLOW_IDMAP in its FUSE_INIT reply. dfuse does not, so such a pod cannot use a dfuse mount: runc fails with "failed to set MOUNT_ATTR_IDMAP on : invalid argument". Add --allow-idmap, which requests FUSE_CAP_ALLOW_IDMAP. The kernel only accepts it with default_permissions, which dfuse always uses. It is off by default because it changes what the kernel tells dfuse about the caller of a request: for requests that do not create an inode, the uid and gid in the header are FUSE_INVALID_UIDGID, on every mount of the filesystem and not only on idmapped ones. The one place where dfuse needs them for such a request is the check that it only shares its pool and container handles with interception libraries that run as the same user and group. When the kernel does not send them, read the fsuid and fsgid of the calling thread from /proc, as libfuse's fuse_req_getgroups() does for supplementary groups. The caller is blocked in ioctl() until dfuse replies, so its thread id cannot be reused in the meantime. The pid in the request is relative to the pid namespace of the process that mounted the filesystem. That is dfuse, unless dfuse was given the mounted /dev/fuse as /dev/fd/N, and then it cannot tell which namespace it is. So a caller outside of it (pid 0), any caller when dfuse sees the /proc of another pid namespace, and any caller of a /dev/fd/N mount cannot be identified and do not get the handles. As for callers running as another user, libpil4dfs then leaves their I/O to dfuse and libioil opens its own handles. Why a caller is refused is logged at DEBUG level. Without --allow-idmap nothing changes. FUSE_CAP_ALLOW_IDMAP is defined by fuse libraries that support it (libfuse commit 1f4c0608, not in a release yet). With another one, --allow-idmap has no effect and dfuse logs an error, as when the kernel does not support idmapped mounts. Add an NLT test that runs libioil and libpil4dfs against dfuse started with --allow-idmap. On Linux 6.12 and later with a fuse library that supports idmapped mounts, it fails if dfuse does not share its handles: dfuse logs that it refused the caller and libpil4dfs reports no intercepted calls. Otherwise it is a plain interception test, and the log check of NLT ignores that error of dfuse started with --allow-idmap. Document the option. Features: dfuse Signed-off-by: Dmitry Polonsky --- docs/user/filesystem.md | 28 +++++++++++++ src/client/dfuse/dfuse.h | 2 + src/client/dfuse/dfuse_fuseops.c | 14 +++++++ src/client/dfuse/dfuse_main.c | 8 ++++ src/client/dfuse/ops/ioctl.c | 71 +++++++++++++++++++++++++++++++- src/tests/nlt/dfuse.py | 21 ++++++++-- src/tests/nlt/logging_utils.py | 5 +++ src/tests/nlt/posix_tests.py | 31 ++++++++++++++ utils/cq/words.dict | 2 + 9 files changed, 177 insertions(+), 5 deletions(-) diff --git a/docs/user/filesystem.md b/docs/user/filesystem.md index 02632870d32..5262a4c177e 100644 --- a/docs/user/filesystem.md +++ b/docs/user/filesystem.md @@ -233,6 +233,33 @@ Multi-user mode requires the fuse package to be reconfigured as it's disabled by setting `user_allow_other` needs to be set in `/etc/fuse.conf` or `/etc/fuse3.conf`, which will need to be done as root and takes effect for all users on that node. +### Idmapped mounts + +Container runtimes use idmapped mounts to make a host directory available to a container that runs +in a user namespace, such as a Kubernetes pod with `hostUsers: false`: the mount maps the user and +group IDs of the container to those stored in the filesystem. For a DFuse mount this requires Linux +6.12 or later, a fuse library that supports it and the `--allow-idmap` option, otherwise the runtime +fails with an error such as `failed to set MOUNT_ATTR_IDMAP on : invalid argument`. DFuse +logs an error if the kernel or the fuse library does not support them, and at INFO level that they +are allowed. The processes of such a container do not run as the user running DFuse, so DFuse also +has to run in multi-user mode. + +When idmapped mounts are allowed, the kernel only tells DFuse the user and group of the caller for +requests that create files, directories or symbolic links. The interception libraries ask DFuse for +its pool and container handles, which DFuse only shares with processes running as its own user and +group, so it reads the user and group of the process from `/proc` instead. This only works for +processes in the PID namespace of DFuse or in one nested in it, for example in any container if +DFuse runs in the PID namespace of the host. It does not work if DFuse is given a mounted FUSE +device as `/dev/fd/N`, because the kernel then identifies processes in the PID namespace of the +process that mounted it. Other processes do not get the handles of DFuse: `libioil` then connects +to the pool and container itself, and `libpil4dfs` leaves their I/O to DFuse. Why a process did not +get them is logged at DEBUG level. + +The option does not change who gets the handles of DFuse or who may access the files: the user and +group that the kernel used to send are read from `/proc` instead, and the kernel keeps checking the +permissions of the files. DFuse should not run as the overflow user (`nobody`) with this option, +because a process whose IDs are not mapped in the user namespace of DFuse appears as that user. + ### Launching #### Via dfuse command @@ -257,6 +284,7 @@ Additionally, there are several optional command-line options: | --max-background= | Hard limit on outstanding background requests | | --congestion-threshold= | Background requests at which the kernel throttles | | --multi-user | Run in multi user mode | +| --allow-idmap | Allow idmapped mounts | | --read-only | Mount in read-only mode | The `--pool` and `--container` options can also be passed as the second and third positional diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index 9b9ef3e7cbc..16aa212777d 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -44,6 +44,8 @@ struct dfuse_info { bool di_wb_cache; bool di_read_only; bool di_local_flock; + bool di_allow_idmap; + bool di_fd_mount; /* Per process spinlock * This is used to lock readdir against closedir where they share a readdir handle, diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c index 3aadcc0afd9..bfcac9f8318 100644 --- a/src/client/dfuse/dfuse_fuseops.c +++ b/src/client/dfuse/dfuse_fuseops.c @@ -93,6 +93,20 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn) #ifdef FUSE_CAP_CACHE_SYMLINKS conn->want |= FUSE_CAP_CACHE_SYMLINKS; #endif + + if (dfuse_info->di_allow_idmap) { +#ifdef FUSE_CAP_ALLOW_IDMAP + /* The kernel needs default_permissions for this, which dfuse always sets. The + * flag is above bit 31, so it can only be set with fuse_set_feature_flag(). + */ + if (fuse_set_feature_flag(conn, FUSE_CAP_ALLOW_IDMAP)) + DFUSE_TRA_INFO(dfuse_info, "Idmapped mounts allowed"); + else + DFUSE_TRA_ERROR(dfuse_info, "Kernel does not support idmapped mounts"); +#else + DFUSE_TRA_ERROR(dfuse_info, "Fuse library does not support idmapped mounts"); +#endif + } dfuse_show_flags(dfuse_info, conn->capable, conn->want); conn->time_gran = 1; diff --git a/src/client/dfuse/dfuse_main.c b/src/client/dfuse/dfuse_main.c index 97941e82da8..dea5949a1c2 100644 --- a/src/client/dfuse/dfuse_main.c +++ b/src/client/dfuse/dfuse_main.c @@ -326,6 +326,7 @@ show_help(char *name) " -o options mount style options string\n" "\n" " --multi-user Run dfuse in multi user mode\n" + " --allow-idmap Allow idmapped mounts, e.g. for user namespaces\n" " --read-only Mount dfuse read-only\n" " --dump-handles=file Serialize and dump the pool, container, and dfs handles" " to file using the local2global DAOS functionality\n" @@ -479,6 +480,7 @@ main(int argc, char **argv) struct option long_options[] = {{"mountpoint", required_argument, 0, 'm'}, {"multi-user", no_argument, 0, 'M'}, + {"allow-idmap", no_argument, 0, 'I'}, {"path", required_argument, 0, 'P'}, {"pool", required_argument, 0, 'p'}, {"container", required_argument, 0, 'c'}, @@ -571,6 +573,9 @@ main(int argc, char **argv) case 'M': dfuse_info->di_multi_user = true; break; + case 'I': + dfuse_info->di_allow_idmap = true; + break; case 'P': path = optarg; break; @@ -653,6 +658,9 @@ main(int argc, char **argv) show_help(argv[0]); D_GOTO(out_debug, rc = -DER_INVAL); } + /* With /dev/fd/N, libfuse uses a /dev/fuse descriptor that something else has mounted */ + dfuse_info->di_fd_mount = check_fd_mountpoint(dfuse_info->di_mountpoint) >= 0; + if (dfuse_info->di_dump_handles && dfuse_info->di_read_handles) { printf("Cannot dump and read handles file at the same time\n"); show_help(argv[0]); diff --git a/src/client/dfuse/ops/ioctl.c b/src/client/dfuse/ops/ioctl.c index 8d073b859ff..89ad1cfba9c 100644 --- a/src/client/dfuse/ops/ioctl.c +++ b/src/client/dfuse/ops/ioctl.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -377,6 +378,74 @@ handle_cont_stat_query(fuse_req_t req, struct dfuse_obj_hdl *oh) DFUSE_REPLY_IOCTL_SIZE(oh, req, &stat, sizeof(stat)); } +/* Check that the thread making a request runs as uid/gid. + * + * The kernel sends the caller's fsuid/fsgid with each request, except when idmapped mounts are + * allowed (FUSE_CAP_ALLOW_IDMAP): then it only does so for requests that create inodes and sends + * FUSE_INVALID_UIDGID ((uint32_t)-1) otherwise. In that case read them from /proc, as libfuse's + * fuse_req_getgroups() does for supplementary groups. The caller is blocked in ioctl() until + * dfuse replies, so its thread id cannot be reused in the meantime. The pid is relative to the + * pid namespace of the process that mounted the filesystem: it is 0 when the caller is not in that + * namespace or one nested in it, and it can only be looked up in a /proc of that namespace. This + * is the namespace of dfuse, unless dfuse was given a mounted /dev/fuse as /dev/fd/N by a process + * that may be in another one. The caller cannot be identified otherwise. Why a caller is refused + * is logged at DEBUG level. + */ +static bool +caller_matches(struct dfuse_info *dfuse_info, const struct fuse_ctx *fc, uid_t uid, gid_t gid) +{ + char path[64]; + char line[256]; + FILE *fp; + ssize_t len; + unsigned int id[4]; + bool uid_ok = false; + bool gid_ok = false; + + if (fc->uid != (uid_t)-1 && fc->gid != (gid_t)-1) + return fc->uid == uid && fc->gid == gid; + + if (dfuse_info->di_fd_mount) { + DFUSE_TRA_DEBUG(dfuse_info, "Callers of a /dev/fd/N mount cannot be identified"); + return false; + } + if (fc->pid == 0) { + DFUSE_TRA_DEBUG(dfuse_info, "Caller is outside the pid namespace of dfuse"); + return false; + } + + len = readlink("/proc/self", path, sizeof(path) - 1); + if (len <= 0) { + DFUSE_TRA_DEBUG(dfuse_info, "Cannot read /proc/self: %d", errno); + return false; + } + path[len] = '\0'; + if (atoi(path) != getpid()) { + DFUSE_TRA_DEBUG(dfuse_info, "/proc belongs to another pid namespace"); + return false; + } + + snprintf(path, sizeof(path), "/proc/%d/task/%d/status", fc->pid, fc->pid); + fp = fopen(path, "r"); + if (fp == NULL) { + DFUSE_TRA_DEBUG(dfuse_info, "Cannot read %s: %d", path, errno); + return false; + } + + /* Real, effective, saved and filesystem ids: the kernel would send the filesystem ones */ + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "Uid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4) + uid_ok = (id[3] == uid); + else if (sscanf(line, "Gid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4) + gid_ok = (id[3] == gid); + } + fclose(fp); + + if (!uid_ok || !gid_ok) + DFUSE_TRA_DEBUG(dfuse_info, "pid %d does not run as %u:%u", fc->pid, uid, gid); + return uid_ok && gid_ok; +} + #ifdef FUSE_IOCTL_USE_INT void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, unsigned int flags, @@ -467,7 +536,7 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg, uid = getuid(); gid = getgid(); - if (fc->uid != uid || fc->gid != gid) + if (!caller_matches(fuse_req_userdata(req), fc, uid, gid)) D_GOTO(out_err, rc = EPERM); DFUSE_TRA_DEBUG(oh, "trusted pid %d", fc->pid); diff --git a/src/tests/nlt/dfuse.py b/src/tests/nlt/dfuse.py index ee095b25e14..6f1840e526a 100644 --- a/src/tests/nlt/dfuse.py +++ b/src/tests/nlt/dfuse.py @@ -32,7 +32,7 @@ class DFuse(): # pylint: disable-next=too-many-arguments def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_path=None, caching=True, wbcache=True, multi_user=False, ro=False, dump_h=False, - read_h=False, file_h=None, extra_args=None): + read_h=False, file_h=None, extra_args=None, allow_idmap=False): if mount_path: self.dir = mount_path else: @@ -50,6 +50,7 @@ def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_p self.pool = container.pool.id() self.conf = conf self.multi_user = multi_user + self.allow_idmap = allow_idmap self.cores = 0 self._daos = daos self.caching = caching @@ -125,6 +126,9 @@ def start(self, v_hint=None, use_oopt=False): if self.multi_user: cmd.append('--multi-user') + if self.allow_idmap: + cmd.append('--allow-idmap') + if not self.cores: # Use a lower default thread-count for NLT due to running tests in parallel. cmd.extend(['--thread-count', '4']) @@ -238,7 +242,8 @@ def stop(self, ignore_einval=False): fatal_errors = True run_leak_test = False self._sp = None - log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval) + log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval, + ignore_no_idmap=self.allow_idmap) # Finally, modify the valgrind xml file to remove the # prefix to the src dir. @@ -252,7 +257,7 @@ def wait_for_exit(self): ret = self._sp.wait() print(f'rc from dfuse {ret}') self._sp = None - log_test(self.conf, self.log_file) + log_test(self.conf, self.log_file, ignore_no_idmap=self.allow_idmap) # Finally, modify the valgrind xml file to remove the # prefix to the src dir. @@ -417,11 +422,13 @@ class needs_dfuse_with_opt(): wrapping_lock = threading.Lock() # pylint: disable=too-few-public-methods - def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False): + def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False, + allow_idmap=False): self.caching_variants = caching_variants if caching_variants else [False, True] self.wbcache = wbcache self.dfuse_inval = dfuse_inval self.ro = ro + self.allow_idmap = allow_idmap def __call__(self, method): """Wrapper function""" @@ -458,11 +465,17 @@ def _helper(obj): if self.ro: args["ro"] = True + if self.allow_idmap: + args["allow_idmap"] = True + obj.dfuse = DFuse(obj.server, obj.conf, caching=caching, wbcache=self.wbcache, **args) + if self.allow_idmap: + # The test reads the log of dfuse while it runs. + obj.dfuse.log_flush = True obj.dfuse.start(v_hint=method.__name__) try: rc = method(obj) diff --git a/src/tests/nlt/logging_utils.py b/src/tests/nlt/logging_utils.py index 2ec5f05613d..14a765284bd 100644 --- a/src/tests/nlt/logging_utils.py +++ b/src/tests/nlt/logging_utils.py @@ -163,6 +163,7 @@ def log_test(conf, leak_wf=None, ignore_einval=False, ignore_busy=False, + ignore_no_idmap=False, check_read=False, check_write=False, check_fstat=False, @@ -208,6 +209,10 @@ def sizeof_fmt(num, suffix='B'): if ignore_busy: lto.skip_suffixes.append(" DER_BUSY(-1012): 'Device or resource busy'") + # dfuse --allow-idmap logs an error when the kernel or the fuse library does not support it + if ignore_no_idmap: + lto.skip_suffixes.append(' does not support idmapped mounts') + def _issue_total(): """Findings recorded so far, used to tell whether this log produced any.""" total = len(conf.wf.issues) if conf.wf else 0 diff --git a/src/tests/nlt/posix_tests.py b/src/tests/nlt/posix_tests.py index 2a0762bbf54..d19674d7602 100644 --- a/src/tests/nlt/posix_tests.py +++ b/src/tests/nlt/posix_tests.py @@ -1095,6 +1095,37 @@ def test_il(self): 'bs=128k'], check_fstat=False) + @needs_dfuse_with_opt(caching_variants=[False], allow_idmap=True) + def test_il_allow_idmap(self): + """Check that the interception libraries get the handles of dfuse with --allow-idmap + + When idmapped mounts are allowed, Linux 6.12 and later no longer send the uid and gid of + the caller with ioctl requests, so dfuse has to identify it before sharing its handles. + libpil4dfs only intercepts calls on a dfuse mount with these handles, whereas libioil + would open its own. Without a kernel and a fuse library that support idmapped mounts + the ids are sent as before, and this is a plain interception test. + """ + with open(self.dfuse.log_file, 'r') as log: + if 'Idmapped mounts allowed' not in log.read(): + print('Idmapped mounts are not allowed by the kernel or the fuse library, ' + 'dfuse identifies callers as before') + + create_and_read_via_il(self.dfuse, self.dfuse.dir) + + # dfuse logs at DEBUG level why it refused a caller. There must be no such line: the + # caller runs as the user of dfuse and in its pid namespace. + refusals = ('does not run as', 'outside the pid namespace', + 'belongs to another pid namespace', 'cannot be identified', 'Cannot read /proc') + with open(self.dfuse.log_file, 'r') as log: + for line in log: + if any(reason in line for reason in refusals): + raise NLTestFail(f'dfuse did not share its handles: {line.strip()}') + + fname = join(self.dfuse.dir, 'file') + with open(fname, 'w') as fd: + fd.write('Hello') + self.server.run_daos_client_cmd_pil4dfs(['cat', fname]) + @needs_dfuse def test_xattr(self): """Perform basic tests with extended attributes""" diff --git a/utils/cq/words.dict b/utils/cq/words.dict index 643f134100e..c8b514f3b3d 100644 --- a/utils/cq/words.dict +++ b/utils/cq/words.dict @@ -235,6 +235,8 @@ hugepage hugepages hwloc ib +idmap +idmapped idx iface indata