diff --git a/docs/user/filesystem.md b/docs/user/filesystem.md index 02632870d32..5262a4c177e 100644 --- a/docs/user/filesystem.md +++ b/docs/user/filesystem.md @@ -233,6 +233,33 @@ Multi-user mode requires the fuse package to be reconfigured as it's disabled by setting `user_allow_other` needs to be set in `/etc/fuse.conf` or `/etc/fuse3.conf`, which will need to be done as root and takes effect for all users on that node. +### Idmapped mounts + +Container runtimes use idmapped mounts to make a host directory available to a container that runs +in a user namespace, such as a Kubernetes pod with `hostUsers: false`: the mount maps the user and +group IDs of the container to those stored in the filesystem. For a DFuse mount this requires Linux +6.12 or later, a fuse library that supports it and the `--allow-idmap` option, otherwise the runtime +fails with an error such as `failed to set MOUNT_ATTR_IDMAP on : invalid argument`. DFuse +logs an error if the kernel or the fuse library does not support them, and at INFO level that they +are allowed. The processes of such a container do not run as the user running DFuse, so DFuse also +has to run in multi-user mode. + +When idmapped mounts are allowed, the kernel only tells DFuse the user and group of the caller for +requests that create files, directories or symbolic links. The interception libraries ask DFuse for +its pool and container handles, which DFuse only shares with processes running as its own user and +group, so it reads the user and group of the process from `/proc` instead. This only works for +processes in the PID namespace of DFuse or in one nested in it, for example in any container if +DFuse runs in the PID namespace of the host. It does not work if DFuse is given a mounted FUSE +device as `/dev/fd/N`, because the kernel then identifies processes in the PID namespace of the +process that mounted it. Other processes do not get the handles of DFuse: `libioil` then connects +to the pool and container itself, and `libpil4dfs` leaves their I/O to DFuse. Why a process did not +get them is logged at DEBUG level. + +The option does not change who gets the handles of DFuse or who may access the files: the user and +group that the kernel used to send are read from `/proc` instead, and the kernel keeps checking the +permissions of the files. DFuse should not run as the overflow user (`nobody`) with this option, +because a process whose IDs are not mapped in the user namespace of DFuse appears as that user. + ### Launching #### Via dfuse command @@ -257,6 +284,7 @@ Additionally, there are several optional command-line options: | --max-background= | Hard limit on outstanding background requests | | --congestion-threshold= | Background requests at which the kernel throttles | | --multi-user | Run in multi user mode | +| --allow-idmap | Allow idmapped mounts | | --read-only | Mount in read-only mode | The `--pool` and `--container` options can also be passed as the second and third positional diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index 9b9ef3e7cbc..16aa212777d 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -44,6 +44,8 @@ struct dfuse_info { bool di_wb_cache; bool di_read_only; bool di_local_flock; + bool di_allow_idmap; + bool di_fd_mount; /* Per process spinlock * This is used to lock readdir against closedir where they share a readdir handle, diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c index 3aadcc0afd9..bfcac9f8318 100644 --- a/src/client/dfuse/dfuse_fuseops.c +++ b/src/client/dfuse/dfuse_fuseops.c @@ -93,6 +93,20 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn) #ifdef FUSE_CAP_CACHE_SYMLINKS conn->want |= FUSE_CAP_CACHE_SYMLINKS; #endif + + if (dfuse_info->di_allow_idmap) { +#ifdef FUSE_CAP_ALLOW_IDMAP + /* The kernel needs default_permissions for this, which dfuse always sets. The + * flag is above bit 31, so it can only be set with fuse_set_feature_flag(). + */ + if (fuse_set_feature_flag(conn, FUSE_CAP_ALLOW_IDMAP)) + DFUSE_TRA_INFO(dfuse_info, "Idmapped mounts allowed"); + else + DFUSE_TRA_ERROR(dfuse_info, "Kernel does not support idmapped mounts"); +#else + DFUSE_TRA_ERROR(dfuse_info, "Fuse library does not support idmapped mounts"); +#endif + } dfuse_show_flags(dfuse_info, conn->capable, conn->want); conn->time_gran = 1; diff --git a/src/client/dfuse/dfuse_main.c b/src/client/dfuse/dfuse_main.c index 97941e82da8..dea5949a1c2 100644 --- a/src/client/dfuse/dfuse_main.c +++ b/src/client/dfuse/dfuse_main.c @@ -326,6 +326,7 @@ show_help(char *name) " -o options mount style options string\n" "\n" " --multi-user Run dfuse in multi user mode\n" + " --allow-idmap Allow idmapped mounts, e.g. for user namespaces\n" " --read-only Mount dfuse read-only\n" " --dump-handles=file Serialize and dump the pool, container, and dfs handles" " to file using the local2global DAOS functionality\n" @@ -479,6 +480,7 @@ main(int argc, char **argv) struct option long_options[] = {{"mountpoint", required_argument, 0, 'm'}, {"multi-user", no_argument, 0, 'M'}, + {"allow-idmap", no_argument, 0, 'I'}, {"path", required_argument, 0, 'P'}, {"pool", required_argument, 0, 'p'}, {"container", required_argument, 0, 'c'}, @@ -571,6 +573,9 @@ main(int argc, char **argv) case 'M': dfuse_info->di_multi_user = true; break; + case 'I': + dfuse_info->di_allow_idmap = true; + break; case 'P': path = optarg; break; @@ -653,6 +658,9 @@ main(int argc, char **argv) show_help(argv[0]); D_GOTO(out_debug, rc = -DER_INVAL); } + /* With /dev/fd/N, libfuse uses a /dev/fuse descriptor that something else has mounted */ + dfuse_info->di_fd_mount = check_fd_mountpoint(dfuse_info->di_mountpoint) >= 0; + if (dfuse_info->di_dump_handles && dfuse_info->di_read_handles) { printf("Cannot dump and read handles file at the same time\n"); show_help(argv[0]); diff --git a/src/client/dfuse/ops/ioctl.c b/src/client/dfuse/ops/ioctl.c index 8d073b859ff..89ad1cfba9c 100644 --- a/src/client/dfuse/ops/ioctl.c +++ b/src/client/dfuse/ops/ioctl.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -377,6 +378,74 @@ handle_cont_stat_query(fuse_req_t req, struct dfuse_obj_hdl *oh) DFUSE_REPLY_IOCTL_SIZE(oh, req, &stat, sizeof(stat)); } +/* Check that the thread making a request runs as uid/gid. + * + * The kernel sends the caller's fsuid/fsgid with each request, except when idmapped mounts are + * allowed (FUSE_CAP_ALLOW_IDMAP): then it only does so for requests that create inodes and sends + * FUSE_INVALID_UIDGID ((uint32_t)-1) otherwise. In that case read them from /proc, as libfuse's + * fuse_req_getgroups() does for supplementary groups. The caller is blocked in ioctl() until + * dfuse replies, so its thread id cannot be reused in the meantime. The pid is relative to the + * pid namespace of the process that mounted the filesystem: it is 0 when the caller is not in that + * namespace or one nested in it, and it can only be looked up in a /proc of that namespace. This + * is the namespace of dfuse, unless dfuse was given a mounted /dev/fuse as /dev/fd/N by a process + * that may be in another one. The caller cannot be identified otherwise. Why a caller is refused + * is logged at DEBUG level. + */ +static bool +caller_matches(struct dfuse_info *dfuse_info, const struct fuse_ctx *fc, uid_t uid, gid_t gid) +{ + char path[64]; + char line[256]; + FILE *fp; + ssize_t len; + unsigned int id[4]; + bool uid_ok = false; + bool gid_ok = false; + + if (fc->uid != (uid_t)-1 && fc->gid != (gid_t)-1) + return fc->uid == uid && fc->gid == gid; + + if (dfuse_info->di_fd_mount) { + DFUSE_TRA_DEBUG(dfuse_info, "Callers of a /dev/fd/N mount cannot be identified"); + return false; + } + if (fc->pid == 0) { + DFUSE_TRA_DEBUG(dfuse_info, "Caller is outside the pid namespace of dfuse"); + return false; + } + + len = readlink("/proc/self", path, sizeof(path) - 1); + if (len <= 0) { + DFUSE_TRA_DEBUG(dfuse_info, "Cannot read /proc/self: %d", errno); + return false; + } + path[len] = '\0'; + if (atoi(path) != getpid()) { + DFUSE_TRA_DEBUG(dfuse_info, "/proc belongs to another pid namespace"); + return false; + } + + snprintf(path, sizeof(path), "/proc/%d/task/%d/status", fc->pid, fc->pid); + fp = fopen(path, "r"); + if (fp == NULL) { + DFUSE_TRA_DEBUG(dfuse_info, "Cannot read %s: %d", path, errno); + return false; + } + + /* Real, effective, saved and filesystem ids: the kernel would send the filesystem ones */ + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "Uid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4) + uid_ok = (id[3] == uid); + else if (sscanf(line, "Gid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4) + gid_ok = (id[3] == gid); + } + fclose(fp); + + if (!uid_ok || !gid_ok) + DFUSE_TRA_DEBUG(dfuse_info, "pid %d does not run as %u:%u", fc->pid, uid, gid); + return uid_ok && gid_ok; +} + #ifdef FUSE_IOCTL_USE_INT void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, unsigned int flags, @@ -467,7 +536,7 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg, uid = getuid(); gid = getgid(); - if (fc->uid != uid || fc->gid != gid) + if (!caller_matches(fuse_req_userdata(req), fc, uid, gid)) D_GOTO(out_err, rc = EPERM); DFUSE_TRA_DEBUG(oh, "trusted pid %d", fc->pid); diff --git a/src/tests/nlt/dfuse.py b/src/tests/nlt/dfuse.py index ee095b25e14..6f1840e526a 100644 --- a/src/tests/nlt/dfuse.py +++ b/src/tests/nlt/dfuse.py @@ -32,7 +32,7 @@ class DFuse(): # pylint: disable-next=too-many-arguments def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_path=None, caching=True, wbcache=True, multi_user=False, ro=False, dump_h=False, - read_h=False, file_h=None, extra_args=None): + read_h=False, file_h=None, extra_args=None, allow_idmap=False): if mount_path: self.dir = mount_path else: @@ -50,6 +50,7 @@ def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_p self.pool = container.pool.id() self.conf = conf self.multi_user = multi_user + self.allow_idmap = allow_idmap self.cores = 0 self._daos = daos self.caching = caching @@ -125,6 +126,9 @@ def start(self, v_hint=None, use_oopt=False): if self.multi_user: cmd.append('--multi-user') + if self.allow_idmap: + cmd.append('--allow-idmap') + if not self.cores: # Use a lower default thread-count for NLT due to running tests in parallel. cmd.extend(['--thread-count', '4']) @@ -238,7 +242,8 @@ def stop(self, ignore_einval=False): fatal_errors = True run_leak_test = False self._sp = None - log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval) + log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval, + ignore_no_idmap=self.allow_idmap) # Finally, modify the valgrind xml file to remove the # prefix to the src dir. @@ -252,7 +257,7 @@ def wait_for_exit(self): ret = self._sp.wait() print(f'rc from dfuse {ret}') self._sp = None - log_test(self.conf, self.log_file) + log_test(self.conf, self.log_file, ignore_no_idmap=self.allow_idmap) # Finally, modify the valgrind xml file to remove the # prefix to the src dir. @@ -417,11 +422,13 @@ class needs_dfuse_with_opt(): wrapping_lock = threading.Lock() # pylint: disable=too-few-public-methods - def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False): + def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False, + allow_idmap=False): self.caching_variants = caching_variants if caching_variants else [False, True] self.wbcache = wbcache self.dfuse_inval = dfuse_inval self.ro = ro + self.allow_idmap = allow_idmap def __call__(self, method): """Wrapper function""" @@ -458,11 +465,17 @@ def _helper(obj): if self.ro: args["ro"] = True + if self.allow_idmap: + args["allow_idmap"] = True + obj.dfuse = DFuse(obj.server, obj.conf, caching=caching, wbcache=self.wbcache, **args) + if self.allow_idmap: + # The test reads the log of dfuse while it runs. + obj.dfuse.log_flush = True obj.dfuse.start(v_hint=method.__name__) try: rc = method(obj) diff --git a/src/tests/nlt/logging_utils.py b/src/tests/nlt/logging_utils.py index 2ec5f05613d..14a765284bd 100644 --- a/src/tests/nlt/logging_utils.py +++ b/src/tests/nlt/logging_utils.py @@ -163,6 +163,7 @@ def log_test(conf, leak_wf=None, ignore_einval=False, ignore_busy=False, + ignore_no_idmap=False, check_read=False, check_write=False, check_fstat=False, @@ -208,6 +209,10 @@ def sizeof_fmt(num, suffix='B'): if ignore_busy: lto.skip_suffixes.append(" DER_BUSY(-1012): 'Device or resource busy'") + # dfuse --allow-idmap logs an error when the kernel or the fuse library does not support it + if ignore_no_idmap: + lto.skip_suffixes.append(' does not support idmapped mounts') + def _issue_total(): """Findings recorded so far, used to tell whether this log produced any.""" total = len(conf.wf.issues) if conf.wf else 0 diff --git a/src/tests/nlt/posix_tests.py b/src/tests/nlt/posix_tests.py index 2a0762bbf54..d19674d7602 100644 --- a/src/tests/nlt/posix_tests.py +++ b/src/tests/nlt/posix_tests.py @@ -1095,6 +1095,37 @@ def test_il(self): 'bs=128k'], check_fstat=False) + @needs_dfuse_with_opt(caching_variants=[False], allow_idmap=True) + def test_il_allow_idmap(self): + """Check that the interception libraries get the handles of dfuse with --allow-idmap + + When idmapped mounts are allowed, Linux 6.12 and later no longer send the uid and gid of + the caller with ioctl requests, so dfuse has to identify it before sharing its handles. + libpil4dfs only intercepts calls on a dfuse mount with these handles, whereas libioil + would open its own. Without a kernel and a fuse library that support idmapped mounts + the ids are sent as before, and this is a plain interception test. + """ + with open(self.dfuse.log_file, 'r') as log: + if 'Idmapped mounts allowed' not in log.read(): + print('Idmapped mounts are not allowed by the kernel or the fuse library, ' + 'dfuse identifies callers as before') + + create_and_read_via_il(self.dfuse, self.dfuse.dir) + + # dfuse logs at DEBUG level why it refused a caller. There must be no such line: the + # caller runs as the user of dfuse and in its pid namespace. + refusals = ('does not run as', 'outside the pid namespace', + 'belongs to another pid namespace', 'cannot be identified', 'Cannot read /proc') + with open(self.dfuse.log_file, 'r') as log: + for line in log: + if any(reason in line for reason in refusals): + raise NLTestFail(f'dfuse did not share its handles: {line.strip()}') + + fname = join(self.dfuse.dir, 'file') + with open(fname, 'w') as fd: + fd.write('Hello') + self.server.run_daos_client_cmd_pil4dfs(['cat', fname]) + @needs_dfuse def test_xattr(self): """Perform basic tests with extended attributes""" diff --git a/utils/cq/words.dict b/utils/cq/words.dict index 643f134100e..c8b514f3b3d 100644 --- a/utils/cq/words.dict +++ b/utils/cq/words.dict @@ -235,6 +235,8 @@ hugepage hugepages hwloc ib +idmap +idmapped idx iface indata