DAOS-19898 dfuse: Add an option to allow idmapped mounts

docs/user/filesystem.md

@@ -233,6 +233,33 @@ Multi-user mode requires the fuse package to be reconfigured as it's disabled by
 setting `user_allow_other` needs to be set in `/etc/fuse.conf` or `/etc/fuse3.conf`, which will need
 to be done as root and takes effect for all users on that node.
 
+### Idmapped mounts
+
+Container runtimes use idmapped mounts to make a host directory available to a container that runs
+in a user namespace, such as a Kubernetes pod with `hostUsers: false`: the mount maps the user and
+group IDs of the container to those stored in the filesystem.  For a DFuse mount this requires Linux
+6.12 or later, a fuse library that supports it and the `--allow-idmap` option, otherwise the runtime
+fails with an error such as `failed to set MOUNT_ATTR_IDMAP on <path>: invalid argument`.  DFuse
+logs an error if the kernel or the fuse library does not support them, and at INFO level that they
+are allowed.  The processes of such a container do not run as the user running DFuse, so DFuse also
+has to run in multi-user mode.
+
+When idmapped mounts are allowed, the kernel only tells DFuse the user and group of the caller for
+requests that create files, directories or symbolic links.  The interception libraries ask DFuse for
+its pool and container handles, which DFuse only shares with processes running as its own user and
+group, so it reads the user and group of the process from `/proc` instead.  This only works for
+processes in the PID namespace of DFuse or in one nested in it, for example in any container if
+DFuse runs in the PID namespace of the host.  It does not work if DFuse is given a mounted FUSE
+device as `/dev/fd/N`, because the kernel then identifies processes in the PID namespace of the
+process that mounted it.  Other processes do not get the handles of DFuse: `libioil` then connects
+to the pool and container itself, and `libpil4dfs` leaves their I/O to DFuse.  Why a process did not
+get them is logged at DEBUG level.
+
+The option does not change who gets the handles of DFuse or who may access the files: the user and
+group that the kernel used to send are read from `/proc` instead, and the kernel keeps checking the
+permissions of the files.  DFuse should not run as the overflow user (`nobody`) with this option,
+because a process whose IDs are not mapped in the user namespace of DFuse appears as that user.
+
 ### Launching
 
 #### Via dfuse command
@@ -257,6 +284,7 @@ Additionally, there are several optional command-line options:
 | --max-background=<count\>       | Hard limit on outstanding background requests     |
 | --congestion-threshold=<count\> | Background requests at which the kernel throttles |
 | --multi-user                    | Run in multi user mode                            |
+| --allow-idmap                   | Allow idmapped mounts                             |
 | --read-only                     | Mount in read-only mode                           |
 
 The `--pool` and `--container` options can also be passed as the second and third positional

src/client/dfuse/dfuse.h

@@ -44,6 +44,8 @@ struct dfuse_info {
 	bool                 di_wb_cache;
 	bool                 di_read_only;
 	bool                 di_local_flock;
+	bool                 di_allow_idmap;
+	bool                 di_fd_mount;
 
 	/* Per process spinlock
 	 * This is used to lock readdir against closedir where they share a readdir handle,

src/client/dfuse/dfuse_fuseops.c

@@ -93,6 +93,20 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn)
 #ifdef FUSE_CAP_CACHE_SYMLINKS
 	conn->want |= FUSE_CAP_CACHE_SYMLINKS;
 #endif
+
+	if (dfuse_info->di_allow_idmap) {
+#ifdef FUSE_CAP_ALLOW_IDMAP
+		/* The kernel needs default_permissions for this, which dfuse always sets. The
+		 * flag is above bit 31, so it can only be set with fuse_set_feature_flag().
+		 */
+		if (fuse_set_feature_flag(conn, FUSE_CAP_ALLOW_IDMAP))
+			DFUSE_TRA_INFO(dfuse_info, "Idmapped mounts allowed");
+		else
+			DFUSE_TRA_ERROR(dfuse_info, "Kernel does not support idmapped mounts");
+#else
+		DFUSE_TRA_ERROR(dfuse_info, "Fuse library does not support idmapped mounts");
+#endif
+	}
 	dfuse_show_flags(dfuse_info, conn->capable, conn->want);
 
 	conn->time_gran            = 1;

src/client/dfuse/dfuse_main.c

@@ -326,6 +326,7 @@ show_help(char *name)
 	    "	-o options		mount style options string\n"
 	    "\n"
 	    "	   --multi-user		Run dfuse in multi user mode\n"
+	    "	   --allow-idmap	Allow idmapped mounts, e.g. for user namespaces\n"
 	    "	   --read-only		Mount dfuse read-only\n"
 	    "      --dump-handles=file  Serialize and dump the pool, container, and dfs handles"
 	    "				to file using the local2global DAOS functionality\n"
@@ -479,6 +480,7 @@ main(int argc, char **argv)
 
 	struct option      long_options[] = {{"mountpoint", required_argument, 0, 'm'},
 					     {"multi-user", no_argument, 0, 'M'},
+					     {"allow-idmap", no_argument, 0, 'I'},
 					     {"path", required_argument, 0, 'P'},
 					     {"pool", required_argument, 0, 'p'},
 					     {"container", required_argument, 0, 'c'},
@@ -571,6 +573,9 @@ main(int argc, char **argv)
 		case 'M':
 			dfuse_info->di_multi_user = true;
 			break;
+		case 'I':
+			dfuse_info->di_allow_idmap = true;
+			break;
 		case 'P':
 			path = optarg;
 			break;
@@ -653,6 +658,9 @@ main(int argc, char **argv)
 		show_help(argv[0]);
 		D_GOTO(out_debug, rc = -DER_INVAL);
 	}
+	/* With /dev/fd/N, libfuse uses a /dev/fuse descriptor that something else has mounted */
+	dfuse_info->di_fd_mount = check_fd_mountpoint(dfuse_info->di_mountpoint) >= 0;
+
 	if (dfuse_info->di_dump_handles && dfuse_info->di_read_handles) {
 		printf("Cannot dump and read handles file at the same time\n");
 		show_help(argv[0]);

src/client/dfuse/ops/ioctl.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -377,6 +378,74 @@ handle_cont_stat_query(fuse_req_t req, struct dfuse_obj_hdl *oh)
 	DFUSE_REPLY_IOCTL_SIZE(oh, req, &stat, sizeof(stat));
 }
 
+/* Check that the thread making a request runs as uid/gid.
+ *
+ * The kernel sends the caller's fsuid/fsgid with each request, except when idmapped mounts are
+ * allowed (FUSE_CAP_ALLOW_IDMAP): then it only does so for requests that create inodes and sends
+ * FUSE_INVALID_UIDGID ((uint32_t)-1) otherwise.  In that case read them from /proc, as libfuse's
+ * fuse_req_getgroups() does for supplementary groups.  The caller is blocked in ioctl() until
+ * dfuse replies, so its thread id cannot be reused in the meantime.  The pid is relative to the
+ * pid namespace of the process that mounted the filesystem: it is 0 when the caller is not in that
+ * namespace or one nested in it, and it can only be looked up in a /proc of that namespace.  This
+ * is the namespace of dfuse, unless dfuse was given a mounted /dev/fuse as /dev/fd/N by a process
+ * that may be in another one.  The caller cannot be identified otherwise.  Why a caller is refused
+ * is logged at DEBUG level.
+ */
+static bool
+caller_matches(struct dfuse_info *dfuse_info, const struct fuse_ctx *fc, uid_t uid, gid_t gid)
+{
+	char         path[64];
+	char         line[256];
+	FILE        *fp;
+	ssize_t      len;
+	unsigned int id[4];
+	bool         uid_ok = false;
+	bool         gid_ok = false;
+
+	if (fc->uid != (uid_t)-1 && fc->gid != (gid_t)-1)
+		return fc->uid == uid && fc->gid == gid;
+
+	if (dfuse_info->di_fd_mount) {
+		DFUSE_TRA_DEBUG(dfuse_info, "Callers of a /dev/fd/N mount cannot be identified");
+		return false;
+	}
+	if (fc->pid == 0) {
+		DFUSE_TRA_DEBUG(dfuse_info, "Caller is outside the pid namespace of dfuse");
+		return false;
+	}
+
+	len = readlink("/proc/self", path, sizeof(path) - 1);
+	if (len <= 0) {
+		DFUSE_TRA_DEBUG(dfuse_info, "Cannot read /proc/self: %d", errno);
+		return false;
+	}
+	path[len] = '\0';
+	if (atoi(path) != getpid()) {
+		DFUSE_TRA_DEBUG(dfuse_info, "/proc belongs to another pid namespace");
+		return false;
+	}
+
+	snprintf(path, sizeof(path), "/proc/%d/task/%d/status", fc->pid, fc->pid);
+	fp = fopen(path, "r");
+	if (fp == NULL) {
+		DFUSE_TRA_DEBUG(dfuse_info, "Cannot read %s: %d", path, errno);
+		return false;
+	}
+
+	/* Real, effective, saved and filesystem ids: the kernel would send the filesystem ones */
+	while (fgets(line, sizeof(line), fp) != NULL) {
+		if (sscanf(line, "Uid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4)
+			uid_ok = (id[3] == uid);
+		else if (sscanf(line, "Gid: %u %u %u %u", &id[0], &id[1], &id[2], &id[3]) == 4)
+			gid_ok = (id[3] == gid);
+	}
+	fclose(fp);
+
+	if (!uid_ok || !gid_ok)
+		DFUSE_TRA_DEBUG(dfuse_info, "pid %d does not run as %u:%u", fc->pid, uid, gid);
+	return uid_ok && gid_ok;
+}
+
 #ifdef FUSE_IOCTL_USE_INT
 void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg,
 		    struct fuse_file_info *fi, unsigned int flags,
@@ -467,7 +536,7 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg,
 	uid = getuid();
 	gid = getgid();
 
-	if (fc->uid != uid || fc->gid != gid)
+	if (!caller_matches(fuse_req_userdata(req), fc, uid, gid))
 		D_GOTO(out_err, rc = EPERM);
 
 	DFUSE_TRA_DEBUG(oh, "trusted pid %d", fc->pid);

src/tests/nlt/dfuse.py

@@ -32,7 +32,7 @@ class DFuse():
     # pylint: disable-next=too-many-arguments
     def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_path=None,
                  caching=True, wbcache=True, multi_user=False, ro=False, dump_h=False,
-                 read_h=False, file_h=None, extra_args=None):
+                 read_h=False, file_h=None, extra_args=None, allow_idmap=False):
         if mount_path:
             self.dir = mount_path
         else:
@@ -50,6 +50,7 @@ def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_p
             self.pool = container.pool.id()
         self.conf = conf
         self.multi_user = multi_user
+        self.allow_idmap = allow_idmap
         self.cores = 0
         self._daos = daos
         self.caching = caching
@@ -125,6 +126,9 @@ def start(self, v_hint=None, use_oopt=False):
         if self.multi_user:
             cmd.append('--multi-user')
 
+        if self.allow_idmap:
+            cmd.append('--allow-idmap')
+
         if not self.cores:
             # Use a lower default thread-count for NLT due to running tests in parallel.
             cmd.extend(['--thread-count', '4'])
@@ -238,7 +242,8 @@ def stop(self, ignore_einval=False):
             fatal_errors = True
             run_leak_test = False
         self._sp = None
-        log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval)
+        log_test(self.conf, self.log_file, show_memleaks=run_leak_test, ignore_einval=ignore_einval,
+                 ignore_no_idmap=self.allow_idmap)
 
         # Finally, modify the valgrind xml file to remove the
         # prefix to the src dir.
@@ -252,7 +257,7 @@ def wait_for_exit(self):
         ret = self._sp.wait()
         print(f'rc from dfuse {ret}')
         self._sp = None
-        log_test(self.conf, self.log_file)
+        log_test(self.conf, self.log_file, ignore_no_idmap=self.allow_idmap)
 
         # Finally, modify the valgrind xml file to remove the
         # prefix to the src dir.
@@ -417,11 +422,13 @@ class needs_dfuse_with_opt():
     wrapping_lock = threading.Lock()
 
     # pylint: disable=too-few-public-methods
-    def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False):
+    def __init__(self, caching_variants=None, wbcache=True, dfuse_inval=True, ro=False,
+                 allow_idmap=False):
         self.caching_variants = caching_variants if caching_variants else [False, True]
         self.wbcache = wbcache
         self.dfuse_inval = dfuse_inval
         self.ro = ro
+        self.allow_idmap = allow_idmap
 
     def __call__(self, method):
         """Wrapper function"""
@@ -458,11 +465,17 @@ def _helper(obj):
             if self.ro:
                 args["ro"] = True
 
+            if self.allow_idmap:
+                args["allow_idmap"] = True
+
             obj.dfuse = DFuse(obj.server,
                               obj.conf,
                               caching=caching,
                               wbcache=self.wbcache,
                               **args)
+            if self.allow_idmap:
+                # The test reads the log of dfuse while it runs.
+                obj.dfuse.log_flush = True
             obj.dfuse.start(v_hint=method.__name__)
             try:
                 rc = method(obj)

src/tests/nlt/logging_utils.py

@@ -163,6 +163,7 @@ def log_test(conf,
              leak_wf=None,
              ignore_einval=False,
              ignore_busy=False,
+             ignore_no_idmap=False,
              check_read=False,
              check_write=False,
              check_fstat=False,
@@ -208,6 +209,10 @@ def sizeof_fmt(num, suffix='B'):
     if ignore_busy:
         lto.skip_suffixes.append(" DER_BUSY(-1012): 'Device or resource busy'")
 
+    # dfuse --allow-idmap logs an error when the kernel or the fuse library does not support it
+    if ignore_no_idmap:
+        lto.skip_suffixes.append(' does not support idmapped mounts')
+
     def _issue_total():
         """Findings recorded so far, used to tell whether this log produced any."""
         total = len(conf.wf.issues) if conf.wf else 0

src/tests/nlt/posix_tests.py

@@ -1095,6 +1095,37 @@ def test_il(self):
                            'bs=128k'],
                           check_fstat=False)
 
+    @needs_dfuse_with_opt(caching_variants=[False], allow_idmap=True)
+    def test_il_allow_idmap(self):
+        """Check that the interception libraries get the handles of dfuse with --allow-idmap
+
+        When idmapped mounts are allowed, Linux 6.12 and later no longer send the uid and gid of
+        the caller with ioctl requests, so dfuse has to identify it before sharing its handles.
+        libpil4dfs only intercepts calls on a dfuse mount with these handles, whereas libioil
+        would open its own.  Without a kernel and a fuse library that support idmapped mounts
+        the ids are sent as before, and this is a plain interception test.
+        """
+        with open(self.dfuse.log_file, 'r') as log:
+            if 'Idmapped mounts allowed' not in log.read():
+                print('Idmapped mounts are not allowed by the kernel or the fuse library, '
+                      'dfuse identifies callers as before')
+
+        create_and_read_via_il(self.dfuse, self.dfuse.dir)
+
+        # dfuse logs at DEBUG level why it refused a caller. There must be no such line: the
+        # caller runs as the user of dfuse and in its pid namespace.
+        refusals = ('does not run as', 'outside the pid namespace',
+                    'belongs to another pid namespace', 'cannot be identified', 'Cannot read /proc')
+        with open(self.dfuse.log_file, 'r') as log:
+            for line in log:
+                if any(reason in line for reason in refusals):
+                    raise NLTestFail(f'dfuse did not share its handles: {line.strip()}')
+
+        fname = join(self.dfuse.dir, 'file')
+        with open(fname, 'w') as fd:
+            fd.write('Hello')
+        self.server.run_daos_client_cmd_pil4dfs(['cat', fname])
+
     @needs_dfuse
     def test_xattr(self):
         """Perform basic tests with extended attributes"""

utils/cq/words.dict

@@ -235,6 +235,8 @@ hugepage
 hugepages
 hwloc
 ib
+idmap
+idmapped
 idx
 iface
 indata