feat(services): add 3FS USRBIO support

.env.example

@@ -2,6 +2,9 @@
 # fs
 OPENDAL_FS_ROOT=/path/to/dir
 OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir
+# hf3fs
+OPENDAL_HF3FS_ROOT=/path/to/3fs/testdir
+OPENDAL_HF3FS_LIBRARY_PATH=/path/to/libhf3fs_api_shared.so
 # cos
 OPENDAL_COS_BUCKET=opendal-testing-1318209832
 OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com

.github/scripts/test_behavior/plan.py

@@ -285,6 +285,9 @@ def generate_language_binding_cases(
     # Bindings may be treated as parallel requests, so we need to disable it for all languages.
     cases = [v for v in cases if v["service"] != "aliyun_drive"]
 
+    # 3FS currently has a Rust-only fixture that requires Ubuntu 22.04's RXE module.
+    cases = [v for v in cases if v["service"] != "hf3fs"]
+
     # Remove invalid cases for java.
     if language == "java":
         cases = [v for v in cases if v["service"] not in [
@@ -387,7 +390,12 @@ def plan(changed_files: list[str]) -> dict[str, Any]:
 
     if len(core_cases) > 0:
         jobs["components"]["core"] = True
-        jobs["core"].append({"os": "ubuntu-latest", "cases": core_cases})
+        linux_cases = [v for v in core_cases if v["service"] != "hf3fs"]
+        hf3fs_cases = [v for v in core_cases if v["service"] == "hf3fs"]
+        if linux_cases:
+            jobs["core"].append({"os": "ubuntu-latest", "cases": linux_cases})
+        if hf3fs_cases:
+            jobs["core"].append({"os": "ubuntu-22.04", "cases": hf3fs_cases})
 
         # fs is the only services need to run upon windows, let's hard code it here.
         if "fs" in [v["service"] for v in core_cases]:

.github/scripts/test_behavior/test_plan.py

@@ -54,6 +54,19 @@ def test_core_services_hdfs_native_mapping(self):
         self.assertTrue("hdfs_native" in cases)
         self.assertFalse("fs" in cases)
 
+    def test_hf3fs_uses_ubuntu_22(self):
+        result = plan(["core/services/hf3fs/src/lib.rs"])
+        self.assertEqual(result["core"], [{
+            "os": "ubuntu-22.04",
+            "cases": [{
+                "service": "hf3fs",
+                "feature": "services-hf3fs",
+                "setups": ["single_node"],
+            }],
+        }])
+        for language in ["java", "python", "ruby", "nodejs", "go", "c", "cpp", "dotnet"]:
+            self.assertEqual(result[f"binding_{language}"], [])
+
     def test_group_core_cases_by_service(self):
         cases = [
             {

.github/services/hf3fs/README.md

@@ -0,0 +1,91 @@
+# 3FS behavior tests
+
+The `single_node` setup starts a real, single-replica 3FS cluster on a hosted
+Ubuntu 22.04 runner. It uses SoftRoCE, FoundationDB, and ordinary data directories;
+it requires neither a physical RDMA adapter nor a dedicated disk. Ubuntu 24.04's
+current Azure kernel disables RXE, so the behavior planner selects Ubuntu 22.04.
+
+CI caches the pinned 3FS build, runs the focused USRBIO tests and the standard
+behavior suite, and stops the cluster and removes the RXE device afterward.
+The first build is expensive; subsequent jobs restore the compiled backend.
+The job skips the common setup's standalone RocksDB build because the Rust
+service does not depend on it. 3FS builds its own bundled RocksDB.
+
+To test an existing Linux deployment, use a real 3FS FUSE mount and the matching
+USRBIO client library.
+
+Create a dedicated test directory on the mount. Set `OPENDAL_HF3FS_ROOT` to its
+absolute path and `OPENDAL_HF3FS_LIBRARY_PATH` to `libhf3fs_api_shared.so`.
+Set `OPENDAL_TEST=hf3fs` and `OPENDAL_DISABLE_RANDOM_ROOT=true`, since the service
+requires its root to exist before construction.
+
+From `core/`, run:
+
+```shell
+cargo test --test behavior --features tests,services-hf3fs
+cargo test -p opendal-service-hf3fs --test real_service -- --ignored
+cargo run --release -p opendal-service-hf3fs --example bench_read
+```
+
+The benchmark compares 1 MiB streaming reads at concurrency 1 and 16 against
+`Fs` on the same file and mount. It warms each path and reports four measured
+passes. Results depend on client configuration, caching, and cluster load.
+
+For the read/write matrix, build the example from `core/`. The default profile
+requires direct I/O on the FUSE path. `Fs` has no direct-I/O builder option, so a
+Linux preload helper adds `O_DIRECT` only to files below the dedicated benchmark
+root. It also verifies each intercepted descriptor's flags. This helper is for
+benchmarking on a real 3FS mount, not for application use.
+
+```shell
+cargo build --release -p opendal-service-hf3fs --example bench_matrix
+cc -shared -fPIC -O2 -Wall -Wextra -Werror \
+  ../.github/services/hf3fs/bench_direct.c -o /tmp/opendal-bench-direct.so -ldl -pthread
+OPENDAL_BENCH_DIRECT_ROOT="$OPENDAL_HF3FS_ROOT" \
+  LD_PRELOAD=/tmp/opendal-bench-direct.so \
+  target/release/examples/bench_matrix > matrix.csv
+```
+
+The benchmark refuses to label a run `direct` unless the helper covers its root.
+For a new client deployment, verify the data-file opens with `strace -f -e openat`
+and check page residency with `mincore` before interpreting results. The helper
+intercepts libc `open` and `open64`; it does not cover arbitrary direct syscalls
+or relative `openat` calls. It prints the verified read/write open counts on exit.
+Direct I/O bypasses the client's kernel page cache, not storage-server caches.
+
+Both services use one Operator, the same 1 GiB input file, application block
+sizes of 4 KiB, 64 KiB, 1 MiB, and 4 MiB, and 1, 4, 16, or 64 concurrent workers.
+Each worker issues one application operation at a time, with read and write
+concurrency set to one. Both paths split application operations into service
+chunks of `min(BS, 1 MiB)`; `Hf3fs` uses that native block size and queue depth 64
+per direction. Kernel and server request splitting can still differ. Internal
+threads, copies, and ring scheduling remain properties of each implementation;
+the benchmark does not add Operators to compensate for an implementation limit.
+Each worker performs one untimed input read before the start barrier.
+
+Workloads include sequential range reads, random range reads, streaming writes,
+single-block object overwrites, and mixed random reads/overwrites with 70% or
+50% reads by operation count. Mixed workloads use separate input and output
+files; they do not test random writes within an existing file. Overwrites include
+open, truncate, write, close, and fsync. Streaming writers close and synchronize
+every 64 MiB and at the end. Both services buffer writes, so streaming write
+latency measures API admission; throughput includes all writes and final
+synchronization. These latency columns are not device completion latencies.
+
+For application-level cache comparisons, run without the preload and explicitly
+set `BENCH_CACHE=buffered` or `BENCH_CACHE=hot`. The latter warms the input through
+an existing `Fs` reader after all workers open the file. USRBIO bypasses that
+file cache in every profile. Keep buffered/hot results separate from the direct
+comparison; they do not isolate data-path performance. CSV rows include the
+cache profile and service chunk size to preserve these distinctions.
+
+Each case runs for 2 seconds and repeats 4 times. Rounds must be a positive even
+number so each service runs first equally often. Set `BENCH_SECONDS`,
+`BENCH_ROUNDS`, `BENCH_BS` (bytes), `BENCH_CONCURRENCY`, or `BENCH_WORKLOADS` to
+override the defaults; the last three accept comma-separated lists. Use longer
+cases and additional rounds to confirm differences found in the matrix.
+
+The benchmark uses constant payloads and checks returned lengths and sampled
+data. CPU usage covers only the benchmark process, excluding the FUSE process
+and servers. Short cases and shared-cluster load limit the precision of tail
+latencies. Temporary files are removed after a successful run.

.github/services/hf3fs/bench_direct.c

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdatomic.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static pthread_once_t initialized = PTHREAD_ONCE_INIT;
+static int (*next_open)(const char *, int, ...);
+static int (*next_open64)(const char *, int, ...);
+static char *root;
+static size_t root_len;
+static _Atomic unsigned long reads, writes;
+
+static void initialize(void) {
+    next_open = dlsym(RTLD_NEXT, "open");
+    next_open64 = dlsym(RTLD_NEXT, "open64");
+    const char *value = getenv("OPENDAL_BENCH_DIRECT_ROOT");
+    if (!next_open || !next_open64 || !value || !(root = realpath(value, NULL))) {
+        fputs("direct I/O helper requires OPENDAL_BENCH_DIRECT_ROOT\n", stderr);
+        _exit(1);
+    }
+    root_len = strlen(root);
+}
+
+/* Queried by the benchmark so a missing preload cannot silently use page cache. */
+int opendal_bench_direct_for(const char *path) {
+    pthread_once(&initialized, initialize);
+    return strncmp(path, root, root_len) == 0 && path[root_len] == '/';
+}
+
+static int open_file(const char *path, int flags, mode_t mode, int large) {
+    pthread_once(&initialized, initialize);
+    int direct = opendal_bench_direct_for(path) && !(flags & (O_DIRECTORY | O_PATH));
+    int fd = (large ? next_open64 : next_open)(path, flags | (direct ? O_DIRECT : 0), mode);
+    if (fd >= 0 && direct) {
+        int actual = fcntl(fd, F_GETFL);
+        if (actual < 0 || !(actual & O_DIRECT)) {
+            close(fd);
+            errno = EIO;
+            return -1;
+        }
+        if ((flags & O_ACCMODE) == O_RDONLY) {
+            atomic_fetch_add(&reads, 1);
+        } else {
+            atomic_fetch_add(&writes, 1);
+        }
+    }
+    return fd;
+}
+
+int open(const char *path, int flags, ...) {
+    mode_t mode = 0;
+    if ((flags & O_CREAT) || (flags & O_TMPFILE) == O_TMPFILE) {
+        va_list args;
+        va_start(args, flags);
+        mode = va_arg(args, mode_t);
+        va_end(args);
+    }
+    return open_file(path, flags, mode, 0);
+}
+
+int open64(const char *path, int flags, ...) {
+    mode_t mode = 0;
+    if ((flags & O_CREAT) || (flags & O_TMPFILE) == O_TMPFILE) {
+        va_list args;
+        va_start(args, flags);
+        mode = va_arg(args, mode_t);
+        va_end(args);
+    }
+    return open_file(path, flags, mode, 1);
+}
+
+__attribute__((destructor)) static void report(void) {
+    fprintf(stderr, "direct_io: verified_read_opens=%lu verified_write_opens=%lu\n",
+            atomic_load(&reads), atomic_load(&writes));
+}

.github/services/hf3fs/single_node/action.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: single_node
+description: Test 3FS with one storage node, one replica, and SoftRoCE
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/cache/restore@v5
+      id: cache
+      with:
+        path: ${{ runner.temp }}/3fs
+        key: hf3fs-ubuntu22-${{ hashFiles('.github/services/hf3fs/single_node/setup.sh') }}
+    - name: Prepare 3FS
+      shell: bash
+      env:
+        CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
+      run: bash .github/services/hf3fs/single_node/setup.sh
+    - uses: actions/cache/save@v5
+      if: steps.cache.outputs.cache-hit != 'true'
+      with:
+        path: ${{ runner.temp }}/3fs
+        key: hf3fs-ubuntu22-${{ hashFiles('.github/services/hf3fs/single_node/setup.sh') }}
+    - name: Start 3FS
+      shell: bash
+      run: |
+        source_dir="$RUNNER_TEMP/3fs"
+        cluster="$RUNNER_TEMP/hf3fs-cluster"
+        sed -i '/\[ib_devices\]/a device_filter = ["opendal_rxe"]' \
+          "$source_dir"/tests/fuse/config/*launcher.toml "$source_dir"/tests/fuse/config/admin_cli.toml
+        sed -i "s/start_level = 'NONE'/start_level = 'WARN'/" "$source_dir"/tests/fuse/config/*.toml
+        sudo systemd-run --unit=opendal-hf3fs --uid="$(id -u)" \
+          --working-directory="$RUNNER_TEMP" \
+          --property=LimitMEMLOCK=infinity --property=TimeoutStopSec=10s \
+          bash "$source_dir/tests/fuse/run.sh" "$source_dir/build/bin" "$cluster"
+        for _ in $(seq 1 120); do
+          if mountpoint -q "$cluster/mnt" && [[ -d "$cluster/mnt/test" ]]; then
+            break
+          fi
+          if ! systemctl is-active --quiet opendal-hf3fs; then
+            sudo journalctl -u opendal-hf3fs --no-pager
+            exit 1
+          fi
+          sleep 1
+        done
+        mountpoint "$cluster/mnt"
+        test -d "$cluster/mnt/test"
+        echo "OPENDAL_HF3FS_ROOT=$cluster/mnt/test" >> "$GITHUB_ENV"
+        echo "OPENDAL_HF3FS_LIBRARY_PATH=$source_dir/build/src/lib/api/libhf3fs_api_shared.so" >> "$GITHUB_ENV"
+        echo "OPENDAL_DISABLE_RANDOM_ROOT=true" >> "$GITHUB_ENV"
+        echo "RUST_TEST_THREADS=2" >> "$GITHUB_ENV"
+    - name: Test USRBIO
+      shell: bash
+      working-directory: core
+      run: cargo test --locked -p opendal-service-hf3fs --test real_service -- --ignored

.github/services/hf3fs/single_node/setup.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+sudo apt-get update -qq
+sudo apt-get install -y "linux-modules-extra-$(uname -r)" rdma-core ibverbs-utils \
+  cmake ninja-build libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev \
+  libdwarf-dev libunwind-dev libaio-dev libgflags-dev libgoogle-glog-dev \
+  libgtest-dev libgmock-dev clang-14 clang-format-14 lld-14 libgoogle-perftools-dev \
+  libssl-dev gcc-12 g++-12 libboost-all-dev meson libibverbs-dev libnuma-dev \
+  libevent-dev libfmt-dev autoconf automake libtool gettext fuse3
+sudo modprobe rdma_rxe
+device=$(ip -4 route show default | awk 'NR == 1 {print $5}')
+sudo rdma link add opendal_rxe type rxe netdev "$device"
+ibv_devinfo -d opendal_rxe
+
+curl -fL --retry 3 -o "$RUNNER_TEMP/fdb.deb" \
+  https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb
+sudo dpkg -i "$RUNNER_TEMP/fdb.deb"
+curl -fL --retry 3 -o "$RUNNER_TEMP/fdbserver" \
+  https://github.com/apple/foundationdb/releases/download/7.1.61/fdbserver.x86_64
+sudo install -m 755 "$RUNNER_TEMP/fdbserver" /usr/local/bin/fdbserver
+
+git clone -q --depth 1 --branch fuse-3.16.2 https://github.com/libfuse/libfuse.git "$RUNNER_TEMP/libfuse"
+meson setup "$RUNNER_TEMP/libfuse/build" "$RUNNER_TEMP/libfuse" -Dexamples=false -Dtests=false
+ninja -C "$RUNNER_TEMP/libfuse/build"
+sudo ninja -C "$RUNNER_TEMP/libfuse/build" install
+sudo ldconfig
+
+if [[ "$CACHE_HIT" == true ]]; then
+  exit 0
+fi
+
+git clone -q https://github.com/deepseek-ai/3FS.git "$RUNNER_TEMP/3fs"
+cd "$RUNNER_TEMP/3fs"
+git checkout 22fca04564c7cc230fd8b9523b8b92864e1dad47
+git submodule update --init --recursive --depth 1 --jobs 4
+./patches/apply.sh
+cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang-14 -DCMAKE_CXX_COMPILER=clang++-14 \
+  -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS_RELEASE='-O1 -DNDEBUG' \
+  -DCMAKE_CXX_FLAGS_RELEASE='-O1 -DNDEBUG' -DSHUFFLE_METHOD=g++11
+cmake --build build -j 3 --target mgmtd_main meta_main storage_main hf3fs_fuse_main admin_cli hf3fs_api_shared