diff --git a/.container_build_image b/.container_build_image new file mode 100644 index 0000000000000..787e80d3642d7 --- /dev/null +++ b/.container_build_image @@ -0,0 +1 @@ +rocky-10-kernel-builder diff --git a/.github/workflows/build-check_aarch64-rt.yml b/.github/workflows/build-check_aarch64-rt.yml index 9f667f3216e31..97913e53a54a3 100644 --- a/.github/workflows/build-check_aarch64-rt.yml +++ b/.github/workflows/build-check_aarch64-rt.yml @@ -26,7 +26,7 @@ jobs: dnf groupinstall 'Development Tools' -y dnf install openssl -y - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: "${{ github.event.pull_request.head.sha }}" fetch-depth: 0 @@ -35,4 +35,4 @@ jobs: git config --global --add safe.directory /__w/kernel-src-tree/kernel-src-tree cp configs/kernel-aarch64-rt-rhel.config .config make olddefconfig - make -j8 + make -j$(nproc) diff --git a/.github/workflows/build-check_aarch64.yml b/.github/workflows/build-check_aarch64.yml index 62234d160d291..dd892cf789a1f 100644 --- a/.github/workflows/build-check_aarch64.yml +++ b/.github/workflows/build-check_aarch64.yml @@ -26,7 +26,7 @@ jobs: dnf groupinstall 'Development Tools' -y dnf install openssl -y - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: "${{ github.event.pull_request.head.sha }}" fetch-depth: 0 @@ -35,4 +35,4 @@ jobs: git config --global --add safe.directory /__w/kernel-src-tree/kernel-src-tree cp configs/kernel-aarch64-rhel.config .config make olddefconfig - make -j8 + make -j$(nproc) diff --git a/.github/workflows/build-check_x86_64-rt.yml b/.github/workflows/build-check_x86_64-rt.yml index 8dadc0f02e404..38559e1b1fb88 100644 --- a/.github/workflows/build-check_x86_64-rt.yml +++ b/.github/workflows/build-check_x86_64-rt.yml @@ -26,7 +26,7 @@ jobs: dnf groupinstall 'Development Tools' -y dnf install openssl -y - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: "${{ github.event.pull_request.head.sha }}" fetch-depth: 0 @@ -35,4 +35,4 @@ jobs: git config --global --add safe.directory /__w/kernel-src-tree/kernel-src-tree cp configs/kernel-x86_64-rt-rhel.config .config make olddefconfig - make -j8 + make -j$(nproc) diff --git a/.github/workflows/build-check_x86_64.yml b/.github/workflows/build-check_x86_64.yml index f5f8da22fcf43..2c5de23997536 100644 --- a/.github/workflows/build-check_x86_64.yml +++ b/.github/workflows/build-check_x86_64.yml @@ -26,7 +26,7 @@ jobs: dnf groupinstall 'Development Tools' -y dnf install openssl -y - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: "${{ github.event.pull_request.head.sha }}" fetch-depth: 0 @@ -35,4 +35,4 @@ jobs: git config --global --add safe.directory /__w/kernel-src-tree/kernel-src-tree cp configs/kernel-x86_64-rhel.config .config make olddefconfig - make -j8 + make -j$(nproc) diff --git a/.github/workflows/kernel-build-and-test-multiarch-trigger.yml b/.github/workflows/kernel-build-and-test-multiarch-trigger.yml new file mode 100644 index 0000000000000..ae7c4550640f4 --- /dev/null +++ b/.github/workflows/kernel-build-and-test-multiarch-trigger.yml @@ -0,0 +1,11 @@ +name: Trigger Automated kernel build and test (multi-arch) + +on: + push: + branches: + - '*_rlc-10/**' + +jobs: + kernelCI: + uses: ctrliq/kernel-src-tree/.github/workflows/kernel-build-and-test-multiarch-trigger.yml@main + secrets: inherit diff --git a/.github/workflows/validate-kernel-commits.yml b/.github/workflows/validate-kernel-commits.yml new file mode 100644 index 0000000000000..c74434336e251 --- /dev/null +++ b/.github/workflows/validate-kernel-commits.yml @@ -0,0 +1,10 @@ +name: Validate Kernel Commits + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + check: + uses: ctrliq/kernel-src-tree/.github/workflows/validate-kernel-commits.yml@main + secrets: inherit diff --git a/MAINTAINERS b/MAINTAINERS index 9523da3c767cb..f297f7cc2936b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9443,6 +9443,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git F: Documentation/filesystems/fuse.rst F: fs/fuse/ F: include/uapi/linux/fuse.h +F: tools/testing/selftests/filesystems/fuse/ FUTEX SUBSYSTEM M: Thomas Gleixner diff --git a/configs/kernel-aarch64-64k-debug-rhel.config b/configs/kernel-aarch64-64k-debug-rhel.config index 8c954151d3f05..d54503fa6b579 100644 --- a/configs/kernel-aarch64-64k-debug-rhel.config +++ b/configs/kernel-aarch64-64k-debug-rhel.config @@ -1284,7 +1284,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1294,8 +1295,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2300,6 +2301,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-64k-rhel.config b/configs/kernel-aarch64-64k-rhel.config index d7184d179d8a2..63c69dce59d74 100644 --- a/configs/kernel-aarch64-64k-rhel.config +++ b/configs/kernel-aarch64-64k-rhel.config @@ -1284,7 +1284,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1294,8 +1295,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2284,6 +2285,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-debug-rhel.config b/configs/kernel-aarch64-debug-rhel.config index c728fb0f15ace..2273d0fac4881 100644 --- a/configs/kernel-aarch64-debug-rhel.config +++ b/configs/kernel-aarch64-debug-rhel.config @@ -1282,7 +1282,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1292,8 +1293,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2297,6 +2298,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-rhel.config b/configs/kernel-aarch64-rhel.config index 7a7f317b61f37..d4e44633b11ff 100644 --- a/configs/kernel-aarch64-rhel.config +++ b/configs/kernel-aarch64-rhel.config @@ -1282,7 +1282,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1292,8 +1293,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2281,6 +2282,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-rt-64k-debug-rhel.config b/configs/kernel-aarch64-rt-64k-debug-rhel.config index fe4a1928f05bd..7dfb453abff76 100644 --- a/configs/kernel-aarch64-rt-64k-debug-rhel.config +++ b/configs/kernel-aarch64-rt-64k-debug-rhel.config @@ -1285,7 +1285,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1295,8 +1296,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2339,6 +2340,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-rt-64k-rhel.config b/configs/kernel-aarch64-rt-64k-rhel.config index c15f4ee5a01c6..00d3045264c40 100644 --- a/configs/kernel-aarch64-rt-64k-rhel.config +++ b/configs/kernel-aarch64-rt-64k-rhel.config @@ -1285,7 +1285,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1295,8 +1296,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2323,6 +2324,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-rt-debug-rhel.config b/configs/kernel-aarch64-rt-debug-rhel.config index 2bb35d529aafa..d8f7f2beb2c98 100644 --- a/configs/kernel-aarch64-rt-debug-rhel.config +++ b/configs/kernel-aarch64-rt-debug-rhel.config @@ -1283,7 +1283,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1293,8 +1294,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2336,6 +2337,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-aarch64-rt-rhel.config b/configs/kernel-aarch64-rt-rhel.config index 39be57f9d2ea8..3e9f9e4d7d34d 100644 --- a/configs/kernel-aarch64-rt-rhel.config +++ b/configs/kernel-aarch64-rt-rhel.config @@ -1283,7 +1283,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1293,8 +1294,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2320,6 +2321,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-ppc64le-debug-rhel.config b/configs/kernel-ppc64le-debug-rhel.config index 13e50f79c0117..6cdcd62d166b6 100644 --- a/configs/kernel-ppc64le-debug-rhel.config +++ b/configs/kernel-ppc64le-debug-rhel.config @@ -1045,7 +1045,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1054,8 +1055,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set diff --git a/configs/kernel-ppc64le-rhel.config b/configs/kernel-ppc64le-rhel.config index c14817c4b65b8..e0217bb0c91a3 100644 --- a/configs/kernel-ppc64le-rhel.config +++ b/configs/kernel-ppc64le-rhel.config @@ -1045,7 +1045,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1054,8 +1055,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set diff --git a/configs/kernel-s390x-debug-rhel.config b/configs/kernel-s390x-debug-rhel.config index 4612cb7922f4d..e8b2bb7e40b42 100644 --- a/configs/kernel-s390x-debug-rhel.config +++ b/configs/kernel-s390x-debug-rhel.config @@ -1041,7 +1041,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1052,8 +1053,8 @@ CONFIG_CRYPTO_HMAC_S390=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set diff --git a/configs/kernel-s390x-rhel.config b/configs/kernel-s390x-rhel.config index 40a2b54fc955e..a67c0c78c90b9 100644 --- a/configs/kernel-s390x-rhel.config +++ b/configs/kernel-s390x-rhel.config @@ -1041,7 +1041,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1052,8 +1053,8 @@ CONFIG_CRYPTO_HMAC_S390=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set diff --git a/configs/kernel-s390x-zfcpdump-rhel.config b/configs/kernel-s390x-zfcpdump-rhel.config index 990151517043b..61bd6210098a0 100644 --- a/configs/kernel-s390x-zfcpdump-rhel.config +++ b/configs/kernel-s390x-zfcpdump-rhel.config @@ -1042,7 +1042,8 @@ CONFIG_CRYPTO_ECHAINIV=y # CONFIG_CRYPTO_ECRDSA is not set # CONFIG_CRYPTO_ESSIV is not set CONFIG_CRYPTO_FCRYPT=y -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1053,8 +1054,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set diff --git a/configs/kernel-x86_64-debug-rhel.config b/configs/kernel-x86_64-debug-rhel.config index 2985ba5f55986..1ff4d7a435afd 100644 --- a/configs/kernel-x86_64-debug-rhel.config +++ b/configs/kernel-x86_64-debug-rhel.config @@ -1099,7 +1099,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1109,8 +1110,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2167,6 +2168,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-x86_64-rhel.config b/configs/kernel-x86_64-rhel.config index b390068c4142a..4e87a4a76bda0 100644 --- a/configs/kernel-x86_64-rhel.config +++ b/configs/kernel-x86_64-rhel.config @@ -1099,7 +1099,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1109,8 +1110,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2151,6 +2152,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-x86_64-rt-debug-rhel.config b/configs/kernel-x86_64-rt-debug-rhel.config index 8b1a48730fb57..dff75ead3d1d7 100644 --- a/configs/kernel-x86_64-rt-debug-rhel.config +++ b/configs/kernel-x86_64-rt-debug-rhel.config @@ -1100,7 +1100,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1110,8 +1111,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2206,6 +2207,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/configs/kernel-x86_64-rt-rhel.config b/configs/kernel-x86_64-rt-rhel.config index 6823de63e269a..c78467ed24948 100644 --- a/configs/kernel-x86_64-rt-rhel.config +++ b/configs/kernel-x86_64-rt-rhel.config @@ -1100,7 +1100,8 @@ CONFIG_CRYPTO_ECHAINIV=m # CONFIG_CRYPTO_ECRDSA is not set CONFIG_CRYPTO_ESSIV=m CONFIG_CRYPTO_FCRYPT=m -# CONFIG_CRYPTO_FIPS_CUSTOM_VERSION is not set +CONFIG_CRYPTO_FIPS_CUSTOM_VERSION=y +CONFIG_CRYPTO_FIPS_VERSION="ciq.6.12.20260826" CONFIG_CRYPTO_FIPS_NAME="Linux Kernel Cryptographic API" CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_GCM=y @@ -1110,8 +1111,8 @@ CONFIG_CRYPTO_GHASH=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_HW=y # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_1024 is not set -# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128 is not set -CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2=y +CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_128=y +# CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_2 is not set # CONFIG_CRYPTO_JITTERENTROPY_MEMSIZE_8192 is not set CONFIG_CRYPTO_JITTERENTROPY_OSR=3 # CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE is not set @@ -2190,6 +2191,7 @@ CONFIG_FUNCTION_TRACER=y # CONFIG_FUN_ETH is not set CONFIG_FUSE_DAX=y CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y CONFIG_FUSE_PASSTHROUGH=y # CONFIG_FUSION_CTL is not set # CONFIG_FUSION_FC is not set diff --git a/crypto/Kconfig b/crypto/Kconfig index f6f2cd8c9d1c1..2f2246c9c5242 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -24,12 +24,12 @@ if CRYPTO menu "Crypto core or helper" config CRYPTO_FIPS - bool "FIPS 200 compliance" - depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS + bool "FIPS compliance" + depends on CRYPTO_DRBG=y && !CRYPTO_MANAGER_DISABLE_TESTS depends on (MODULE_SIG || !MODULES) help This option enables the fips boot option which is - required if you want the system to operate in a FIPS 200 + required if you want the system to operate in a FIPS certification. You should say no unless you know what this is. diff --git a/crypto/dh.c b/crypto/dh.c index e13258c3d1d1a..f8d57fc577f9d 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -479,6 +479,60 @@ static void *dh_safe_prime_gen_privkey(const struct dh_safe_prime *safe_prime, return ERR_PTR(err); } +/* + * SP800-56Ar3, sec. 5.6.2.1.2: obtain owner assurance that an externally + * provided private key is within the valid range 1 <= key <= q - 1, with + * q = (p - 1) / 2 for the safe-prime groups. + * + * The upper bound is q - 1 rather than this implementation's generation + * bound of 2^N - 1 (see dh_safe_prime_gen_privkey()): per SP800-56Ar3 + * sec. 5.6.1.1.1 a compliant generator may pick any N up to log2(q) + 1, + * so an externally generated key can legitimately be as large as q - 1. + * q - 1 is also the only bound derivable from the group alone. + */ +static int dh_safe_prime_check_privkey(const struct dh_safe_prime *safe_prime, + const struct dh *params) +{ + MPI x, p, q; + int ret; + + x = mpi_read_raw_data(params->key, params->key_size); + if (!x) + return -EINVAL; + + p = mpi_read_raw_data(safe_prime->p, safe_prime->p_size); + if (!p) { + ret = -EINVAL; + goto out_free_x; + } + + q = mpi_alloc(mpi_get_nlimbs(p)); + if (!q) { + ret = -ENOMEM; + goto out_free_p; + } + + /* + * p is an odd (safe) prime, so (p - 1) / 2 == p >> 1; no need to + * explicitly subtract one before shifting, cf. dh_is_pubkey_valid(). + */ + ret = mpi_rshift(q, p, 1); + if (ret) + goto out_free_q; + + /* Reject key < 1 (i.e. key == 0) or key > q - 1 (i.e. key >= q). */ + if (mpi_cmp_ui(x, 1) < 0 || mpi_cmp(x, q) >= 0) + ret = -EINVAL; + +out_free_q: + mpi_free(q); +out_free_p: + mpi_free(p); +out_free_x: + mpi_free(x); + return ret; +} + static int dh_safe_prime_set_secret(struct crypto_kpp *tfm, const void *buffer, unsigned int len) { @@ -509,6 +563,15 @@ static int dh_safe_prime_set_secret(struct crypto_kpp *tfm, const void *buffer, if (IS_ERR(key)) return PTR_ERR(key); params.key = key; + } else if (fips_enabled) { + /* + * A locally generated key is in range by construction; an + * externally provided one must be validated per + * SP800-56Ar3 sec. 5.6.2.1.2. + */ + err = dh_safe_prime_check_privkey(inst_ctx->safe_prime, ¶ms); + if (err) + return err; } buf_size = crypto_dh_key_len(¶ms); diff --git a/crypto/drbg.c b/crypto/drbg.c index 20321c8140e0d..250afd830b898 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1292,6 +1292,12 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) if (ret < 0) goto err; + /* + * Align to at least a cache line for better performance. This also + * prevents false sharing of cache lines between different instances. + */ + ret = max(ret, L1_CACHE_BYTES - 1); + drbg->Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); if (!drbg->Vbuf) { ret = -ENOMEM; diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 72cfd15901563..e069c7bb51164 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -11,6 +11,7 @@ #include #include #include +#include struct ecdh_ctx { unsigned int curve_id; @@ -99,6 +100,36 @@ static int ecdh_compute_value(struct kpp_request *req) ctx->private_key, public_key); buf = public_key; nbytes = public_key_sz; + + /* + * SP800-56Arev3, 5.6.2.1.4: ("Owner Assurance of + * Pair-wise Consistency"): recompute the public key + * and check if the results match. + */ + if (fips_enabled) { + u64 *public_key_pct; + + if (ret < 0) + goto free_all; + + public_key_pct = kmalloc(public_key_sz, GFP_KERNEL); + if (!public_key_pct) { + ret = -ENOMEM; + goto free_all; + } + + ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits, + ctx->private_key, + public_key_pct); + if (ret < 0) { + kfree(public_key_pct); + goto free_all; + } + + if (memcmp(public_key, public_key_pct, public_key_sz)) + panic("ECDH PCT failed in FIPS mode"); + kfree(public_key_pct); + } } if (ret < 0) diff --git a/crypto/essiv.c b/crypto/essiv.c index 1c00c3324058c..b715ec33e3eaf 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -114,13 +114,16 @@ static int essiv_aead_setkey(struct crypto_aead *tfm, const u8 *key, crypto_shash_update(desc, keys.enckey, keys.enckeylen) ?: crypto_shash_finup(desc, keys.authkey, keys.authkeylen, salt); if (err) - return err; + goto out; crypto_cipher_clear_flags(tctx->essiv_cipher, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(tctx->essiv_cipher, crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); - return crypto_cipher_setkey(tctx->essiv_cipher, salt, + err = crypto_cipher_setkey(tctx->essiv_cipher, salt, crypto_shash_digestsize(tctx->hash)); +out: + memzero_explicit(&keys, sizeof(keys)); + return err; } static int essiv_aead_setauthsize(struct crypto_aead *tfm, diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c index c3f9938e1ad27..50f90a16d7ddd 100644 --- a/crypto/kdf_sp800108.c +++ b/crypto/kdf_sp800108.c @@ -25,6 +25,10 @@ int crypto_kdf108_ctr_generate(struct crypto_shash *kmd, int err = 0; u8 *dst_orig = dst; + /* SP800-131Arev2: minimum 112-bit output length */ + if (fips_enabled && dlen < 112 / 8) + return -EINVAL; + desc->tfm = kmd; while (dlen) { diff --git a/crypto/rng.c b/crypto/rng.c index 3c9b158ac2b07..f757195dcd634 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -6,6 +6,9 @@ * * Copyright (c) 2008 Neil Horman * Copyright (c) 2015 Herbert Xu + * + * Copyright (C) 2025 Ctrl IQ, Inc. + * Author: Sultan Alsawaf */ #include @@ -15,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -26,12 +29,39 @@ #include "internal.h" -static ____cacheline_aligned_in_smp DEFINE_MUTEX(crypto_reseed_rng_lock); -static struct crypto_rng *crypto_reseed_rng; -static ____cacheline_aligned_in_smp DEFINE_MUTEX(crypto_default_rng_lock); +static ____cacheline_aligned_in_smp DEFINE_RT_MUTEX(crypto_default_rng_lock); struct crypto_rng *crypto_default_rng; EXPORT_SYMBOL_GPL(crypto_default_rng); -static int crypto_default_rng_refcnt; +static unsigned int crypto_default_rng_refcnt; +static bool drbg_registered __ro_after_init; + +/* + * Per-CPU RNG instances are only used by crypto_devrandom_rng. The global RNG, + * crypto_default_rng, is only used directly by other drivers. + * + * Per-CPU instances of the DRBG are efficient because the DRBG itself supports + * an arbitrary number of instances and can be seeded on a per-CPU basis. + * + * Specifically, the DRBG is seeded by the CRNG and the Jitter RNG. The CRNG is + * globally accessible and is already per-CPU. And while the Jitter RNG _isn't_ + * per-CPU, creating a DRBG instance also creates a Jitter RNG instance; + * therefore, per-CPU DRBG instances implies per-CPU Jitter RNG instances. + */ +struct cpu_rng_inst { + local_lock_t lock; + struct rt_mutex mlock; + struct crypto_rng *rng; + void *page; +}; + +static DEFINE_PER_CPU_ALIGNED(struct cpu_rng_inst, pcpu_default_rng) = { + .lock = INIT_LOCAL_LOCK(pcpu_default_rng.lock), + .mlock = __RT_MUTEX_INITIALIZER(pcpu_default_rng.mlock) +}; +static DEFINE_PER_CPU_ALIGNED(struct cpu_rng_inst, pcpu_reseed_rng) = { + /* The reseed instances don't use the local lock */ + .mlock = __RT_MUTEX_INITIALIZER(pcpu_reseed_rng.mlock) +}; int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) { @@ -138,11 +168,11 @@ int crypto_get_default_rng(void) { int err; - mutex_lock(&crypto_default_rng_lock); + rt_mutex_lock(&crypto_default_rng_lock); err = crypto_get_rng(&crypto_default_rng); if (!err) crypto_default_rng_refcnt++; - mutex_unlock(&crypto_default_rng_lock); + rt_mutex_unlock(&crypto_default_rng_lock); return err; } @@ -150,42 +180,33 @@ EXPORT_SYMBOL_GPL(crypto_get_default_rng); void crypto_put_default_rng(void) { - mutex_lock(&crypto_default_rng_lock); + rt_mutex_lock(&crypto_default_rng_lock); crypto_default_rng_refcnt--; - mutex_unlock(&crypto_default_rng_lock); + rt_mutex_unlock(&crypto_default_rng_lock); } EXPORT_SYMBOL_GPL(crypto_put_default_rng); #if defined(CONFIG_CRYPTO_RNG) || defined(CONFIG_CRYPTO_RNG_MODULE) -static int crypto_del_rng(struct crypto_rng **rngp, int *refcntp, - struct mutex *lock) +int crypto_del_default_rng(void) { - int err = -EBUSY; + bool busy; - mutex_lock(lock); - if (refcntp && *refcntp) - goto out; - - crypto_free_rng(*rngp); - *rngp = NULL; - - err = 0; - -out: - mutex_unlock(lock); + rt_mutex_lock(&crypto_default_rng_lock); + if (!(busy = crypto_default_rng_refcnt)) { + crypto_free_rng(crypto_default_rng); + crypto_default_rng = NULL; + } + rt_mutex_unlock(&crypto_default_rng_lock); - return err; + return busy ? -EBUSY : 0; } +EXPORT_SYMBOL_GPL(crypto_del_default_rng); +#endif -int crypto_del_default_rng(void) +static void rng_default_set_ent(struct crypto_rng *tfm, const u8 *data, + unsigned int len) { - return crypto_del_rng(&crypto_default_rng, &crypto_default_rng_refcnt, - &crypto_default_rng_lock) ?: - crypto_del_rng(&crypto_reseed_rng, NULL, - &crypto_reseed_rng_lock); } -EXPORT_SYMBOL_GPL(crypto_del_default_rng); -#endif int crypto_register_rng(struct rng_alg *alg) { @@ -194,10 +215,26 @@ int crypto_register_rng(struct rng_alg *alg) if (alg->seedsize > PAGE_SIZE / 8) return -EINVAL; + /* + * In FIPS mode, the DRBG must take precedence over all other "stdrng" + * algorithms. Therefore, forbid registration of a non-DRBG stdrng in + * FIPS mode. All of the DRBG's driver names are prefixed with "drbg_". + * This also stops new stdrng instances from getting registered after it + * is known that the DRBG is registered, so a new module can't come in + * and pretend to be the DRBG. And when CONFIG_CRYPTO_FIPS is enabled, + * the DRBG is built into the kernel directly; it can't be a module. + */ + if (fips_enabled && !strcmp(base->cra_name, "stdrng") && + (drbg_registered || strncmp(base->cra_driver_name, "drbg_", 5))) + return -EINVAL; + base->cra_type = &crypto_rng_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_RNG; + if (!alg->set_ent) + alg->set_ent = rng_default_set_ent; + return crypto_register_alg(base); } EXPORT_SYMBOL_GPL(crypto_register_rng); @@ -218,6 +255,18 @@ int crypto_register_rngs(struct rng_alg *algs, int count) goto err; } + /* + * Track when the DRBG is registered in FIPS mode. The DRBG calls + * crypto_register_rngs() to register its stdrng instances, and since + * crypto_register_rng() only allows stdrng instances from the DRBG in + * FIPS mode, a successful stdrng registration means it was the DRBG. + * Just check the first alg in the array to see if it's called "stdrng", + * since all of the DRBG's algorithms are named "stdrng". Once + * drbg_registered is set to true, this if-statement is always false. + */ + if (fips_enabled && !strcmp(algs->base.cra_name, "stdrng")) + drbg_registered = true; + return 0; err: @@ -237,109 +286,429 @@ void crypto_unregister_rngs(struct rng_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_rngs); -static ssize_t crypto_devrandom_read_iter(struct iov_iter *iter, bool reseed) +/* + * On non-PREEMPT_RT kernels, local locks disable preemption. When there's no + * rng allocated, one must be allocated by calling crypto_get_rng(), which can + * sleep. Therefore, crypto_get_rng() cannot be called under local_lock(), so if + * our CPU's RNG instance doesn't have an rng allocated, we drop the local lock + * and take a mutex lock instead. After the local lock is dropped, the current + * task can be freely migrated to another CPU, which means that calling + * local_lock() again might not result in the same instance getting locked as + * before. That's why this function exists: to loop on calling local_lock() and + * allocating an rng as needed with crypto_get_rng() until the current CPU's + * instance is found to have an rng allocated. If crypto_get_rng() ever fails, + * this function returns an error even if there are instances for other CPUs + * which _do_ have an rng allocated. + */ +static __always_inline struct cpu_rng_inst * +lock_default_rng(struct crypto_rng **rng) __acquires(&cri->lock) { - struct crypto_rng *rng; - u8 tmp[256]; - ssize_t ret; + struct cpu_rng_inst __percpu *pcri = &pcpu_default_rng; + struct cpu_rng_inst *cri; + int ret; + + while (1) { + local_lock(&pcri->lock); + cri = this_cpu_ptr(pcri); + /* + * cri->rng can only transition from NULL to non-NULL. This may + * occur on a different CPU, thus cri->rng must be read + * atomically to prevent data races; this elides mlock by + * pairing with the WRITE_ONCE() in the slow path below. + * + * And if cri->rng is non-NULL, then it is good to go. To avoid + * data races due to load speculation on torn cri->rng loads + * _after_ the NULL check, one of the following is required: + * 1. smp_acquire__after_ctrl_dep() in the if-statement + * 2. All cri->rng reads are performed with READ_ONCE() + * 3. cri->rng is never read again outside this function + * + * Option #3 yields the best performance, so this function + * provides the rng pointer as an output for the caller to use. + */ + *rng = READ_ONCE(cri->rng); + if (likely(*rng)) + return cri; + + /* + * Slow path: there's no rng currently allocated to this instance. + * Release the local lock and acquire this instance's mlock to + * perform the allocation. + * + * Note that this task may be migrated to a different CPU now! + */ + __local_unlock(&cri->lock); + rt_mutex_lock(&cri->mlock); + if (!cri->rng) { + struct crypto_rng *new_rng = NULL; + + ret = crypto_get_rng(&new_rng); + if (ret) { + rt_mutex_unlock(&cri->mlock); + break; + } - if (unlikely(!iov_iter_count(iter))) - return 0; + /* + * Pairs with READ_ONCE() above, because we might not be + * on the same CPU anymore as when we first got `cri`. + */ + WRITE_ONCE(cri->rng, new_rng); + } + rt_mutex_unlock(&cri->mlock); + } - if (reseed) { - u32 flags = 0; + /* + * Even if this task got migrated to another CPU that _does_ have an rng + * allocated, just bail out if crypto_get_rng() ever fails in order to + * avoid looping forever. + */ + return ERR_PTR(ret); +} - /* If reseeding is requested, acquire a lock on - * crypto_reseed_rng so it is not swapped out until - * the initial random bytes are generated. - * - * The algorithm implementation is also protected with - * a separate mutex (drbg->drbg_mutex) around the - * reseed-and-generate operation. +static __always_inline struct cpu_rng_inst * +lock_reseed_rng(struct crypto_rng **rng) __acquires(&cri->mlock) +{ + struct cpu_rng_inst __percpu *pcri = &pcpu_reseed_rng; + struct cpu_rng_inst *cri; + int ret; + + /* + * Use whichever CPU this task is currently running on, knowing full + * well that the task can freely migrate to other CPUs. The reseed RNG + * requires holding a lock across the entire devrandom read, so that + * another task cannot extract entropy from the same seed. In other + * words, when reseeding is requested, reseeding must be done every time + * every time mlock is acquired. + */ + cri = raw_cpu_ptr(pcri); + rt_mutex_lock(&cri->mlock); + if (likely(cri->rng)) { + /* + * Since this rng instance wasn't just allocated, it needs to be + * explicitly reseeded. New rng instances are seeded on creation + * in crypto_get_rng() and thus don't need explicit reseeding. */ - mutex_lock(&crypto_reseed_rng_lock); + crypto_tfm_set_flags(crypto_rng_tfm(cri->rng), + CRYPTO_TFM_REQ_NEED_RESEED); + } else { + ret = crypto_get_rng(&cri->rng); + if (ret) { + rt_mutex_unlock(&cri->mlock); + return ERR_PTR(ret); + } + } - /* If crypto_default_rng is not set, it will be seeded - * at creation in __crypto_get_default_rng and thus no - * reseeding is needed. - */ - if (crypto_reseed_rng) - flags |= CRYPTO_TFM_REQ_NEED_RESEED; + *rng = cri->rng; + return cri; +} - ret = crypto_get_rng(&crypto_reseed_rng); - if (ret) { - mutex_unlock(&crypto_reseed_rng_lock); - return ret; +#define lock_local_rng(rng, reseed) \ + ({ (reseed) ? lock_reseed_rng(rng) : lock_default_rng(rng); }) + +#define unlock_local_rng(cri, reseed) \ +do { \ + if (reseed) \ + rt_mutex_unlock(&(cri)->mlock); \ + else \ + __local_unlock(&(cri)->lock); \ +} while (0) + +static __always_inline void +clear_rng_page(struct cpu_rng_inst *cri, size_t count) +{ + /* For zeroing a whole page, clear_page() is faster than memset() */ + count < PAGE_SIZE ? memset(cri->page, 0, count) : clear_page(cri->page); +} + +static __always_inline void +no_reseed_clear_unlock(struct cpu_rng_inst *cri, size_t *page_dirty_len) +{ + /* + * Clear the buffer of our latest random bytes before unlocking and + * potentially migrating CPUs, in which case we wouldn't have the same + * `cri` anymore. + */ + clear_rng_page(cri, *page_dirty_len); + unlock_local_rng(cri, false); + *page_dirty_len = 0; +} + +static ssize_t crypto_devrandom_read_iter(struct iov_iter *iter, bool reseed) +{ + /* lock_local_rng() puts us in atomic context for !reseed on non-RT */ + const bool atomic = !reseed && !IS_ENABLED(CONFIG_PREEMPT_RT); + const bool user_no_reseed = !reseed && user_backed_iter(iter); + /* Align the bounce buffer to a word to prevent alignment traps */ + u8 bounce[SZ_256] __aligned(sizeof(long)); + size_t ulen, page_dirty_len = 0; + struct cpu_rng_inst *cri; + struct crypto_rng *rng; + u8 __user *uaddr; + ssize_t ret = 0; + + if (unlikely(!iov_iter_count(iter))) + return 0; + + /* Set up the starting user destination address and length */ + if (user_no_reseed) { + if (iter_is_ubuf(iter)) { + uaddr = iter->ubuf + iter->iov_offset; + ulen = iov_iter_count(iter); + } else if (iter_is_iovec(iter)) { + /* + * Skip any leading zero-length segments first, since + * uaddr would otherwise point at an empty segment's + * base (which may be NULL or unwritable). Advancing by + * zero walks past every leading empty segment (the + * iovec advance loop stops at the first non-empty one), + * and there's guaranteed to be a non-empty one because + * iov_iter_count() is nonzero above. + */ + iov_iter_advance(iter, 0); + uaddr = iter_iov_addr(iter); + ulen = iter_iov_len(iter); + } else { + /* + * ITER_UBUF and ITER_IOVEC are the only user-backed + * iters. Bug out if a new user-backed iter appears. + */ + BUG(); } + } - rng = crypto_reseed_rng; - crypto_tfm_set_flags(crypto_rng_tfm(rng), flags); - } else { - ret = crypto_get_default_rng(); - if (ret) - return ret; - rng = crypto_default_rng; +restart: + /* + * Prefault the current user destination address in case the page for + * that virtual address is not resident. This is done by simply writing + * a single zero byte to the destination, which forces a COW if the page + * isn't resident, leaving us with a writable mapping. By prefaulting + * without the RNG lock held, the DRBG won't be blocked by time spent on + * page faults for this task, and thus the DRBG can still be used by + * other tasks. + * + * Notably, this doesn't keep the page from getting evicted later on; + * it's just an optimistic prefault under the assumption that the page + * won't be evicted before we copy random bytes into it, which is often + * but _not always_ the case. + * + * While GUP pinning makes it possible to pin the page _backing_ a user + * address, it *doesn't* pin the page table entry (PTE) for that + * mapping. This means the pinned physical page can be separated from + * the user address it was backing, and even back a _different_ user + * address within the same process. PTE zapping naturally happens during + * memory reclaim when memory pressure is elevated, and can even be done + * directly by userspace via madvise(MADV_DONTNEED). Since there's no + * way to pin a PTE, that means it's not possible to guarantee a page + * will be resident when copy_to_user_nofault() runs under the RNG lock + * below; in other words, there's no way to guarantee a 100% success + * rate for the copy_to_user_nofault() attempt. + */ + if (user_no_reseed && put_user((u8)0, uaddr)) + goto exit; + + cri = lock_local_rng(&rng, reseed); + if (IS_ERR(cri)) { + if (!ret) + ret = PTR_ERR(cri); + goto exit; } - for (;;) { - size_t i, copied; + while (1) { + size_t copied, i = min(iov_iter_count(iter), PAGE_SIZE); + bool resched_without_lock = false; int err; - i = min_t(size_t, iov_iter_count(iter), sizeof(tmp)); - err = crypto_rng_get_bytes(rng, tmp, i); + /* + * Generate up to one page at a time, and align to a page + * boundary to track page residence one page at a time. + */ + if (user_no_reseed) + i = min3(i, PAGE_SIZE - offset_in_page(uaddr), ulen); + + /* + * On non-PREEMPT_RT kernels, local locks disable preemption. + * The DRBG's generate() function has a mutex lock, which could + * mean that we'll schedule while atomic if the mutex lock + * sleeps. However, that will never happen if we ensure that + * there's never any contention on the DRBG's mutex lock while + * we're atomic! Our local lock ensures calls to the DRBG are + * always serialized, so there's no contention from here. And + * the DRBG only uses its mutex lock from one other path, when + * an instance of the DRBG is freshly allocated, which we only + * do from crypto_get_rng(). So the DRBG's mutex lock is + * guaranteed to not have contention when we call generate() and + * thus it'll never sleep here. And of course, nothing else in + * generate() ever sleeps. + */ + err = crypto_rng_get_bytes(rng, cri->page, i); if (err) { - ret = ret ?: err; + if (!ret) + ret = err; break; } - copied = copy_to_iter(tmp, i, iter); + /* + * Record the number of bytes used in cri->page and either copy + * directly to the user address without faulting, or copy to the + * iter which is always backed by kernel memory when !reseed && + * !user_backed_iter(). When reseed == true, the iter may be + * backed by user memory, but we copy to it with the possibility + * of page faults anyway because we need to hold the lock across + * the entire call; this is why a mutex is used instead of a + * local lock for the reseed RNG, to permit sleeping without + * yielding the DRBG instance. + * + * From this point on, err != 0 means the nofault copy failed. + */ + page_dirty_len = max(i, page_dirty_len); + if (user_no_reseed && + !(err = copy_to_user_nofault(uaddr, cri->page, i))) { + iov_iter_advance(iter, i); + copied = i; + } else { + const void *src = cri->page; + + /* + * Fall back to a faultable copy attempt when the + * non-faulting attempt failed, likely because the PTE + * was zapped. To ensure forward progress, some of the + * generated bytes are retained in an on-stack bounce + * buffer and copied out rather than just discarding all + * of it and going back to `restart`. + */ + if (unlikely(err)) { + i = min(i, sizeof(bounce)); + memcpy(bounce, cri->page, i); + no_reseed_clear_unlock(cri, &page_dirty_len); + src = bounce; + } + + /* + * We know that copying from cri->page (or the bounce + * buffer) is safe, so use _copy_to_iter() directly to + * skip check_copy_size(). + */ + copied = _copy_to_iter(src, i, iter); + + /* Sanitize the on-stack bounce buffer if it was used */ + if (err) + memzero_explicit(bounce, i); + } ret += copied; - if (!iov_iter_count(iter) || copied != i) + /* + * Quit when the copy fell short, the requested number of bytes + * have been generated, or there is a pending signal. + */ + if (copied != i || !iov_iter_count(iter) || + signal_pending(current)) { + /* Skip the unlock when already unlocked */ + if (unlikely(err)) + goto exit; break; + } - BUILD_BUG_ON(PAGE_SIZE % sizeof(tmp) != 0); - if (ret % PAGE_SIZE == 0) { - if (signal_pending(current)) - break; - cond_resched(); + /* Compute the next user destination address and length */ + if (user_no_reseed) { + ulen -= i; + if (likely(ulen)) { + uaddr += i; + } else { + /* + * This path is only reachable by ITER_IOVEC + * because ulen is initialized to the request + * size for ITER_UBUF, and therefore ITER_UBUF + * will always quit at the iov_iter_count() + * check above before ulen can become zero. + * + * iter->iov_offset is guaranteed to be zero + * here, so iter_iov_{addr|len}() isn't needed. + */ + uaddr = iter_iov(iter)->iov_base; + ulen = iter_iov(iter)->iov_len; + } + } + + /* + * Reschedule right now if needed and we're not atomic. If we're + * atomic, then we must first drop the lock to reschedule. + */ + if (need_resched()) { + if (atomic) + resched_without_lock = true; + else + cond_resched(); + } + + /* + * Optimistically check if the next user page is resident and + * writable with a dummy copy_to_user_nofault() attempt to write + * a single zero byte. If this fails, fall back to prefaulting + * outside of the lock. + */ + if (!err && user_no_reseed && !resched_without_lock && + !copy_to_user_nofault(uaddr, &(u8){ 0 }, 1)) + continue; + + /* + * Restart if either rescheduling is needed (and requires + * dropping the lock since we're atomic) or the destination page + * wasn't resident when copy_to_user_nofault() was attempted. + * + * This always implies `reseed == false`, so unlock_local_rng() + * can just be passed `false` for reseed to eliminate a branch, + * hence using no_reseed_clear_unlock(). + */ + if (resched_without_lock || user_no_reseed) { + if (!err) + no_reseed_clear_unlock(cri, &page_dirty_len); + if (resched_without_lock) + cond_resched(); + goto restart; } } - if (reseed) - mutex_unlock(&crypto_reseed_rng_lock); - else - crypto_put_default_rng(); - memzero_explicit(tmp, sizeof(tmp)); + if (page_dirty_len) + clear_rng_page(cri, page_dirty_len); + unlock_local_rng(cri, reseed); +exit: return ret ? ret : -EFAULT; } static const struct random_extrng crypto_devrandom_rng = { - .extrng_read_iter = crypto_devrandom_read_iter, - .owner = THIS_MODULE, + .extrng_read_iter = crypto_devrandom_read_iter }; -static int __init crypto_rng_init(void) +static void __init alloc_pcpu_inst(struct cpu_rng_inst __percpu *pcri) { - int err; + int cpu; - if (fips_enabled) { - err = crypto_get_default_rng(); - if (err) - return err; - crypto_put_default_rng(); - random_register_extrng(&crypto_devrandom_rng); + for_each_possible_cpu(cpu) { + struct cpu_rng_inst *cri = per_cpu_ptr(pcri, cpu); + + cri->page = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL); + local_lock_init(&cri->lock); } - return 0; } -static void __exit crypto_rng_exit(void) +static int __init crypto_rng_init(void) { - random_unregister_extrng(); + if (!fips_enabled) + return 0; + + /* + * Never fail to register the RNG override in FIPS mode because failure + * would result in the system quietly booting without the FIPS-mandated + * RNG installed. This would be catastrophic for FIPS compliance, hence + * the RNG override setup is *not* allowed to fail. + */ + alloc_pcpu_inst(&pcpu_default_rng); + alloc_pcpu_inst(&pcpu_reseed_rng); + random_register_extrng(&crypto_devrandom_rng); + return 0; } late_initcall(crypto_rng_init); -module_exit(crypto_rng_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Random Number Generator"); diff --git a/drivers/char/random.c b/drivers/char/random.c index 5c09d6b360623..cb5a4205e16e6 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -341,7 +340,7 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], /* * Hook for external RNG. */ -static const struct random_extrng __rcu *extrng; +static const struct random_extrng *extrng __ro_after_init; /* * This function returns a ChaCha state that you may use for generating @@ -999,18 +998,12 @@ void __init add_bootloader_randomness(const void *buf, size_t len) credit_init_bits(len * 8); } -void random_register_extrng(const struct random_extrng *rng) +void __init random_register_extrng(const struct random_extrng *rng) { - rcu_assign_pointer(extrng, rng); + /* Don't allow the registered extrng to be overridden */ + BUG_ON(extrng); + extrng = rng; } -EXPORT_SYMBOL_GPL(random_register_extrng); - -void random_unregister_extrng(void) -{ - RCU_INIT_POINTER(extrng, NULL); - synchronize_rcu(); -} -EXPORT_SYMBOL_GPL(random_unregister_extrng); #if IS_ENABLED(CONFIG_VMGENID) static BLOCKING_NOTIFIER_HEAD(vmfork_chain); @@ -1421,7 +1414,6 @@ SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags { struct iov_iter iter; int ret; - const struct random_extrng *rng; if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE)) return -EINVAL; @@ -1433,19 +1425,11 @@ SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM)) return -EINVAL; - rcu_read_lock(); - rng = rcu_dereference(extrng); - if (rng && !try_module_get(rng->owner)) - rng = NULL; - rcu_read_unlock(); - - if (rng) { + if (extrng) { ret = import_ubuf(ITER_DEST, ubuf, len, &iter); if (unlikely(ret)) return ret; - ret = rng->extrng_read_iter(&iter, !!(flags & GRND_RANDOM)); - module_put(rng->owner); - return ret; + return extrng->extrng_read_iter(&iter, !!(flags & GRND_RANDOM)); } if (!crng_ready() && !(flags & GRND_INSECURE)) { @@ -1616,52 +1600,24 @@ static int random_fasync(int fd, struct file *filp, int on) static int random_open(struct inode *inode, struct file *filp) { - const struct random_extrng *rng; - - rcu_read_lock(); - rng = rcu_dereference(extrng); - if (rng && !try_module_get(rng->owner)) - rng = NULL; - rcu_read_unlock(); - - if (!rng) - return 0; - - filp->f_op = &extrng_random_fops; - filp->private_data = rng->owner; + if (extrng) + filp->f_op = &extrng_random_fops; return 0; } static int urandom_open(struct inode *inode, struct file *filp) { - const struct random_extrng *rng; + if (extrng) + filp->f_op = &extrng_urandom_fops; - rcu_read_lock(); - rng = rcu_dereference(extrng); - if (rng && !try_module_get(rng->owner)) - rng = NULL; - rcu_read_unlock(); - - if (!rng) - return 0; - - filp->f_op = &extrng_urandom_fops; - filp->private_data = rng->owner; - - return 0; -} - -static int extrng_release(struct inode *inode, struct file *filp) -{ - module_put(filp->private_data); return 0; } static ssize_t extrng_read_iter(struct kiocb *kiocb, struct iov_iter *iter) { - return rcu_dereference_raw(extrng)->extrng_read_iter(iter, false); + return extrng->extrng_read_iter(iter, false); } const struct file_operations random_fops = { @@ -1698,7 +1654,6 @@ static const struct file_operations extrng_random_fops = { .compat_ioctl = compat_ptr_ioctl, .fasync = random_fasync, .llseek = noop_llseek, - .release = extrng_release, .splice_read = copy_splice_read, .splice_write = iter_file_splice_write, }; @@ -1711,7 +1666,6 @@ static const struct file_operations extrng_urandom_fops = { .compat_ioctl = compat_ptr_ioctl, .fasync = random_fasync, .llseek = noop_llseek, - .release = extrng_release, .splice_read = copy_splice_read, .splice_write = iter_file_splice_write, }; diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 7e52abe89e767..4b997c8bef460 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -869,6 +869,12 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, rx->rx_hsplit_unsplit_pkt += unsplit; rx->rx_hsplit_bytes += hdr_len; u64_stats_update_end(&rx->statss); + + if (!buf_len) { + gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, + buf_state); + return 0; + } } /* Sync the portion of dma buffer for CPU to read. */ diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 0992c6d11eb06..d6d81c29c915e 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -272,8 +272,9 @@ static void gve_tx_free_ring_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, static int gve_tx_qpl_buf_init(struct gve_tx_ring *tx) { - int num_tx_qpl_bufs = GVE_TX_BUFS_PER_PAGE_DQO * - tx->dqo.qpl->num_entries; + int num_tx_qpl_bufs = min_t(int, + GVE_TX_BUFS_PER_PAGE_DQO * tx->dqo.qpl->num_entries, + S16_MAX); int i; tx->dqo.tx_qpl_buf_next = kvcalloc(num_tx_qpl_bufs, diff --git a/fs/dcache.c b/fs/dcache.c index cd7c019f92f9b..17541c28ae9d0 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1032,6 +1032,15 @@ struct dentry *d_find_alias_rcu(struct inode *inode) return de; } +void d_dispose_if_unused(struct dentry *dentry, struct list_head *dispose) +{ + spin_lock(&dentry->d_lock); + if (!dentry->d_lockref.count) + to_shrink_list(dentry, dispose); + spin_unlock(&dentry->d_lock); +} +EXPORT_SYMBOL(d_dispose_if_unused); + /* * Try to kill dentries associated with this inode. * WARNING: you must own a reference to inode. @@ -1042,12 +1051,8 @@ void d_prune_aliases(struct inode *inode) struct dentry *dentry; spin_lock(&inode->i_lock); - hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { - spin_lock(&dentry->d_lock); - if (!dentry->d_lockref.count) - to_shrink_list(dentry, &dispose); - spin_unlock(&dentry->d_lock); - } + hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) + d_dispose_if_unused(dentry, &dispose); spin_unlock(&inode->i_lock); shrink_dentry_list(&dispose); } @@ -1087,6 +1092,7 @@ void shrink_dentry_list(struct list_head *list) shrink_kill(dentry); } } +EXPORT_SYMBOL(shrink_dentry_list); static enum lru_status dentry_lru_isolate(struct list_head *item, struct list_lru_one *lru, void *arg) diff --git a/fs/exec.c b/fs/exec.c index 2c6e61139ac78..7cb7a1b283957 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1203,6 +1203,17 @@ void __set_task_comm(struct task_struct *tsk, const char *buf, bool exec) perf_event_comm(tsk, exec); } +static void posixtimer_exec(struct task_struct *me) +{ +#ifdef CONFIG_POSIX_TIMERS + spin_lock_irq(&me->sighand->siglock); + posix_cpu_timers_exit(me); + spin_unlock_irq(&me->sighand->siglock); + exit_itimers(me); + flush_itimer_signals(); +#endif +} + /* * Calling this is the point of no return. None of the failures will be * seen by userspace since either the process is already taking a fatal @@ -1239,6 +1250,15 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) goto out; + /* + * This must be done here to ensure that POSIX CPU timers which were + * armed on the current task are dequeued from me::posix_cputimers. + * Otherwise in case of a TID switch the deletion of the related POSIX + * timer would not remove an enqueued timer because the TID lookup + * of the old TID fails. + */ + posixtimer_exec(me); + /* * Cancel any io_uring activity across execve */ @@ -1277,14 +1297,6 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) goto out_unlock; -#ifdef CONFIG_POSIX_TIMERS - spin_lock_irq(&me->sighand->siglock); - posix_cpu_timers_exit(me); - spin_unlock_irq(&me->sighand->siglock); - exit_itimers(me); - flush_itimer_signals(); -#endif - /* * Make the signal table private. */ diff --git a/fs/fuse/Kconfig b/fs/fuse/Kconfig index 8674dbfbe59db..ca215a3cba3e3 100644 --- a/fs/fuse/Kconfig +++ b/fs/fuse/Kconfig @@ -63,3 +63,15 @@ config FUSE_PASSTHROUGH to be performed directly on a backing file. If you want to allow passthrough operations, answer Y. + +config FUSE_IO_URING + bool "FUSE communication over io-uring" + default y + depends on FUSE_FS + depends on IO_URING + help + This allows sending FUSE requests over the io-uring interface and + also adds request core affinity. + + If you want to allow fuse server/client communication through io-uring, + answer Y diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index ce0ff7a9007b9..fcf16b1c391a9 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -14,5 +14,6 @@ fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o +fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o virtiofs-y := virtio_fs.o diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index 12ef91d170bb3..44bd30d448e4e 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -240,11 +240,12 @@ static int fuse_send_removemapping(struct inode *inode, args.opcode = FUSE_REMOVEMAPPING; args.nodeid = fi->nodeid; - args.in_numargs = 2; - args.in_args[0].size = sizeof(*inargp); - args.in_args[0].value = inargp; - args.in_args[1].size = inargp->count * sizeof(*remove_one); - args.in_args[1].value = remove_one; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = sizeof(*inargp); + args.in_args[1].value = inargp; + args.in_args[2].size = inargp->count * sizeof(*remove_one); + args.in_args[2].value = remove_one; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index d370e8a29c636..41830ee9459d3 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -6,7 +6,9 @@ See the file COPYING. */ +#include "dev_uring_i.h" #include "fuse_i.h" +#include "fuse_dev_i.h" #include #include @@ -21,6 +23,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include "fuse_trace.h" @@ -28,23 +31,8 @@ MODULE_ALIAS_MISCDEV(FUSE_MINOR); MODULE_ALIAS("devname:fuse"); -/* Ordinary requests have even IDs, while interrupts IDs are odd */ -#define FUSE_INT_REQ_BIT (1ULL << 0) -#define FUSE_REQ_ID_STEP (1ULL << 1) - static struct kmem_cache *fuse_req_cachep; -static void end_requests(struct list_head *head); - -static struct fuse_dev *fuse_get_dev(struct file *file) -{ - /* - * Lockless access is OK, because file->private data is set - * once during mount and is valid until the file is released. - */ - return READ_ONCE(file->private_data); -} - static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req) { INIT_LIST_HEAD(&req->list); @@ -90,7 +78,14 @@ void fuse_set_initialized(struct fuse_conn *fc) static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background) { - return !fc->initialized || (for_background && fc->blocked); + if (!fc->initialized) + return true; + + /* Pairs with smp_wmb() in fuse_set_initialized() */ + smp_rmb(); + + return (for_background && fc->blocked) || + (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); } static void fuse_drop_waiting(struct fuse_conn *fc) @@ -128,8 +123,6 @@ static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap, !fuse_block_alloc(fc, for_background))) goto out; } - /* Matches smp_wmb() in fuse_set_initialized() */ - smp_rmb(); err = -ENOTCONN; if (!fc->connected) @@ -235,7 +228,7 @@ u64 fuse_get_unique(struct fuse_iqueue *fiq) } EXPORT_SYMBOL_GPL(fuse_get_unique); -static unsigned int fuse_req_hash(u64 unique) +unsigned int fuse_req_hash(u64 unique) { return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS); } @@ -251,7 +244,8 @@ __releases(fiq->lock) spin_unlock(&fiq->lock); } -static void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *forget) +void fuse_dev_queue_forget(struct fuse_iqueue *fiq, + struct fuse_forget_link *forget) { spin_lock(&fiq->lock); if (fiq->connected) { @@ -264,7 +258,7 @@ static void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_li } } -static void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) +void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ @@ -344,6 +338,29 @@ static void flush_bg_queue(struct fuse_conn *fc) } } +void fuse_request_bg_finish(struct fuse_conn *fc, struct fuse_req *req) +{ + lockdep_assert_held(&fc->bg_lock); + + clear_bit(FR_BACKGROUND, &req->flags); + if (fc->num_background == fc->max_background) { + fc->blocked = 0; + wake_up(&fc->blocked_waitq); + } else if (!fc->blocked) { + /* + * Wake up next waiter, if any. It's okay to use + * waitqueue_active(), as we've already synced up + * fc->blocked with waiters with the wake_up() call + * above. + */ + if (waitqueue_active(&fc->blocked_waitq)) + wake_up(&fc->blocked_waitq); + } + + fc->num_background--; + fc->active_background--; +} + /* * This function is called when a request is finished. Either a reply * has arrived or it was aborted (and not yet sent) or some error @@ -376,23 +393,7 @@ void fuse_request_end(struct fuse_req *req) WARN_ON(test_bit(FR_SENT, &req->flags)); if (test_bit(FR_BACKGROUND, &req->flags)) { spin_lock(&fc->bg_lock); - clear_bit(FR_BACKGROUND, &req->flags); - if (fc->num_background == fc->max_background) { - fc->blocked = 0; - wake_up(&fc->blocked_waitq); - } else if (!fc->blocked) { - /* - * Wake up next waiter, if any. It's okay to use - * waitqueue_active(), as we've already synced up - * fc->blocked with waiters with the wake_up() call - * above. - */ - if (waitqueue_active(&fc->blocked_waitq)) - wake_up(&fc->blocked_waitq); - } - - fc->num_background--; - fc->active_background--; + fuse_request_bg_finish(fc, req); flush_bg_queue(fc); spin_unlock(&fc->bg_lock); } else { @@ -420,6 +421,24 @@ static int queue_interrupt(struct fuse_req *req) return 0; } +bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock) +{ + spin_lock(lock); + if (test_bit(FR_PENDING, &req->flags)) { + /* + * FR_PENDING does not get cleared as the request will end + * up in destruction anyway. + */ + list_del(&req->list); + spin_unlock(lock); + __fuse_put_request(req); + req->out.h.error = -EINTR; + return true; + } + spin_unlock(lock); + return false; +} + static void request_wait_answer(struct fuse_req *req) { struct fuse_conn *fc = req->fm->fc; @@ -441,22 +460,20 @@ static void request_wait_answer(struct fuse_req *req) } if (!test_bit(FR_FORCE, &req->flags)) { + bool removed; + /* Only fatal signals may interrupt this */ err = wait_event_killable(req->waitq, test_bit(FR_FINISHED, &req->flags)); if (!err) return; - spin_lock(&fiq->lock); - /* Request is not yet in userspace, bail out */ - if (test_bit(FR_PENDING, &req->flags)) { - list_del(&req->list); - spin_unlock(&fiq->lock); - __fuse_put_request(req); - req->out.h.error = -EINTR; + if (test_bit(FR_URING, &req->flags)) + removed = fuse_uring_remove_pending_req(req); + else + removed = fuse_remove_pending_req(req, &fiq->lock); + if (removed) return; - } - spin_unlock(&fiq->lock); } /* @@ -582,7 +599,25 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, return ret; } -static bool fuse_request_queue_background(struct fuse_req *req) +#ifdef CONFIG_FUSE_IO_URING +static bool fuse_request_queue_background_uring(struct fuse_conn *fc, + struct fuse_req *req) +{ + struct fuse_iqueue *fiq = &fc->iq; + + req->in.h.unique = fuse_get_unique(fiq); + req->in.h.len = sizeof(struct fuse_in_header) + + fuse_len_args(req->args->in_numargs, + (struct fuse_arg *) req->args->in_args); + + return fuse_uring_queue_bq_req(req); +} +#endif + +/* + * @return true if queued + */ +static int fuse_request_queue_background(struct fuse_req *req) { struct fuse_mount *fm = req->fm; struct fuse_conn *fc = fm->fc; @@ -594,6 +629,12 @@ static bool fuse_request_queue_background(struct fuse_req *req) atomic_inc(&fc->num_waiting); } __set_bit(FR_ISREPLY, &req->flags); + +#ifdef CONFIG_FUSE_IO_URING + if (fuse_uring_ready(fc)) + return fuse_request_queue_background_uring(fc, req); +#endif + spin_lock(&fc->bg_lock); if (likely(fc->connected)) { fc->num_background++; @@ -694,22 +735,8 @@ static int unlock_request(struct fuse_req *req) return err; } -struct fuse_copy_state { - int write; - struct fuse_req *req; - struct iov_iter *iter; - struct pipe_buffer *pipebufs; - struct pipe_buffer *currbuf; - struct pipe_inode_info *pipe; - unsigned long nr_segs; - struct page *pg; - unsigned len; - unsigned offset; - unsigned move_pages:1; -}; - -static void fuse_copy_init(struct fuse_copy_state *cs, int write, - struct iov_iter *iter) +void fuse_copy_init(struct fuse_copy_state *cs, int write, + struct iov_iter *iter) { memset(cs, 0, sizeof(*cs)); cs->write = write; @@ -717,7 +744,7 @@ static void fuse_copy_init(struct fuse_copy_state *cs, int write, } /* Unmap and put previous page of userspace buffer */ -static void fuse_copy_finish(struct fuse_copy_state *cs) +void fuse_copy_finish(struct fuse_copy_state *cs) { if (cs->currbuf) { struct pipe_buffer *buf = cs->currbuf; @@ -816,6 +843,9 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) *size -= ncpy; cs->len -= ncpy; cs->offset += ncpy; + if (cs->is_uring) + cs->ring.copied_sz += ncpy; + return ncpy; } @@ -1060,9 +1090,9 @@ static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) } /* Copy request arguments to/from userspace buffer */ -static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, - unsigned argpages, struct fuse_arg *args, - int zeroing) +int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, + unsigned argpages, struct fuse_arg *args, + int zeroing) { int err = 0; unsigned i; @@ -1751,7 +1781,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, args = &ap->args; args->nodeid = outarg->nodeid; args->opcode = FUSE_NOTIFY_REPLY; - args->in_numargs = 2; + args->in_numargs = 3; args->in_pages = true; args->end = fuse_retrieve_end; @@ -1778,9 +1808,10 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, } ra->inarg.offset = outarg->offset; ra->inarg.size = total_len; - args->in_args[0].size = sizeof(ra->inarg); - args->in_args[0].value = &ra->inarg; - args->in_args[1].size = total_len; + fuse_set_zero_arg0(args); + args->in_args[1].size = sizeof(ra->inarg); + args->in_args[1].value = &ra->inarg; + args->in_args[2].size = total_len; err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); if (err) @@ -1871,7 +1902,9 @@ static void fuse_resend(struct fuse_conn *fc) spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - end_requests(&to_queue); + list_for_each_entry(req, &to_queue, list) + clear_bit(FR_PENDING, &req->flags); + fuse_dev_end_requests(&to_queue); return; } /* @@ -1897,6 +1930,19 @@ static int fuse_notify_resend(struct fuse_conn *fc) return 0; } +/* + * Increments the fuse connection epoch. This will result of dentries from + * previous epochs to be invalidated. + * + * XXX optimization: add call to shrink_dcache_sb()? + */ +static int fuse_notify_inc_epoch(struct fuse_conn *fc) +{ + atomic_inc(&fc->epoch); + + return 0; +} + static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, unsigned int size, struct fuse_copy_state *cs) { @@ -1925,6 +1971,9 @@ static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, case FUSE_NOTIFY_RESEND: return fuse_notify_resend(fc); + case FUSE_NOTIFY_INC_EPOCH: + return fuse_notify_inc_epoch(fc); + default: fuse_copy_finish(cs); return -EINVAL; @@ -1932,7 +1981,7 @@ static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, } /* Look up request on processing list by unique ID */ -static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique) +struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique) { unsigned int hash = fuse_req_hash(unique); struct fuse_req *req; @@ -1944,10 +1993,17 @@ static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique) return NULL; } -static int copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, - unsigned nbytes) +int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, + unsigned nbytes) { - unsigned reqsize = sizeof(struct fuse_out_header); + + unsigned int reqsize = 0; + + /* + * Uring has all headers separated from args - args is payload only + */ + if (!cs->is_uring) + reqsize = sizeof(struct fuse_out_header); reqsize += fuse_len_args(args->out_numargs, args->out_args); @@ -1999,7 +2055,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, */ if (!oh.unique) { err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); - goto out; + goto copy_finish; } err = -EINVAL; @@ -2009,7 +2065,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, spin_lock(&fpq->lock); req = NULL; if (fpq->connected) - req = request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT); + req = fuse_request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT); err = -ENOENT; if (!req) { @@ -2047,7 +2103,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, if (oh.error) err = nbytes != sizeof(oh) ? -EINVAL : 0; else - err = copy_out_args(cs, req->args, nbytes); + err = fuse_copy_out_args(cs, req->args, nbytes); fuse_copy_finish(cs); spin_lock(&fpq->lock); @@ -2202,7 +2258,7 @@ static __poll_t fuse_dev_poll(struct file *file, poll_table *wait) } /* Abort all requests on the given list (pending or processing) */ -static void end_requests(struct list_head *head) +void fuse_dev_end_requests(struct list_head *head) { while (!list_empty(head)) { struct fuse_req *req; @@ -2305,7 +2361,13 @@ void fuse_abort_conn(struct fuse_conn *fc) wake_up_all(&fc->blocked_waitq); spin_unlock(&fc->lock); - end_requests(&to_end); + fuse_dev_end_requests(&to_end); + + /* + * fc->lock must not be taken to avoid conflicts with io-uring + * locks + */ + fuse_uring_abort(fc); } else { spin_unlock(&fc->lock); } @@ -2317,6 +2379,8 @@ void fuse_wait_aborted(struct fuse_conn *fc) /* matches implicit memory barrier in fuse_drop_waiting() */ smp_mb(); wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); + + fuse_uring_wait_stopped_queues(fc); } int fuse_dev_release(struct inode *inode, struct file *file) @@ -2335,7 +2399,7 @@ int fuse_dev_release(struct inode *inode, struct file *file) list_splice_init(&fpq->processing[i], &to_end); spin_unlock(&fpq->lock); - end_requests(&to_end); + fuse_dev_end_requests(&to_end); /* Are we the last open device? */ if (atomic_dec_and_test(&fc->dev_count)) { @@ -2463,6 +2527,17 @@ static long fuse_dev_ioctl(struct file *file, unsigned int cmd, } } +#ifdef CONFIG_PROC_FS +static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file) +{ + struct fuse_dev *fud = fuse_get_dev(file); + if (!fud) + return; + + seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev); +} +#endif + const struct file_operations fuse_dev_operations = { .owner = THIS_MODULE, .open = fuse_dev_open, @@ -2475,6 +2550,12 @@ const struct file_operations fuse_dev_operations = { .fasync = fuse_dev_fasync, .unlocked_ioctl = fuse_dev_ioctl, .compat_ioctl = compat_ptr_ioctl, +#ifdef CONFIG_FUSE_IO_URING + .uring_cmd = fuse_uring_cmd, +#endif +#ifdef CONFIG_PROC_FS + .show_fdinfo = fuse_dev_show_fdinfo, +#endif }; EXPORT_SYMBOL_GPL(fuse_dev_operations); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c new file mode 100644 index 0000000000000..d1774eeab4b79 --- /dev/null +++ b/fs/fuse/dev_uring.c @@ -0,0 +1,1385 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2024 DataDirect Networks. + */ + +#include "fuse_i.h" +#include "dev_uring_i.h" +#include "fuse_dev_i.h" + +#include +#include + +static bool __read_mostly enable_uring; +module_param(enable_uring, bool, 0644); +MODULE_PARM_DESC(enable_uring, + "Enable userspace communication through io-uring"); + +#define FUSE_URING_IOV_SEGS 2 /* header and payload */ + + +bool fuse_uring_enabled(void) +{ + return enable_uring; +} + +struct fuse_uring_pdu { + struct fuse_ring_ent *ent; +}; + +static const struct fuse_iqueue_ops fuse_io_uring_ops; + +static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, + struct fuse_ring_ent *ring_ent) +{ + struct fuse_uring_pdu *pdu = + io_uring_cmd_to_pdu(cmd, struct fuse_uring_pdu); + + pdu->ent = ring_ent; +} + +static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) +{ + struct fuse_uring_pdu *pdu = + io_uring_cmd_to_pdu(cmd, struct fuse_uring_pdu); + + return pdu->ent; +} + +static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_conn *fc = ring->fc; + + lockdep_assert_held(&queue->lock); + lockdep_assert_held(&fc->bg_lock); + + /* + * Allow one bg request per queue, ignoring global fc limits. + * This prevents a single queue from consuming all resources and + * eliminates the need for remote queue wake-ups when global + * limits are met but this queue has no more waiting requests. + */ + while ((fc->active_background < fc->max_background || + !queue->active_background) && + (!list_empty(&queue->fuse_req_bg_queue))) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_bg_queue, + struct fuse_req, list); + fc->active_background++; + queue->active_background++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, + int error) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + struct fuse_conn *fc = ring->fc; + + lockdep_assert_not_held(&queue->lock); + spin_lock(&queue->lock); + ent->fuse_req = NULL; + list_del_init(&req->list); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fc->bg_lock); + fuse_request_bg_finish(fc, req); + fuse_uring_flush_bg(queue); + spin_unlock(&fc->bg_lock); + } + + spin_unlock(&queue->lock); + + if (error) + req->out.h.error = error; + + clear_bit(FR_SENT, &req->flags); + fuse_request_end(req); +} + +/* Abort all list queued request on the given ring queue */ +static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) +{ + struct fuse_req *req; + LIST_HEAD(req_list); + + spin_lock(&queue->lock); + list_for_each_entry(req, &queue->fuse_req_queue, list) + clear_bit(FR_PENDING, &req->flags); + list_splice_init(&queue->fuse_req_queue, &req_list); + spin_unlock(&queue->lock); + + /* must not hold queue lock to avoid order issues with fi->lock */ + fuse_dev_end_requests(&req_list); +} + +void fuse_uring_abort_end_requests(struct fuse_ring *ring) +{ + int qid; + struct fuse_ring_queue *queue; + struct fuse_conn *fc = ring->fc; + + for (qid = 0; qid < ring->nr_queues; qid++) { + queue = READ_ONCE(ring->queues[qid]); + if (!queue) + continue; + + WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); + spin_lock(&queue->lock); + queue->stopped = true; + spin_lock(&fc->bg_lock); + fuse_uring_flush_bg(queue); + spin_unlock(&fc->bg_lock); + spin_unlock(&queue->lock); + fuse_uring_abort_end_queue_requests(queue); + } +} + +void fuse_uring_destruct(struct fuse_conn *fc) +{ + struct fuse_ring *ring = fc->ring; + int qid; + + if (!ring) + return; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); + struct fuse_ring_ent *ent, *next; + + if (!queue) + continue; + + WARN_ON(!list_empty(&queue->ent_avail_queue)); + WARN_ON(!list_empty(&queue->ent_w_req_queue)); + WARN_ON(!list_empty(&queue->ent_commit_queue)); + WARN_ON(!list_empty(&queue->ent_in_userspace)); + + list_for_each_entry_safe(ent, next, &queue->ent_released, + list) { + list_del_init(&ent->list); + kfree(ent); + } + + kfree(queue->fpq.processing); + kfree(queue); + WRITE_ONCE(ring->queues[qid], NULL); + } + + kfree(ring->queues); + kfree(ring); + fc->ring = NULL; +} + +/* + * Basic ring setup for this connection based on the provided configuration + */ +static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) +{ + struct fuse_ring *ring; + size_t nr_queues = num_possible_cpus(); + struct fuse_ring *res = NULL; + size_t max_payload_size; + + ring = kzalloc(sizeof(*fc->ring), GFP_KERNEL_ACCOUNT); + if (!ring) + return NULL; + + ring->queues = kcalloc(nr_queues, sizeof(struct fuse_ring_queue *), + GFP_KERNEL_ACCOUNT); + if (!ring->queues) + goto out_err; + + max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write); + max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE); + + spin_lock(&fc->lock); + if (!fc->connected) { + spin_unlock(&fc->lock); + goto out_err; + } + if (fc->ring) { + /* race, another thread created the ring in the meantime */ + spin_unlock(&fc->lock); + res = fc->ring; + goto out_err; + } + + init_waitqueue_head(&ring->stop_waitq); + + ring->nr_queues = nr_queues; + ring->fc = fc; + ring->max_payload_sz = max_payload_size; + smp_store_release(&fc->ring, ring); + + spin_unlock(&fc->lock); + return ring; + +out_err: + kfree(ring->queues); + kfree(ring); + return res; +} + +static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, + int qid) +{ + struct fuse_conn *fc = ring->fc; + struct fuse_ring_queue *queue; + struct list_head *pq; + + queue = kzalloc(sizeof(*queue), GFP_KERNEL_ACCOUNT); + if (!queue) + return NULL; + pq = kcalloc(FUSE_PQ_HASH_SIZE, sizeof(struct list_head), GFP_KERNEL); + if (!pq) { + kfree(queue); + return NULL; + } + + queue->qid = qid; + queue->ring = ring; + spin_lock_init(&queue->lock); + + INIT_LIST_HEAD(&queue->ent_avail_queue); + INIT_LIST_HEAD(&queue->ent_commit_queue); + INIT_LIST_HEAD(&queue->ent_w_req_queue); + INIT_LIST_HEAD(&queue->ent_in_userspace); + INIT_LIST_HEAD(&queue->fuse_req_queue); + INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->ent_released); + + queue->fpq.processing = pq; + fuse_pqueue_init(&queue->fpq); + + spin_lock(&fc->lock); + if (ring->queues[qid]) { + spin_unlock(&fc->lock); + kfree(queue->fpq.processing); + kfree(queue); + return ring->queues[qid]; + } + + /* + * fc->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set + */ + smp_store_release(&ring->queues[qid], queue); + spin_unlock(&fc->lock); + + return queue; +} + +static void fuse_uring_stop_fuse_req_end(struct fuse_req *req) +{ + clear_bit(FR_SENT, &req->flags); + req->out.h.error = -ECONNABORTED; + fuse_request_end(req); +} + +/* + * Release a request/entry on connection tear down + */ +static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) +{ + struct fuse_req *req; + struct io_uring_cmd *cmd; + + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + cmd = ent->cmd; + ent->cmd = NULL; + req = ent->fuse_req; + ent->fuse_req = NULL; + if (req) { + /* remove entry from queue->fpq->processing */ + list_del_init(&req->list); + } + + /* + * The entry must not be freed immediately, due to access of direct + * pointer access of entries through IO_URING_F_CANCEL - there is a risk + * of race between daemon termination (which triggers IO_URING_F_CANCEL + * and accesses entries without checking the list state first + */ + list_move(&ent->list, &queue->ent_released); + ent->state = FRRS_RELEASED; + spin_unlock(&queue->lock); + + if (cmd) + io_uring_cmd_done(cmd, -ENOTCONN, 0, IO_URING_F_UNLOCKED); + + if (req) + fuse_uring_stop_fuse_req_end(req); +} + +static void fuse_uring_stop_list_entries(struct list_head *head, + struct fuse_ring_queue *queue, + enum fuse_ring_req_state exp_state) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_ring_ent *ent, *next; + ssize_t queue_refs = SSIZE_MAX; + LIST_HEAD(to_teardown); + + spin_lock(&queue->lock); + list_for_each_entry_safe(ent, next, head, list) { + if (ent->state != exp_state) { + pr_warn("entry teardown qid=%d state=%d expected=%d", + queue->qid, ent->state, exp_state); + continue; + } + + ent->state = FRRS_TEARDOWN; + list_move(&ent->list, &to_teardown); + } + spin_unlock(&queue->lock); + + /* no queue lock to avoid lock order issues */ + list_for_each_entry_safe(ent, next, &to_teardown, list) { + fuse_uring_entry_teardown(ent); + queue_refs = atomic_dec_return(&ring->queue_refs); + WARN_ON_ONCE(queue_refs < 0); + } +} + +static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue) +{ + fuse_uring_stop_list_entries(&queue->ent_in_userspace, queue, + FRRS_USERSPACE); + fuse_uring_stop_list_entries(&queue->ent_avail_queue, queue, + FRRS_AVAILABLE); +} + +/* + * Log state debug info + */ +static void fuse_uring_log_ent_state(struct fuse_ring *ring) +{ + int qid; + struct fuse_ring_ent *ent; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); + + if (!queue) + continue; + + spin_lock(&queue->lock); + /* + * Log entries from the intermediate queue, the other queues + * should be empty + */ + list_for_each_entry(ent, &queue->ent_w_req_queue, list) { + pr_info(" ent-req-queue ring=%p qid=%d ent=%p state=%d\n", + ring, qid, ent, ent->state); + } + list_for_each_entry(ent, &queue->ent_commit_queue, list) { + pr_info(" ent-commit-queue ring=%p qid=%d ent=%p state=%d\n", + ring, qid, ent, ent->state); + } + spin_unlock(&queue->lock); + } + ring->stop_debug_log = 1; +} + +static void fuse_uring_async_stop_queues(struct work_struct *work) +{ + int qid; + struct fuse_ring *ring = + container_of(work, struct fuse_ring, async_teardown_work.work); + + /* XXX code dup */ + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); + + if (!queue) + continue; + + fuse_uring_teardown_entries(queue); + } + + /* + * Some ring entries might be in the middle of IO operations, + * i.e. in process to get handled by file_operations::uring_cmd + * or on the way to userspace - we could handle that with conditions in + * run time code, but easier/cleaner to have an async tear down handler + * If there are still queue references left + */ + if (atomic_read(&ring->queue_refs) > 0) { + if (time_after(jiffies, + ring->teardown_time + FUSE_URING_TEARDOWN_TIMEOUT)) + fuse_uring_log_ent_state(ring); + + schedule_delayed_work(&ring->async_teardown_work, + FUSE_URING_TEARDOWN_INTERVAL); + } else { + wake_up_all(&ring->stop_waitq); + fuse_conn_put(ring->fc); + } +} + +/* + * Stop the ring queues + */ +void fuse_uring_stop_queues(struct fuse_ring *ring) +{ + int qid; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); + + if (!queue) + continue; + + fuse_uring_teardown_entries(queue); + } + + if (atomic_read(&ring->queue_refs) > 0) { + fuse_conn_get(ring->fc); + ring->teardown_time = jiffies; + INIT_DELAYED_WORK(&ring->async_teardown_work, + fuse_uring_async_stop_queues); + schedule_delayed_work(&ring->async_teardown_work, + FUSE_URING_TEARDOWN_INTERVAL); + } else { + wake_up_all(&ring->stop_waitq); + } +} + +/* + * Handle IO_URING_F_CANCEL, typically should come on daemon termination. + * + * Releasing the last entry should trigger fuse_dev_release() if + * the daemon was terminated + */ +static void fuse_uring_cancel(struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + struct fuse_ring_queue *queue; + bool need_cmd_done = false; + + /* + * direct access on ent - it must not be destructed as long as + * IO_URING_F_CANCEL might come up + */ + queue = ent->queue; + spin_lock(&queue->lock); + if (ent->state == FRRS_AVAILABLE) { + list_del_init(&ent->list); + need_cmd_done = true; + ent->cmd = NULL; + } + spin_unlock(&queue->lock); + + if (need_cmd_done) { + /* no queue lock to avoid lock order issues */ + io_uring_cmd_done(cmd, -ENOTCONN, 0, issue_flags); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } +} + +static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags, + struct fuse_ring_ent *ring_ent) +{ + uring_cmd_set_ring_ent(cmd, ring_ent); + io_uring_cmd_mark_cancelable(cmd, issue_flags); +} + +/* + * Checks for errors and stores it into the request + */ +static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, + struct fuse_req *req, + struct fuse_conn *fc) +{ + int err; + + err = -EINVAL; + if (oh->unique == 0) { + /* Not supported through io-uring yet */ + pr_warn_once("notify through fuse-io-uring not supported\n"); + goto err; + } + + if (oh->error <= -ERESTARTSYS || oh->error > 0) + goto err; + + if (oh->error) { + err = oh->error; + goto err; + } + + err = -ENOENT; + if ((oh->unique & ~FUSE_INT_REQ_BIT) != req->in.h.unique) { + pr_warn_ratelimited("unique mismatch, expected: %llu got %llu\n", + req->in.h.unique, + oh->unique & ~FUSE_INT_REQ_BIT); + goto err; + } + + /* + * Is it an interrupt reply ID? + * XXX: Not supported through fuse-io-uring yet, it should not even + * find the request - should not happen. + */ + WARN_ON_ONCE(oh->unique & FUSE_INT_REQ_BIT); + + err = 0; +err: + return err; +} + +static int fuse_uring_copy_from_ring(struct fuse_ring *ring, + struct fuse_req *req, + struct fuse_ring_ent *ent) +{ + struct fuse_copy_state cs; + struct fuse_args *args = req->args; + struct iov_iter iter; + int err; + struct fuse_uring_ent_in_out ring_in_out; + + err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, + sizeof(ring_in_out)); + if (err) + return -EFAULT; + + err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, + &iter); + if (err) + return err; + + fuse_copy_init(&cs, 0, &iter); + cs.is_uring = 1; + cs.req = req; + + err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); + fuse_copy_finish(&cs); + return err; +} + + /* + * Copy data from the req to the ring buffer + */ +static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent *ent) +{ + struct fuse_copy_state cs; + struct fuse_args *args = req->args; + struct fuse_in_arg *in_args = args->in_args; + int num_args = args->in_numargs; + int err; + struct iov_iter iter; + struct fuse_uring_ent_in_out ent_in_out = { + .flags = 0, + .commit_id = req->in.h.unique, + }; + + err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } + + fuse_copy_init(&cs, 1, &iter); + cs.is_uring = 1; + cs.req = req; + + if (num_args > 0) { + /* + * Expectation is that the first argument is the per op header. + * Some op code have that as zero size. + */ + if (args->in_args[0].size > 0) { + err = copy_to_user(&ent->headers->op_in, in_args->value, + in_args->size); + if (err) { + pr_info_ratelimited( + "Copying the header failed.\n"); + return -EFAULT; + } + } + in_args++; + num_args--; + } + + /* copy the payload */ + err = fuse_copy_args(&cs, num_args, args->in_pages, + (struct fuse_arg *)in_args, 0); + fuse_copy_finish(&cs); + if (err) { + pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); + return err; + } + + ent_in_out.payload_sz = cs.ring.copied_sz; + err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out, + sizeof(ent_in_out)); + return err ? -EFAULT : 0; +} + +static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + struct fuse_in_header in_header; + int err; + + err = -EIO; + if (WARN_ON(ent->state != FRRS_FUSE_REQ)) { + pr_err("qid=%d ring-req=%p invalid state %d on send\n", + queue->qid, ent, ent->state); + return err; + } + + err = -EINVAL; + if (WARN_ON(req->in.h.unique == 0)) + return err; + + /* copy the request */ + err = fuse_uring_args_to_ring(ring, req, ent); + if (unlikely(err)) { + pr_info_ratelimited("Copy to ring failed: %d\n", err); + return err; + } + + /* copy fuse_in_header */ + in_header = req->in.h; + err = copy_to_user(&ent->headers->in_out, &in_header, + sizeof(in_header)); + if (err) { + err = -EFAULT; + return err; + } + + return 0; +} + +static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + int err; + + err = fuse_uring_copy_to_ring(ent, req); + if (!err) { + set_bit(FR_SENT, &req->flags); + } else { + /* + * Copying the request failed. Remove the entry from the + * ent_w_req_queue list and terminate the request + */ + spin_lock(&ent->queue->lock); + list_del_init(&ent->list); + ent->state = FRRS_INVALID; + spin_unlock(&ent->queue->lock); + + fuse_uring_req_end(ent, req, err); + } + + return err; +} + +/* Used to find the request on SQE commit */ +static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_pqueue *fpq = &queue->fpq; + unsigned int hash; + struct fuse_req *req = ent->fuse_req; + + req->ring_entry = ent; + hash = fuse_req_hash(req->in.h.unique); + list_move_tail(&req->list, &fpq->processing[hash]); +} + +/* + * Write data to the ring buffer and send the request to userspace, + * userspace will read it + * This is comparable with classical read(/dev/fuse) + */ +static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, + struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + int err; + struct io_uring_cmd *cmd; + + err = fuse_uring_prepare_send(ent, req); + if (err) + return err; + + spin_lock(&queue->lock); + cmd = ent->cmd; + ent->cmd = NULL; + ent->state = FRRS_USERSPACE; + list_move(&ent->list, &queue->ent_in_userspace); + fuse_uring_add_to_pq(ent); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, 0, 0, issue_flags); + return 0; +} + +/* + * Make a ring entry available for fuse_req assignment + */ +static void fuse_uring_ent_avail(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue) +{ + WARN_ON_ONCE(!ent->cmd); + list_move(&ent->list, &queue->ent_avail_queue); + ent->state = FRRS_AVAILABLE; +} + +/* + * Assign a fuse queue entry to the given entry + */ +static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + struct fuse_ring_queue *queue = ent->queue; + + lockdep_assert_held(&queue->lock); + + if (WARN_ON_ONCE(ent->state != FRRS_AVAILABLE && + ent->state != FRRS_COMMIT)) { + pr_warn("%s qid=%d state=%d\n", __func__, ent->queue->qid, + ent->state); + } + + clear_bit(FR_PENDING, &req->flags); + + /* Until fuse_uring_add_to_pq() the req is not attached to any list */ + list_del_init(&req->list); + + ent->fuse_req = req; + ent->state = FRRS_FUSE_REQ; + list_move(&ent->list, &queue->ent_w_req_queue); +} + +/* Fetch the next fuse request if available */ +static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) + __must_hold(&queue->lock) +{ + struct fuse_req *req; + struct fuse_ring_queue *queue = ent->queue; + struct list_head *req_queue = &queue->fuse_req_queue; + + lockdep_assert_held(&queue->lock); + + /* get and assign the next entry while it is still holding the lock */ + req = list_first_entry_or_null(req_queue, struct fuse_req, list); + if (req) + fuse_uring_add_req_to_ring_ent(ent, req); + + return req; +} + +/* + * Read data from the ring buffer, which user space has written to + * This is comparible with handling of classical write(/dev/fuse). + * Also make the ring request available again for new fuse requests. + */ +static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_ring *ring = ent->queue->ring; + struct fuse_conn *fc = ring->fc; + struct fuse_out_header out_header; + ssize_t err = -EFAULT; + + if (copy_from_user(&out_header, &ent->headers->in_out, + sizeof(out_header))) + goto out; + req->out.h = out_header; + + err = fuse_uring_out_header_has_err(&req->out.h, req, fc); + if (err) { + /* req->out.h.error already set */ + goto out; + } + + err = fuse_uring_copy_from_ring(ring, req, ent); +out: + fuse_uring_req_end(ent, req, err); +} + +/* + * Get the next fuse req and send it + */ +static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue, + unsigned int issue_flags) +{ + int err; + struct fuse_req *req; + +retry: + spin_lock(&queue->lock); + fuse_uring_ent_avail(ent, queue); + req = fuse_uring_ent_assign_req(ent); + spin_unlock(&queue->lock); + + if (req) { + err = fuse_uring_send_next_to_ring(ent, req, issue_flags); + if (err) + goto retry; + } +} + +static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + + lockdep_assert_held(&queue->lock); + + if (WARN_ON_ONCE(ent->state != FRRS_USERSPACE)) + return -EIO; + + ent->state = FRRS_COMMIT; + list_move(&ent->list, &queue->ent_commit_queue); + + return 0; +} + +/* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ +static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, + struct fuse_conn *fc) +{ + const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe_cmd(cmd->sqe); + struct fuse_ring_ent *ent; + int err; + struct fuse_ring *ring = fc->ring; + struct fuse_ring_queue *queue; + uint64_t commit_id = READ_ONCE(cmd_req->commit_id); + unsigned int qid = READ_ONCE(cmd_req->qid); + struct fuse_pqueue *fpq; + struct fuse_req *req; + + err = -ENOTCONN; + if (!ring) + return err; + + if (qid >= ring->nr_queues) + return -EINVAL; + + queue = READ_ONCE(ring->queues[qid]); + if (!queue) + return err; + fpq = &queue->fpq; + + if (!READ_ONCE(fc->connected)) + return err; + + spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return err; + } + + /* Find a request based on the unique ID of the fuse request + * This should get revised, as it needs a hash calculation and list + * search. And full struct fuse_pqueue is needed (memory overhead). + * As well as the link from req to ring_ent. + */ + req = fuse_request_find(fpq, commit_id); + err = -ENOENT; + if (!req) { + pr_info("qid=%d commit_id %llu not found\n", queue->qid, + commit_id); + spin_unlock(&queue->lock); + return err; + } + list_del_init(&req->list); + ent = req->ring_entry; + req->ring_entry = NULL; + + err = fuse_ring_ent_set_commit(ent); + if (err != 0) { + pr_info_ratelimited("qid=%d commit_id %llu state %d", + queue->qid, commit_id, ent->state); + spin_unlock(&queue->lock); + fuse_uring_req_end(ent, req, err); + return err; + } + + ent->cmd = cmd; + spin_unlock(&queue->lock); + + /* without the queue lock, as other locks are taken */ + fuse_uring_prepare_cancel(cmd, issue_flags, ent); + fuse_uring_commit(ent, req, issue_flags); + + /* + * Fetching the next request is absolutely required as queued + * fuse requests would otherwise not get processed - committing + * and fetching is done in one step vs legacy fuse, which has separated + * read (fetch request) and write (commit result). + */ + fuse_uring_next_fuse_req(ent, queue, issue_flags); + return 0; +} + +static bool is_ring_ready(struct fuse_ring *ring, int current_qid) +{ + int qid; + struct fuse_ring_queue *queue; + bool ready = true; + + for (qid = 0; qid < ring->nr_queues && ready; qid++) { + if (current_qid == qid) + continue; + + queue = READ_ONCE(ring->queues[qid]); + if (!queue) { + ready = false; + break; + } + + spin_lock(&queue->lock); + if (list_empty(&queue->ent_avail_queue)) + ready = false; + spin_unlock(&queue->lock); + } + + return ready; +} + +/* + * fuse_uring_req_fetch command handling + */ +static int fuse_uring_do_register(struct fuse_ring_ent *ent, + struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + struct fuse_conn *fc = ring->fc; + struct fuse_iqueue *fiq = &fc->iq; + + spin_lock(&fc->lock); + /* abort teardown path is running or has run */ + if (!fc->connected) { + spin_unlock(&fc->lock); + if (atomic_dec_and_test(&ring->queue_refs)) + wake_up_all(&ring->stop_waitq); + kfree(ent); + return -ECONNABORTED; + } + spin_unlock(&fc->lock); + + fuse_uring_prepare_cancel(cmd, issue_flags, ent); + + spin_lock(&queue->lock); + ent->cmd = cmd; + fuse_uring_ent_avail(ent, queue); + spin_unlock(&queue->lock); + + if (!READ_ONCE(ring->ready)) { + bool ready = is_ring_ready(ring, queue->qid); + + if (ready) { + WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); + smp_store_release(&ring->ready, true); + wake_up_all(&fc->blocked_waitq); + } + } + return 0; +} + +/* + * sqe->addr is a ptr to an iovec array, iov[0] has the headers, iov[1] + * the payload + */ +static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe, + struct iovec iov[FUSE_URING_IOV_SEGS]) +{ + struct iovec __user *uiov = u64_to_user_ptr(READ_ONCE(sqe->addr)); + struct iov_iter iter; + ssize_t ret; + + if (sqe->len != FUSE_URING_IOV_SEGS) + return -EINVAL; + + /* + * Direction for buffer access will actually be READ and WRITE, + * using write for the import should include READ access as well. + */ + ret = import_iovec(WRITE, uiov, FUSE_URING_IOV_SEGS, + FUSE_URING_IOV_SEGS, &iov, &iter); + if (ret < 0) + return ret; + + return 0; +} + +static struct fuse_ring_ent * +fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, + struct fuse_ring_queue *queue) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_ring_ent *ent; + size_t payload_size; + struct iovec iov[FUSE_URING_IOV_SEGS]; + int err; + + err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); + if (err) { + pr_info_ratelimited("Failed to get iovec from sqe, err=%d\n", + err); + return ERR_PTR(err); + } + + err = -EINVAL; + if (iov[0].iov_len < sizeof(struct fuse_uring_req_header)) { + pr_info_ratelimited("Invalid header len %zu\n", iov[0].iov_len); + return ERR_PTR(err); + } + + payload_size = iov[1].iov_len; + if (payload_size < ring->max_payload_sz) { + pr_info_ratelimited("Invalid req payload len %zu\n", + payload_size); + return ERR_PTR(err); + } + + err = -ENOMEM; + ent = kzalloc(sizeof(*ent), GFP_KERNEL_ACCOUNT); + if (!ent) + return ERR_PTR(err); + + INIT_LIST_HEAD(&ent->list); + + ent->queue = queue; + ent->headers = iov[0].iov_base; + ent->payload = iov[1].iov_base; + + atomic_inc(&ring->queue_refs); + return ent; +} + +/* + * Register header and payload buffer with the kernel and puts the + * entry as "ready to get fuse requests" on the queue + */ +static int fuse_uring_register(struct io_uring_cmd *cmd, + unsigned int issue_flags, struct fuse_conn *fc) +{ + const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe_cmd(cmd->sqe); + struct fuse_ring *ring = smp_load_acquire(&fc->ring); + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent; + int err; + unsigned int qid = READ_ONCE(cmd_req->qid); + + err = -ENOMEM; + if (!ring) { + ring = fuse_uring_create(fc); + if (!ring) + return err; + } + + if (qid >= ring->nr_queues) { + pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); + return -EINVAL; + } + + queue = READ_ONCE(ring->queues[qid]); + if (!queue) { + queue = fuse_uring_create_queue(ring, qid); + if (!queue) + return err; + } + + /* + * The created queue above does not need to be destructed in + * case of entry errors below, will be done at ring destruction time. + */ + + ent = fuse_uring_create_ring_ent(cmd, queue); + if (IS_ERR(ent)) + return PTR_ERR(ent); + + return fuse_uring_do_register(ent, cmd, issue_flags); +} + +/* + * Entry function from io_uring to handle the given passthrough command + * (op code IORING_OP_URING_CMD) + */ +int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) +{ + struct fuse_dev *fud; + struct fuse_conn *fc; + u32 cmd_op = cmd->cmd_op; + int err; + + if ((unlikely(issue_flags & IO_URING_F_CANCEL))) { + fuse_uring_cancel(cmd, issue_flags); + return 0; + } + + /* This extra SQE size holds struct fuse_uring_cmd_req */ + if (!(issue_flags & IO_URING_F_SQE128)) + return -EINVAL; + + fud = fuse_get_dev(cmd->file); + if (!fud) { + pr_info_ratelimited("No fuse device found\n"); + return -ENOTCONN; + } + fc = fud->fc; + + /* + * The ring is sized from values negotiated by FUSE_INIT + * + * Pairs with smp_wmb() in fuse_set_initialized() + */ + if (!smp_load_acquire(&fc->initialized)) + return -EAGAIN; + + if (fc->aborted) + return -ECONNABORTED; + if (!fc->connected) + return -ENOTCONN; + + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fc->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled by module parameter\n"); + return -EOPNOTSUPP; + } + + if (!fc->io_uring) { + pr_info_ratelimited( + "fuse-io-uring not enabled on this connection\n"); + return -EOPNOTSUPP; + } + + switch (cmd_op) { + case FUSE_IO_URING_CMD_REGISTER: + err = fuse_uring_register(cmd, issue_flags, fc); + if (err) { + pr_info_once("FUSE_IO_URING_CMD_REGISTER failed err=%d\n", + err); + fc->io_uring = 0; + wake_up_all(&fc->blocked_waitq); + return err; + } + break; + case FUSE_IO_URING_CMD_COMMIT_AND_FETCH: + err = fuse_uring_commit_fetch(cmd, issue_flags, fc); + if (err) { + pr_info_once("FUSE_IO_URING_COMMIT_AND_FETCH failed err=%d\n", + err); + return err; + } + break; + default: + return -EINVAL; + } + + return -EIOCBQUEUED; +} + +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + fuse_uring_add_to_pq(ent); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, ret, 0, issue_flags); +} + +/* + * This prepares and sends the ring request in fuse-uring task context. + * User buffers are not mapped yet - the application does not have permission + * to write to it - this has to be executed in ring task context. + */ +static void fuse_uring_send_in_task(struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + struct fuse_ring_queue *queue = ent->queue; + int err; + + if (!(issue_flags & IO_URING_F_TASK_DEAD)) { + err = fuse_uring_prepare_send(ent, ent->fuse_req); + if (err) { + fuse_uring_next_fuse_req(ent, queue, issue_flags); + return; + } + fuse_uring_send(ent, cmd, err, issue_flags); + } else { + err = -ECANCELED; + + spin_lock(&queue->lock); + list_del_init(&ent->list); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, err, 0, issue_flags); + + fuse_uring_req_end(ent, ent->fuse_req, err); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } +} + +static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) +{ + unsigned int qid; + struct fuse_ring_queue *queue; + + qid = task_cpu(current); + + if (WARN_ONCE(qid >= ring->nr_queues, + "Core number (%u) exceeds nr queues (%zu)\n", qid, + ring->nr_queues)) + qid = 0; + + queue = READ_ONCE(ring->queues[qid]); + WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); + + return queue; +} + +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent) +{ + struct io_uring_cmd *cmd = ent->cmd; + + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, fuse_uring_send_in_task); +} + +/* queue a fuse request and send it if a ring entry is available */ +void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + struct fuse_conn *fc = req->fm->fc; + struct fuse_ring *ring = fc->ring; + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent = NULL; + int err; + + err = -EINVAL; + queue = fuse_uring_task_to_queue(ring); + if (!queue) + goto err; + + if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + req->in.h.unique = fuse_get_unique(fiq); + + spin_lock(&queue->lock); + err = -ENOTCONN; + if (unlikely(queue->stopped)) + goto err_unlock; + + set_bit(FR_URING, &req->flags); + req->ring_queue = queue; + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + fuse_uring_add_req_to_ring_ent(ent, req); + else + list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); + + if (ent) + fuse_uring_dispatch_ent(ent); + + return; + +err_unlock: + spin_unlock(&queue->lock); +err: + req->out.h.error = err; + clear_bit(FR_PENDING, &req->flags); + fuse_request_end(req); +} + +bool fuse_uring_queue_bq_req(struct fuse_req *req) +{ + struct fuse_conn *fc = req->fm->fc; + struct fuse_ring *ring = fc->ring; + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent = NULL; + + queue = fuse_uring_task_to_queue(ring); + if (!queue) + return false; + + spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return false; + } + + set_bit(FR_URING, &req->flags); + req->ring_queue = queue; + list_add_tail(&req->list, &queue->fuse_req_bg_queue); + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + spin_lock(&fc->bg_lock); + fc->num_background++; + if (fc->num_background == fc->max_background) + fc->blocked = 1; + fuse_uring_flush_bg(queue); + spin_unlock(&fc->bg_lock); + + /* + * Due to bg_queue flush limits there might be other bg requests + * in the queue that need to be handled first. Or no further req + * might be available. + */ + req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, + list); + if (ent && req) { + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + + fuse_uring_dispatch_ent(ent); + } else { + spin_unlock(&queue->lock); + } + + return true; +} + +bool fuse_uring_remove_pending_req(struct fuse_req *req) +{ + struct fuse_ring_queue *queue = req->ring_queue; + + return fuse_remove_pending_req(req, &queue->lock); +} + +static const struct fuse_iqueue_ops fuse_io_uring_ops = { + /* should be send over io-uring as enhancement */ + .send_forget = fuse_dev_queue_forget, + + /* + * could be send over io-uring, but interrupts should be rare, + * no need to make the code complex + */ + .send_interrupt = fuse_dev_queue_interrupt, + .send_req = fuse_uring_queue_fuse_req, +}; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h new file mode 100644 index 0000000000000..89d96a06ddd4f --- /dev/null +++ b/fs/fuse/dev_uring_i.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2024 DataDirect Networks. + */ + +#ifndef _FS_FUSE_DEV_URING_I_H +#define _FS_FUSE_DEV_URING_I_H + +#include "fuse_i.h" + +#ifdef CONFIG_FUSE_IO_URING + +#define FUSE_URING_TEARDOWN_TIMEOUT (5 * HZ) +#define FUSE_URING_TEARDOWN_INTERVAL (HZ/20) + +enum fuse_ring_req_state { + FRRS_INVALID = 0, + + /* The ring entry received from userspace and it is being processed */ + FRRS_COMMIT, + + /* The ring entry is waiting for new fuse requests */ + FRRS_AVAILABLE, + + /* The ring entry got assigned a fuse req */ + FRRS_FUSE_REQ, + + /* The ring entry is in or on the way to user space */ + FRRS_USERSPACE, + + /* The ring entry is in teardown */ + FRRS_TEARDOWN, + + /* The ring entry is released, but not freed yet */ + FRRS_RELEASED, +}; + +/** A fuse ring entry, part of the ring queue */ +struct fuse_ring_ent { + /* userspace buffer */ + struct fuse_uring_req_header __user *headers; + void __user *payload; + + /* the ring queue that owns the request */ + struct fuse_ring_queue *queue; + + /* fields below are protected by queue->lock */ + + struct io_uring_cmd *cmd; + + struct list_head list; + + enum fuse_ring_req_state state; + + struct fuse_req *fuse_req; +}; + +struct fuse_ring_queue { + /* + * back pointer to the main fuse uring structure that holds this + * queue + */ + struct fuse_ring *ring; + + /* queue id, corresponds to the cpu core */ + unsigned int qid; + + /* + * queue lock, taken when any value in the queue changes _and_ also + * a ring entry state changes. + */ + spinlock_t lock; + + /* available ring entries (struct fuse_ring_ent) */ + struct list_head ent_avail_queue; + + /* + * entries in the process of being committed or in the process + * to be sent to userspace + */ + struct list_head ent_w_req_queue; + struct list_head ent_commit_queue; + + /* entries in userspace */ + struct list_head ent_in_userspace; + + /* entries that are released */ + struct list_head ent_released; + + /* fuse requests waiting for an entry slot */ + struct list_head fuse_req_queue; + + /* background fuse requests */ + struct list_head fuse_req_bg_queue; + + struct fuse_pqueue fpq; + + unsigned int active_background; + + bool stopped; +}; + +/** + * Describes if uring is for communication and holds alls the data needed + * for uring communication + */ +struct fuse_ring { + /* back pointer */ + struct fuse_conn *fc; + + /* number of ring queues */ + size_t nr_queues; + + /* maximum payload/arg size */ + size_t max_payload_sz; + + struct fuse_ring_queue **queues; + + /* + * Log ring entry states on stop when entries cannot be released + */ + unsigned int stop_debug_log : 1; + + wait_queue_head_t stop_waitq; + + /* async tear down */ + struct delayed_work async_teardown_work; + + /* log */ + unsigned long teardown_time; + + atomic_t queue_refs; + + bool ready; +}; + +bool fuse_uring_enabled(void); +void fuse_uring_destruct(struct fuse_conn *fc); +void fuse_uring_stop_queues(struct fuse_ring *ring); +void fuse_uring_abort_end_requests(struct fuse_ring *ring); +int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); +void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req); +bool fuse_uring_queue_bq_req(struct fuse_req *req); +bool fuse_uring_remove_pending_req(struct fuse_req *req); + +static inline void fuse_uring_abort(struct fuse_conn *fc) +{ + struct fuse_ring *ring = fc->ring; + + if (ring == NULL) + return; + + fuse_uring_abort_end_requests(ring); + + if (atomic_read(&ring->queue_refs) > 0) + fuse_uring_stop_queues(ring); +} + +static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) +{ + struct fuse_ring *ring = fc->ring; + + if (ring) + wait_event(ring->stop_waitq, + atomic_read(&ring->queue_refs) == 0); +} + +static inline bool fuse_uring_ready(struct fuse_conn *fc) +{ + struct fuse_ring *ring = READ_ONCE(fc->ring); + + return ring && smp_load_acquire(&ring->ready); +} + +#else /* CONFIG_FUSE_IO_URING */ + +static inline void fuse_uring_destruct(struct fuse_conn *fc) +{ +} + +static inline bool fuse_uring_enabled(void) +{ + return false; +} + +static inline void fuse_uring_abort(struct fuse_conn *fc) +{ +} + +static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) +{ +} + +static inline bool fuse_uring_ready(struct fuse_conn *fc) +{ + return false; +} + +static inline bool fuse_uring_remove_pending_req(struct fuse_req *req) +{ + return false; +} + +#endif /* CONFIG_FUSE_IO_URING */ + +#endif /* _FS_FUSE_DEV_URING_I_H */ diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 54104dd48af7c..be3a6720fcba1 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -175,9 +175,10 @@ static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args, memset(outarg, 0, sizeof(struct fuse_entry_out)); args->opcode = FUSE_LOOKUP; args->nodeid = nodeid; - args->in_numargs = 1; - args->in_args[0].size = name->len + 1; - args->in_args[0].value = name->name; + args->in_numargs = 2; + fuse_set_zero_arg0(args); + args->in_args[1].size = name->len + 1; + args->in_args[1].value = name->name; args->out_numargs = 1; args->out_args[0].size = sizeof(struct fuse_entry_out); args->out_args[0].value = outarg; @@ -197,9 +198,14 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) struct inode *inode; struct dentry *parent; struct fuse_mount *fm; + struct fuse_conn *fc; struct fuse_inode *fi; int ret; + fc = get_fuse_conn_super(entry->d_sb); + if (entry->d_time < atomic_read(&fc->epoch)) + goto invalid; + inode = d_inode_rcu(entry); if (inode && fuse_is_bad(inode)) goto invalid; @@ -415,16 +421,20 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, unsigned int flags) { - int err; struct fuse_entry_out outarg; + struct fuse_conn *fc; struct inode *inode; struct dentry *newent; + int err, epoch; bool outarg_valid = true; bool locked; if (fuse_is_bad(dir)) return ERR_PTR(-EIO); + fc = get_fuse_conn_super(dir->i_sb); + epoch = atomic_read(&fc->epoch); + locked = fuse_lock_inode(dir); err = fuse_lookup_name(dir->i_sb, get_node_id(dir), &entry->d_name, &outarg, &inode); @@ -446,6 +456,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; + entry->d_time = epoch; if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -618,7 +629,6 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, struct dentry *entry, struct file *file, unsigned int flags, umode_t mode, u32 opcode) { - int err; struct inode *inode; struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); @@ -628,11 +638,13 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, struct fuse_entry_out outentry; struct fuse_inode *fi; struct fuse_file *ff; + int epoch, err; bool trunc = flags & O_TRUNC; /* Userspace expects S_IFREG in create mode */ BUG_ON((mode & S_IFMT) != S_IFREG); + epoch = atomic_read(&fm->fc->epoch); forget = fuse_alloc_forget(); err = -ENOMEM; if (!forget) @@ -701,6 +713,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); + entry->d_time = epoch; fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -787,12 +800,14 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, struct fuse_entry_out outarg; struct inode *inode; struct dentry *d; - int err; struct fuse_forget_link *forget; + int epoch, err; if (fuse_is_bad(dir)) return -EIO; + epoch = atomic_read(&fm->fc->epoch); + forget = fuse_alloc_forget(); if (!forget) return -ENOMEM; @@ -835,9 +850,11 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, return PTR_ERR(d); if (d) { + d->d_time = epoch; fuse_change_entry_timeout(d, &outarg); dput(d); } else { + entry->d_time = epoch; fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); @@ -927,11 +944,12 @@ static int fuse_symlink(struct mnt_idmap *idmap, struct inode *dir, FUSE_ARGS(args); args.opcode = FUSE_SYMLINK; - args.in_numargs = 2; - args.in_args[0].size = entry->d_name.len + 1; - args.in_args[0].value = entry->d_name.name; - args.in_args[1].size = len; - args.in_args[1].value = link; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + args.in_args[2].size = len; + args.in_args[2].value = link; return create_new_entry(idmap, fm, &args, dir, entry, S_IFLNK); } @@ -991,9 +1009,10 @@ static int fuse_unlink(struct inode *dir, struct dentry *entry) args.opcode = FUSE_UNLINK; args.nodeid = get_node_id(dir); - args.in_numargs = 1; - args.in_args[0].size = entry->d_name.len + 1; - args.in_args[0].value = entry->d_name.name; + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; err = fuse_simple_request(fm, &args); if (!err) { fuse_dir_changed(dir); @@ -1014,9 +1033,10 @@ static int fuse_rmdir(struct inode *dir, struct dentry *entry) args.opcode = FUSE_RMDIR; args.nodeid = get_node_id(dir); - args.in_numargs = 1; - args.in_args[0].size = entry->d_name.len + 1; - args.in_args[0].value = entry->d_name.name; + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; err = fuse_simple_request(fm, &args); if (!err) { fuse_dir_changed(dir); @@ -1632,7 +1652,7 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, goto out_err; if (fc->cache_symlinks) - return page_get_link(dentry, inode, callback); + return page_get_link_raw(dentry, inode, callback); err = -ECHILD; if (!dentry) @@ -1680,6 +1700,8 @@ static int fuse_dir_open(struct inode *inode, struct file *file) */ if (ff->open_flags & (FOPEN_STREAM | FOPEN_NONSEEKABLE)) nonseekable_open(inode, file); + if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); } return err; diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h new file mode 100644 index 0000000000000..734606ade0acb --- /dev/null +++ b/fs/fuse/fuse_dev_i.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * FUSE: Filesystem in Userspace + * Copyright (C) 2001-2008 Miklos Szeredi + */ +#ifndef _FS_FUSE_DEV_I_H +#define _FS_FUSE_DEV_I_H + +#include + +/* Ordinary requests have even IDs, while interrupts IDs are odd */ +#define FUSE_INT_REQ_BIT (1ULL << 0) +#define FUSE_REQ_ID_STEP (1ULL << 1) + +struct fuse_arg; +struct fuse_args; +struct fuse_pqueue; +struct fuse_req; +struct fuse_iqueue; +struct fuse_forget_link; + +struct fuse_copy_state { + int write; + struct fuse_req *req; + struct iov_iter *iter; + struct pipe_buffer *pipebufs; + struct pipe_buffer *currbuf; + struct pipe_inode_info *pipe; + unsigned long nr_segs; + struct page *pg; + unsigned int len; + unsigned int offset; + unsigned int move_pages:1; + unsigned int is_uring:1; + struct { + unsigned int copied_sz; /* copied size into the user buffer */ + } ring; +}; + +static inline struct fuse_dev *fuse_get_dev(struct file *file) +{ + /* + * Lockless access is OK, because file->private data is set + * once during mount and is valid until the file is released. + */ + return READ_ONCE(file->private_data); +} + +unsigned int fuse_req_hash(u64 unique); +struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique); + +void fuse_dev_end_requests(struct list_head *head); +void fuse_request_bg_finish(struct fuse_conn *fc, struct fuse_req *req); + +void fuse_copy_init(struct fuse_copy_state *cs, int write, + struct iov_iter *iter); +void fuse_copy_finish(struct fuse_copy_state *cs); +int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs, + unsigned int argpages, struct fuse_arg *args, + int zeroing); +int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, + unsigned int nbytes); +void fuse_dev_queue_forget(struct fuse_iqueue *fiq, + struct fuse_forget_link *forget); +void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req); +bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock); + +#endif + diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e6cc3d552b138..7cd634f497411 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -309,7 +309,7 @@ struct fuse_args { bool may_block:1; bool is_ext:1; bool is_pinned:1; - struct fuse_in_arg in_args[3]; + struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); }; @@ -375,6 +375,7 @@ struct fuse_io_priv { * FR_FINISHED: request is finished * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous + * FR_URING: request is handled through fuse-io-uring */ enum fuse_req_flag { FR_ISREPLY, @@ -389,6 +390,7 @@ enum fuse_req_flag { FR_FINISHED, FR_PRIVATE, FR_ASYNC, + FR_URING, }; /** @@ -435,6 +437,11 @@ struct fuse_req { /** fuse_mount this request belongs to */ struct fuse_mount *fm; + +#ifdef CONFIG_FUSE_IO_URING + void *ring_entry; + void *ring_queue; +#endif }; struct fuse_iqueue; @@ -604,6 +611,9 @@ struct fuse_conn { /** Number of fuse_dev's */ atomic_t dev_count; + /** Current epoch for up-to-date dentries */ + atomic_t epoch; + struct rcu_head rcu; /** The user id for this mount */ @@ -857,6 +867,9 @@ struct fuse_conn { /** Passthrough support for read/write IO */ unsigned int passthrough:1; + /* Use io_uring for communication */ + unsigned int io_uring; + /** Maximum stack depth for passthrough backing files */ int max_stack_depth; @@ -914,6 +927,11 @@ struct fuse_conn { /** IDR for backing files ids */ struct idr backing_files_map; #endif + +#ifdef CONFIG_FUSE_IO_URING + /** uring connection information*/ + struct fuse_ring *ring; +#endif }; /* @@ -938,6 +956,19 @@ struct fuse_mount { struct rcu_head rcu; }; +/* + * Empty header for FUSE opcodes without specific header needs. + * Used as a placeholder in args->in_args[0] for consistency + * across all FUSE operations, simplifying request handling. + */ +struct fuse_zero_header {}; + +static inline void fuse_set_zero_arg0(struct fuse_args *args) +{ + args->in_args[0].size = sizeof(struct fuse_zero_header); + args->in_args[0].value = NULL; +} + static inline struct fuse_mount *get_fuse_mount_super(struct super_block *sb) { return sb->s_fs_info; @@ -1203,6 +1234,11 @@ void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); */ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc); +/** + * Initialize the fuse processing queue + */ +void fuse_pqueue_init(struct fuse_pqueue *fpq); + /** * Initialize fuse_conn */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index fd3321e29a3e5..8df3a89453854 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -7,6 +7,7 @@ */ #include "fuse_i.h" +#include "dev_uring_i.h" #include #include @@ -905,7 +906,7 @@ static void fuse_iqueue_init(struct fuse_iqueue *fiq, fiq->priv = priv; } -static void fuse_pqueue_init(struct fuse_pqueue *fpq) +void fuse_pqueue_init(struct fuse_pqueue *fpq) { unsigned int i; @@ -926,6 +927,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, init_rwsem(&fc->killsb); refcount_set(&fc->count, 1); atomic_set(&fc->dev_count, 1); + atomic_set(&fc->epoch, 1); init_waitqueue_head(&fc->blocked_waitq); fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv); INIT_LIST_HEAD(&fc->bg_queue); @@ -959,6 +961,8 @@ static void delayed_release(struct rcu_head *p) { struct fuse_conn *fc = container_of(p, struct fuse_conn, rcu); + fuse_uring_destruct(fc); + put_user_ns(fc->user_ns); fc->release(fc); } @@ -1354,6 +1358,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, else ok = false; } + if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) + fc->io_uring = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1413,6 +1419,9 @@ void fuse_send_init(struct fuse_mount *fm) if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) flags |= FUSE_PASSTHROUGH; + if (fuse_uring_enabled()) + flags |= FUSE_OVER_IO_URING; + ia->in.flags = flags; ia->in.flags2 = flags >> 32; diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0377b6dc24c80..1987cdbeeb075 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -161,6 +161,7 @@ static int fuse_direntplus_link(struct file *file, struct fuse_conn *fc; struct inode *inode; DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); + int epoch; if (!o->nodeid) { /* @@ -190,6 +191,7 @@ static int fuse_direntplus_link(struct file *file, return -EIO; fc = get_fuse_conn(dir); + epoch = atomic_read(&fc->epoch); name.hash = full_name_hash(parent, name.name, name.len); dentry = d_lookup(parent, &name); @@ -256,6 +258,7 @@ static int fuse_direntplus_link(struct file *file, } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); + dentry->d_time = epoch; fuse_change_entry_timeout(dentry, o); dput(dentry); diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c index 9f568d345c512..93dfb06b6cea0 100644 --- a/fs/fuse/xattr.c +++ b/fs/fuse/xattr.c @@ -164,9 +164,10 @@ int fuse_removexattr(struct inode *inode, const char *name) args.opcode = FUSE_REMOVEXATTR; args.nodeid = get_node_id(inode); - args.in_numargs = 1; - args.in_args[0].size = strlen(name) + 1; - args.in_args[0].value = name; + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = strlen(name) + 1; + args.in_args[1].value = name; err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { fm->fc->no_removexattr = 1; diff --git a/fs/namei.c b/fs/namei.c index 858592582449b..3e961dd22da8b 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -5334,10 +5334,9 @@ const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done) EXPORT_SYMBOL(vfs_get_link); /* get the link contents into pagecache */ -const char *page_get_link(struct dentry *dentry, struct inode *inode, - struct delayed_call *callback) +static char *__page_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) { - char *kaddr; struct page *page; struct address_space *mapping = inode->i_mapping; @@ -5356,8 +5355,23 @@ const char *page_get_link(struct dentry *dentry, struct inode *inode, } set_delayed_call(callback, page_put_link, page); BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM); - kaddr = page_address(page); - nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1); + return page_address(page); +} + +const char *page_get_link_raw(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + return __page_get_link(dentry, inode, callback); +} +EXPORT_SYMBOL_GPL(page_get_link_raw); + +const char *page_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + char *kaddr = __page_get_link(dentry, inode, callback); + + if (!IS_ERR(kaddr)) + nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1); return kaddr; } diff --git a/include/linux/crypto.h b/include/linux/crypto.h index cd78d16ee5d65..8c7f5331ba4fa 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -133,7 +133,8 @@ #define CRYPTO_TFM_REQ_FORBID_WEAK_KEYS 0x00000100 #define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200 #define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400 -#define CRYPTO_TFM_REQ_NEED_RESEED 0x00000800 +#define CRYPTO_TFM_REQ_ON_STACK 0x00000800 +#define CRYPTO_TFM_REQ_NEED_RESEED 0x00001000 #define CRYPTO_TFM_FIPS_COMPLIANCE 0x80000000 diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 69782344e7670..d0d64d13cdbc3 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -256,6 +256,8 @@ extern void d_tmpfile(struct file *, struct inode *); extern struct dentry *d_find_alias(struct inode *); extern void d_prune_aliases(struct inode *); +extern void d_dispose_if_unused(struct dentry *, struct list_head *); +extern void shrink_dentry_list(struct list_head *); extern struct dentry *d_find_alias_rcu(struct inode *); diff --git a/include/linux/fs.h b/include/linux/fs.h index 96a3b653d1fdf..5a74045e8a94b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3404,6 +3404,8 @@ extern const struct file_operations generic_ro_fops; extern int readlink_copy(char __user *, int, const char *, int); extern int page_readlink(struct dentry *, char __user *, int); +extern const char *page_get_link_raw(struct dentry *, struct inode *, + struct delayed_call *); extern const char *page_get_link(struct dentry *, struct inode *, struct delayed_call *); extern void page_put_link(void *); diff --git a/include/linux/local_lock.h b/include/linux/local_lock.h index 16a2ee4f8310b..2ba8464195244 100644 --- a/include/linux/local_lock.h +++ b/include/linux/local_lock.h @@ -13,13 +13,13 @@ * local_lock - Acquire a per CPU local lock * @lock: The lock variable */ -#define local_lock(lock) __local_lock(lock) +#define local_lock(lock) __local_lock(this_cpu_ptr(lock)) /** * local_lock_irq - Acquire a per CPU local lock and disable interrupts * @lock: The lock variable */ -#define local_lock_irq(lock) __local_lock_irq(lock) +#define local_lock_irq(lock) __local_lock_irq(this_cpu_ptr(lock)) /** * local_lock_irqsave - Acquire a per CPU local lock, save and disable @@ -28,19 +28,19 @@ * @flags: Storage for interrupt flags */ #define local_lock_irqsave(lock, flags) \ - __local_lock_irqsave(lock, flags) + __local_lock_irqsave(this_cpu_ptr(lock), flags) /** * local_unlock - Release a per CPU local lock * @lock: The lock variable */ -#define local_unlock(lock) __local_unlock(lock) +#define local_unlock(lock) __local_unlock(this_cpu_ptr(lock)) /** * local_unlock_irq - Release a per CPU local lock and enable interrupts * @lock: The lock variable */ -#define local_unlock_irq(lock) __local_unlock_irq(lock) +#define local_unlock_irq(lock) __local_unlock_irq(this_cpu_ptr(lock)) /** * local_unlock_irqrestore - Release a per CPU local lock and restore @@ -49,7 +49,7 @@ * @flags: Interrupt flags to restore */ #define local_unlock_irqrestore(lock, flags) \ - __local_unlock_irqrestore(lock, flags) + __local_unlock_irqrestore(this_cpu_ptr(lock), flags) /** * local_lock_init - Runtime initialize a lock instance @@ -64,7 +64,7 @@ * locking constrains it will _always_ fail to acquire the lock in NMI or * HARDIRQ context on PREEMPT_RT. */ -#define local_trylock(lock) __local_trylock(lock) +#define local_trylock(lock) __local_trylock(this_cpu_ptr(lock)) /** * local_trylock_irqsave - Try to acquire a per CPU local lock, save and disable @@ -77,7 +77,7 @@ * HARDIRQ context on PREEMPT_RT. */ #define local_trylock_irqsave(lock, flags) \ - __local_trylock_irqsave(lock, flags) + __local_trylock_irqsave(this_cpu_ptr(lock), flags) DEFINE_GUARD(local_lock, local_lock_t __percpu*, local_lock(_T), @@ -91,10 +91,10 @@ DEFINE_LOCK_GUARD_1(local_lock_irqsave, local_lock_t __percpu, unsigned long flags) #define local_lock_nested_bh(_lock) \ - __local_lock_nested_bh(_lock) + __local_lock_nested_bh(this_cpu_ptr(_lock)) #define local_unlock_nested_bh(_lock) \ - __local_unlock_nested_bh(_lock) + __local_unlock_nested_bh(this_cpu_ptr(_lock)) DEFINE_GUARD(local_lock_nested_bh, local_lock_t __percpu*, local_lock_nested_bh(_T), diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 8d5ac16a9b179..d80b5306a2c0c 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -99,14 +99,14 @@ do { \ local_trylock_t *tl; \ local_lock_t *l; \ \ - l = (local_lock_t *)this_cpu_ptr(lock); \ + l = (local_lock_t *)(lock); \ tl = (local_trylock_t *)l; \ _Generic((lock), \ - __percpu local_trylock_t *: ({ \ + local_trylock_t *: ({ \ lockdep_assert(tl->acquired == 0); \ WRITE_ONCE(tl->acquired, 1); \ }), \ - __percpu local_lock_t *: (void)0); \ + local_lock_t *: (void)0); \ local_lock_acquire(l); \ } while (0) @@ -133,7 +133,7 @@ do { \ local_trylock_t *tl; \ \ preempt_disable(); \ - tl = this_cpu_ptr(lock); \ + tl = (lock); \ if (READ_ONCE(tl->acquired)) { \ preempt_enable(); \ tl = NULL; \ @@ -150,7 +150,7 @@ do { \ local_trylock_t *tl; \ \ local_irq_save(flags); \ - tl = this_cpu_ptr(lock); \ + tl = (lock); \ if (READ_ONCE(tl->acquired)) { \ local_irq_restore(flags); \ tl = NULL; \ @@ -167,15 +167,15 @@ do { \ local_trylock_t *tl; \ local_lock_t *l; \ \ - l = (local_lock_t *)this_cpu_ptr(lock); \ + l = (local_lock_t *)(lock); \ tl = (local_trylock_t *)l; \ local_lock_release(l); \ _Generic((lock), \ - __percpu local_trylock_t *: ({ \ + local_trylock_t *: ({ \ lockdep_assert(tl->acquired == 1); \ WRITE_ONCE(tl->acquired, 0); \ }), \ - __percpu local_lock_t *: (void)0); \ + local_lock_t *: (void)0); \ } while (0) #define __local_unlock(lock) \ @@ -199,11 +199,11 @@ do { \ #define __local_lock_nested_bh(lock) \ do { \ lockdep_assert_in_softirq(); \ - local_lock_acquire(this_cpu_ptr(lock)); \ + local_lock_acquire((lock)); \ } while (0) #define __local_unlock_nested_bh(lock) \ - local_lock_release(this_cpu_ptr(lock)) + local_lock_release((lock)) #else /* !CONFIG_PREEMPT_RT */ @@ -227,7 +227,7 @@ typedef spinlock_t local_trylock_t; #define __local_lock(__lock) \ do { \ migrate_disable(); \ - spin_lock(this_cpu_ptr((__lock))); \ + spin_lock((__lock)); \ } while (0) #define __local_lock_irq(lock) __local_lock(lock) @@ -241,7 +241,7 @@ typedef spinlock_t local_trylock_t; #define __local_unlock(__lock) \ do { \ - spin_unlock(this_cpu_ptr((__lock))); \ + spin_unlock((__lock)); \ migrate_enable(); \ } while (0) @@ -252,12 +252,12 @@ typedef spinlock_t local_trylock_t; #define __local_lock_nested_bh(lock) \ do { \ lockdep_assert_in_softirq_func(); \ - spin_lock(this_cpu_ptr(lock)); \ + spin_lock((lock)); \ } while (0) #define __local_unlock_nested_bh(lock) \ do { \ - spin_unlock(this_cpu_ptr((lock))); \ + spin_unlock((lock)); \ } while (0) #define __local_trylock(lock) \ @@ -268,7 +268,7 @@ do { \ __locked = 0; \ } else { \ migrate_disable(); \ - __locked = spin_trylock(this_cpu_ptr((lock))); \ + __locked = spin_trylock((lock)); \ if (!__locked) \ migrate_enable(); \ } \ diff --git a/include/linux/random.h b/include/linux/random.h index 8a52424fd0d50..53b79b68a9546 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -9,13 +9,6 @@ #include -struct iov_iter; - -struct random_extrng { - ssize_t (*extrng_read_iter)(struct iov_iter *iter, bool reseed); - struct module *owner; -}; - struct notifier_block; void add_device_randomness(const void *buf, size_t len); @@ -164,11 +157,15 @@ int random_prepare_cpu(unsigned int cpu); int random_online_cpu(unsigned int cpu); #endif -void random_register_extrng(const struct random_extrng *rng); -void random_unregister_extrng(void); - #ifndef MODULE extern const struct file_operations random_fops, urandom_fops; + +struct iov_iter; +struct random_extrng { + ssize_t (*extrng_read_iter)(struct iov_iter *iter, bool reseed); +}; + +void __init random_register_extrng(const struct random_extrng *rng); #endif #endif /* _LINUX_RANDOM_H */ diff --git a/include/net/ip.h b/include/net/ip.h index 7e7db511c9630..01c7ed96585ee 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -74,6 +74,13 @@ static inline unsigned int ip_hdrlen(const struct sk_buff *skb) return ip_hdr(skb)->ihl * 4; } +static inline int ip_check_csum_start(const struct sk_buff *skb) +{ + if (unlikely(skb->csum_start < skb->network_header + ip_hdrlen(skb))) + return -EINVAL; + return 0; +} + struct ipcm_cookie { struct sockcm_cookie sockc; __be32 addr; diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 521a86ae64855..9e3d782342f6a 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -319,7 +319,8 @@ struct sctp_cookie { __u8 auth_random[sizeof(struct sctp_paramhdr) + SCTP_AUTH_RANDOM_LENGTH]; - __u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2]; + __u8 auth_hmacs[sizeof(struct sctp_paramhdr) + + SCTP_AUTH_NUM_HMACS * sizeof(__u16)]; __u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS]; /* This is a shim for my peer's INIT packet, followed by diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index f1e99458e29e4..914aa300bcc1a 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -220,6 +220,19 @@ * * 7.41 * - add FUSE_ALLOW_IDMAP + * + * 7.42 + * - Add FUSE_OVER_IO_URING and all other io-uring related flags and data + * structures: + * - struct fuse_uring_ent_in_out + * - struct fuse_uring_req_header + * - struct fuse_uring_cmd_req + * - FUSE_URING_IN_OUT_HEADER_SZ + * - FUSE_URING_OP_IN_OUT_SZ + * - enum fuse_uring_cmd + * + * 7.44 + * - add FUSE_NOTIFY_INC_EPOCH */ #ifndef _LINUX_FUSE_H @@ -255,7 +268,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 41 +#define FUSE_KERNEL_MINOR_VERSION 44 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -425,6 +438,7 @@ struct fuse_file_lock { * FUSE_HAS_RESEND: kernel supports resending pending requests, and the high bit * of the request ID indicates resend requests * FUSE_ALLOW_IDMAP: allow creation of idmapped mounts + * FUSE_OVER_IO_URING: Indicate that client supports io-uring */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -471,6 +485,7 @@ struct fuse_file_lock { /* Obsolete alias for FUSE_DIRECT_IO_ALLOW_MMAP */ #define FUSE_DIRECT_IO_RELAX FUSE_DIRECT_IO_ALLOW_MMAP #define FUSE_ALLOW_IDMAP (1ULL << 40) +#define FUSE_OVER_IO_URING (1ULL << 41) /** * CUSE INIT request/reply flags @@ -655,6 +670,7 @@ enum fuse_notify_code { FUSE_NOTIFY_RETRIEVE = 5, FUSE_NOTIFY_DELETE = 6, FUSE_NOTIFY_RESEND = 7, + FUSE_NOTIFY_INC_EPOCH = 8, FUSE_NOTIFY_CODE_MAX, }; @@ -1206,4 +1222,67 @@ struct fuse_supp_groups { uint32_t groups[]; }; +/** + * Size of the ring buffer header + */ +#define FUSE_URING_IN_OUT_HEADER_SZ 128 +#define FUSE_URING_OP_IN_OUT_SZ 128 + +/* Used as part of the fuse_uring_req_header */ +struct fuse_uring_ent_in_out { + uint64_t flags; + + /* + * commit ID to be used in a reply to a ring request (see also + * struct fuse_uring_cmd_req) + */ + uint64_t commit_id; + + /* size of user payload buffer */ + uint32_t payload_sz; + uint32_t padding; + + uint64_t reserved; +}; + +/** + * Header for all fuse-io-uring requests + */ +struct fuse_uring_req_header { + /* struct fuse_in_header / struct fuse_out_header */ + char in_out[FUSE_URING_IN_OUT_HEADER_SZ]; + + /* per op code header */ + char op_in[FUSE_URING_OP_IN_OUT_SZ]; + + struct fuse_uring_ent_in_out ring_ent_in_out; +}; + +/** + * sqe commands to the kernel + */ +enum fuse_uring_cmd { + FUSE_IO_URING_CMD_INVALID = 0, + + /* register the request buffer and fetch a fuse request */ + FUSE_IO_URING_CMD_REGISTER = 1, + + /* commit fuse request result and fetch next request */ + FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, +}; + +/** + * In the 80B command area of the SQE. + */ +struct fuse_uring_cmd_req { + uint64_t flags; + + /* entry identifier for commits */ + uint64_t commit_id; + + /* queue the command is for (queue index) */ + uint16_t qid; + uint8_t padding[6]; +}; + #endif /* _LINUX_FUSE_H */ diff --git a/kernel/exit.c b/kernel/exit.c index 1ab844a573474..8fe1ce3fb0ea5 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -280,12 +280,13 @@ void release_task(struct task_struct *p) put_pid(thread_pid); release_thread(p); /* - * This task was already removed from the process/thread/pid lists - * and lock_task_sighand(p) can't succeed. Nobody else can touch - * ->pending or, if group dead, signal->shared_pending. We can call - * flush_sigqueue() lockless. + * This task was already removed from the process/thread/pid lists and + * lock_task_sighand(p) can't succeed. If it's the group leader then + * flush tsk->signal->shared_pending. tsk->pending has been flushed + * already in exit_signals(). Nothing else can touch + * signal->shared_pending anymore, so flush_sigqueue() can be invoked + * lockless. */ - flush_sigqueue(&p->pending); if (thread_group_leader(p)) flush_sigqueue(&p->signal->shared_pending); diff --git a/kernel/signal.c b/kernel/signal.c index 575e2450666d8..561dfef818014 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -474,18 +474,42 @@ static void __sigqueue_free(struct sigqueue *q) kmem_cache_free(sigqueue_cachep, q); } -void flush_sigqueue(struct sigpending *queue) +/* + * flush_sigqueue_list() can only be invoked without holding sighand::siglock in + * the following cases: + * + * 1) When flushing task::pending _after_ setting task::flags PF_EXITING + * + * All functions which try to send a signal to @task will observe PF_EXITING + * and drop the signal. + * + * 2) When flushing task::signal::shared_pending _after_ the last task in a + * thread group was unhashed and task::sighand is NULL. + * + * Nothing can queue a signal anymore because sighand is NULL. + */ +static void flush_sigqueue_list(struct list_head *head) { - struct sigqueue *q; + struct sigqueue *q, *tmp; - sigemptyset(&queue->signal); - while (!list_empty(&queue->list)) { - q = list_entry(queue->list.next, struct sigqueue , list); + list_for_each_entry_safe(q, tmp, head, list) { list_del_init(&q->list); __sigqueue_free(q); } } +void flush_sigqueue(struct sigpending *queue) +{ + sigemptyset(&queue->signal); + flush_sigqueue_list(&queue->list); +} + +static void sigqueue_dequeue_pending(struct sigpending *queue, struct list_head *head) +{ + sigemptyset(&queue->signal); + list_splice_init(&queue->list, head); +} + /* * Flush all pending signals for this kthread. */ @@ -1038,6 +1062,21 @@ static inline bool legacy_queue(struct sigpending *signals, int sig) return (sig < SIGRTMIN) && sigismember(&signals->signal, sig); } +/* + * When PF_EXITING is set the task is on the way out and has t::pending + * flushed already. Prevent queueing of PIDTYPE_PID signals as they would + * be leaked. + */ +static inline bool task_can_queue_signal(struct task_struct *t, enum pid_type type) +{ + lockdep_assert_held(&t->sighand->siglock); + + if (!(t->flags & PF_EXITING)) + return true; + + return type != PIDTYPE_PID; +} + static int __send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type, bool force) { @@ -1049,6 +1088,10 @@ static int __send_signal_locked(int sig, struct kernel_siginfo *info, lockdep_assert_held(&t->sighand->siglock); result = TRACE_SIGNAL_IGNORED; + + if (!task_can_queue_signal(t, type)) + goto ret; + if (!prepare_signal(sig, t, force)) goto ret; @@ -1967,18 +2010,33 @@ static void posixtimer_queue_sigqueue(struct sigqueue *q, struct task_struct *t, * * Where type is not PIDTYPE_PID, signals must be delivered to the * process. In this case, prefer to deliver to current if it is in - * the same thread group as the target process, which avoids - * unnecessarily waking up a potentially idle task. + * the same thread group as the target process and its sighand is + * stable, which avoids unnecessarily waking up a potentially idle task. */ static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr) { struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type); - if (t && tmr->it_pid_type != PIDTYPE_PID && same_thread_group(t, current)) + if (t && tmr->it_pid_type != PIDTYPE_PID && + same_thread_group(t, current) && !(current->flags & PF_EXITING)) t = current; return t; } +/* + * Find the target task for the POSIX timer signal and prevent that a + * PIDTYPE_PID signal is queued on a task which has PF_EXITING set. + */ +static inline struct task_struct *posixtimer_get_unignore_target(struct k_itimer *tmr) +{ + struct task_struct *t = posixtimer_get_target(tmr); + + if (t && task_can_queue_signal(t, tmr->it_pid_type)) + return t; + + return NULL; +} + void posixtimer_send_sigqueue(struct k_itimer *tmr) { struct sigqueue *q = &tmr->sigq; @@ -1996,6 +2054,9 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) if (!likely(lock_task_sighand(t, &flags))) return; + if (!task_can_queue_signal(t, tmr->it_pid_type)) + goto unlock; + /* * Update @tmr::sigqueue_seq for posix timer signals with sighand * locked to prevent a race against dequeue_signal(). @@ -2087,6 +2148,7 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) result = TRACE_SIGNAL_DELIVERED; out: trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result); +unlock: unlock_task_sighand(t, &flags); } @@ -2142,7 +2204,7 @@ static void posixtimer_sig_unignore(struct task_struct *tsk, int sig) * has exited by now, drop the reference count. */ guard(rcu)(); - target = posixtimer_get_target(tmr); + target = posixtimer_get_unignore_target(tmr); if (target) posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type); else @@ -3124,44 +3186,37 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which) void exit_signals(struct task_struct *tsk) { + LIST_HEAD(sigq_list); int group_stop = 0; - sigset_t unblocked; /* * @tsk is about to have PF_EXITING set - lock out users which - * expect stable threadgroup. + * expect a stable threadgroup. */ cgroup_threadgroup_change_begin(tsk); - if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + scoped_guard(spinlock_irq, &tsk->sighand->siglock) { sched_mm_cid_exit_signals(tsk); tsk->flags |= PF_EXITING; - cgroup_threadgroup_change_end(tsk); - return; - } - spin_lock_irq(&tsk->sighand->siglock); - /* - * From now this task is not visible for group-wide signals, - * see wants_signal(), do_signal_stop(). - */ - sched_mm_cid_exit_signals(tsk); - tsk->flags |= PF_EXITING; + sigqueue_dequeue_pending(&tsk->pending, &sigq_list); - cgroup_threadgroup_change_end(tsk); + if (task_sigpending(tsk) && !thread_group_empty(tsk) && + !(tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + sigset_t unblocked = tsk->blocked; - if (!task_sigpending(tsk)) - goto out; + signotset(&unblocked); + retarget_shared_pending(tsk, &unblocked); - unblocked = tsk->blocked; - signotset(&unblocked); - retarget_shared_pending(tsk, &unblocked); + if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && + task_participate_group_stop(tsk)) + group_stop = CLD_STOPPED; + } + } - if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && - task_participate_group_stop(tsk)) - group_stop = CLD_STOPPED; -out: - spin_unlock_irq(&tsk->sighand->siglock); + cgroup_threadgroup_change_end(tsk); + + flush_sigqueue_list(&sigq_list); /* * If group stop has completed, deliver the notification. This diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 749dc1258a44b..3072750c39080 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -11,6 +11,7 @@ #include #include +#include static void aescfb_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, const void *src) @@ -220,6 +221,8 @@ static int __init libaescfb_init(void) if (aes_expandkey(&ctx, aescfb_tv[i].key, aescfb_tv[i].klen)) { pr_err("aes_expandkey() failed on vector %d\n", i); + if (fips_enabled) + panic("aes_expandkey() failed on vector %d\n", i); return -ENODEV; } @@ -227,6 +230,8 @@ static int __init libaescfb_init(void) aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) { pr_err("aescfb_encrypt() #1 failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_encrypt() #1 failed on vector %d\n", i); return -ENODEV; } @@ -234,6 +239,8 @@ static int __init libaescfb_init(void) aescfb_decrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ptext, aescfb_tv[i].len)) { pr_err("aescfb_decrypt() failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_decrypt() failed on vector %d\n", i); return -ENODEV; } @@ -241,6 +248,8 @@ static int __init libaescfb_init(void) aescfb_encrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) { pr_err("aescfb_encrypt() #2 failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_encrypt() #2 failed on vector %d\n", i); return -ENODEV; } diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 6bba6473fdf34..1aa5b325935b9 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -12,6 +12,7 @@ #include #include +#include static void aesgcm_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, const void *src) @@ -702,6 +703,8 @@ static int __init libaesgcm_init(void) if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, aesgcm_tv[i].clen - plen)) { pr_err("aesgcm_expandkey() failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_expandkey() failed on vector %d\n", i); return -ENODEV; } @@ -710,6 +713,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].iv, aesgcm_tv[i].ctext + plen) || memcmp(buf, aesgcm_tv[i].ptext, plen)) { pr_err("aesgcm_decrypt() #1 failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_decrypt() #1 failed on vector %d\n", i); return -ENODEV; } @@ -718,6 +723,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf); if (memcmp(buf, aesgcm_tv[i].ctext, plen)) { pr_err("aesgcm_encrypt() failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_encrypt() failed on vector %d\n", i); return -ENODEV; } @@ -726,6 +733,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf) || memcmp(buf, aesgcm_tv[i].ptext, plen)) { pr_err("aesgcm_decrypt() #2 failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_decrypt() #2 failed on vector %d\n", i); return -ENODEV; } } diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index ebb60519ae939..955617749e71a 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -137,5 +137,51 @@ void sha1_init(__u32 *buf) } EXPORT_SYMBOL(sha1_init); +#if defined(CONFIG_CRYPTO_FIPS) && !defined(__DISABLE_EXPORTS) +#include +#include + +static int __init sha1_mod_init(void) +{ + if (fips_enabled) { + /* + * FIPS known answer test for SHA-1. + * Input: NIST one-block message "abc" (pre-padded). + * Expected digest: a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d + */ + static const u8 fips_input[SHA1_BLOCK_SIZE] __initconst = { + 0x61, 0x62, 0x63, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + }; + static const __u32 fips_digest[SHA1_DIGEST_WORDS] __initconst = { + 0xa9993e36, 0x4706816a, 0xba3e2571, + 0x7850c26c, 0x9cd0d89d, + }; + __u32 digest[SHA1_DIGEST_WORDS]; + __u32 workspace[SHA1_WORKSPACE_WORDS]; + + sha1_init(digest); + sha1_transform(digest, (const char *)fips_input, workspace); + memzero_explicit(workspace, sizeof(workspace)); + + if (memcmp(digest, fips_digest, sizeof(digest)) != 0) + panic("sha1: FIPS self-test failed\n"); + } + return 0; +} +subsys_initcall(sha1_mod_init); + +static void __exit sha1_mod_exit(void) +{ +} +module_exit(sha1_mod_exit); +#endif + MODULE_DESCRIPTION("SHA-1 Algorithm"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 04c1f2557e6c2..58490a67be245 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -165,5 +165,43 @@ void sha256(const u8 *data, unsigned int len, u8 *out) } EXPORT_SYMBOL(sha256); +#if defined(CONFIG_CRYPTO_FIPS) && !defined(__DISABLE_EXPORTS) +#include +#include + +static int __init sha256_mod_init(void) +{ + if (fips_enabled) { + /* + * FIPS known answer test for SHA-256. + * Input: NIST one-block message "abc". + * Expected digest: + * ba7816bf 8f01cfea 414140de 5dae2223 + * b00361a3 96177a9c b410ff61 f20015ad + */ + static const u8 fips_input[] __initconst = { 'a', 'b', 'c' }; + static const u8 fips_digest[SHA256_DIGEST_SIZE] __initconst = { + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, + }; + u8 out[SHA256_DIGEST_SIZE]; + + sha256(fips_input, sizeof(fips_input), out); + + if (memcmp(out, fips_digest, sizeof(out)) != 0) + panic("sha256: FIPS self-test failed\n"); + } + return 0; +} +subsys_initcall(sha256_mod_init); + +static void __exit sha256_mod_exit(void) +{ +} +module_exit(sha256_mod_exit); +#endif + MODULE_DESCRIPTION("SHA-256 Algorithm"); MODULE_LICENSE("GPL"); diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 6482de4d87509..10127b7588efa 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -40,9 +40,13 @@ static int nf_br_ip_fragment(struct net *net, struct sock *sk, int err = 0; /* for offloaded checksums cleanup checksum before fragmentation */ - if (skb->ip_summed == CHECKSUM_PARTIAL && - (err = skb_checksum_help(skb))) - goto blackhole; + if (skb->ip_summed == CHECKSUM_PARTIAL) { + err = ip_check_csum_start(skb); + if (!err) + err = skb_checksum_help(skb); + if (err) + goto blackhole; + } iph = ip_hdr(skb); diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 7625bcc856155..780d0a6e50958 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -779,9 +779,13 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int err = 0; /* for offloaded checksums cleanup checksum before fragmentation */ - if (skb->ip_summed == CHECKSUM_PARTIAL && - (err = skb_checksum_help(skb))) - goto fail; + if (skb->ip_summed == CHECKSUM_PARTIAL) { + err = ip_check_csum_start(skb); + if (!err) + err = skb_checksum_help(skb); + if (err) + goto fail; + } /* * Point into the IP datagram header. diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index e807c5e15b052..c6d6f28d5fa32 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -678,6 +678,9 @@ ip6ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, if (!skb2) return 0; + /* Remove debris left by outer IPv6 stack. */ + memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, offset); skb_reset_network_header(skb2); diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 3373b6b34dc7d..63d5ec55afc55 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -38,6 +38,8 @@ #define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1) +static seqcount_t mpls_platform_label_seq = SEQCNT_ZERO(mpls_platform_label_seq); + static int label_limit = (1 << 20) - 1; static int ttl_max = 255; @@ -75,16 +77,32 @@ static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt, struct nlmsghdr *nlh, struct net *net, u32 portid, unsigned int nlm_flags); +static struct mpls_route __rcu **mpls_platform_label_rcu(struct net *net, + size_t *platform_labels) +{ + struct mpls_route __rcu **platform_label; + unsigned int sequence; + + do { + sequence = read_seqcount_begin(&mpls_platform_label_seq); + platform_label = rcu_dereference_rtnl(net->mpls.platform_label); + *platform_labels = net->mpls.platform_labels; + } while (read_seqcount_retry(&mpls_platform_label_seq, sequence)); + + return platform_label; +} + static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index) { - struct mpls_route *rt = NULL; + struct mpls_route __rcu **platform_label; + size_t platform_labels; - if (index < net->mpls.platform_labels) { - struct mpls_route __rcu **platform_label = - rcu_dereference_rtnl(net->mpls.platform_label); - rt = rcu_dereference_rtnl(platform_label[index]); - } - return rt; + platform_label = mpls_platform_label_rcu(net, &platform_labels); + + if (index < platform_labels) + return rcu_dereference_rtnl(platform_label[index]); + + return NULL; } bool mpls_output_possible(const struct net_device *dev) @@ -2190,8 +2208,7 @@ static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb) if (index < MPLS_LABEL_FIRST_UNRESERVED) index = MPLS_LABEL_FIRST_UNRESERVED; - platform_label = rtnl_dereference(net->mpls.platform_label); - platform_labels = net->mpls.platform_labels; + platform_label = mpls_platform_label_rcu(net, &platform_labels); if (filter.filter_set) flags |= NLM_F_DUMP_FILTERED; @@ -2576,8 +2593,14 @@ static int resize_platform_label_table(struct net *net, size_t limit) } /* Update the global pointers */ + local_bh_disable(); + preempt_disable_nested(); + write_seqcount_begin(&mpls_platform_label_seq); net->mpls.platform_labels = limit; rcu_assign_pointer(net->mpls.platform_label, labels); + write_seqcount_end(&mpls_platform_label_seq); + preempt_enable_nested(); + local_bh_enable(); rtnl_unlock(); diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 508276f9e034a..bf12e5bc417ca 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -653,6 +654,12 @@ static int nfqnl_put_bridge(struct nf_queue_entry *entry, struct sk_buff *skb) static int nf_queue_checksum_help(struct sk_buff *entskb) { + if (entskb->protocol == htons(ETH_P_IP)) { + if (!pskb_network_may_pull(entskb, sizeof(struct iphdr)) || + ip_check_csum_start(entskb)) + return -EINVAL; + } + if (skb_csum_is_sctp(entskb)) return skb_crc32c_csum_help(entskb); diff --git a/net/netfilter/xt_CHECKSUM.c b/net/netfilter/xt_CHECKSUM.c index 9d99f5a3d1764..1fb0f8118404d 100644 --- a/net/netfilter/xt_CHECKSUM.c +++ b/net/netfilter/xt_CHECKSUM.c @@ -15,6 +15,7 @@ #include #include +#include MODULE_LICENSE("GPL"); MODULE_AUTHOR("Michael S. Tsirkin "); @@ -25,8 +26,11 @@ MODULE_ALIAS("ip6t_CHECKSUM"); static unsigned int checksum_tg(struct sk_buff *skb, const struct xt_action_param *par) { - if (skb->ip_summed == CHECKSUM_PARTIAL && !skb_is_gso(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && !skb_is_gso(skb)) { + if (xt_family(par) == NFPROTO_IPV4 && ip_check_csum_start(skb)) + return NF_DROP; skb_checksum_help(skb); + } return XT_CONTINUE; } diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index c688dee96503f..a1df551e915bc 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -196,11 +196,13 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev, 0, NULL); - netdev_put(vport->dev, &vport->dev_tracker); - vport->dev = NULL; - rtnl_unlock(); + /* We can't put the device reference yet, since it can still be in + * use, but rtnl_unlock()->netdev_run_todo() will block until all + * the references are released, so the RCU call must be before it. + */ call_rcu(&vport->rcu, vport_netdev_free); + rtnl_unlock(); } EXPORT_SYMBOL_GPL(ovs_netdev_tunnel_destroy); diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 0615e4426341c..af167ca19bc6d 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -1038,6 +1038,7 @@ struct sctp_chunk *sctp_process_strreset_resp( stsn, rtsn, GFP_ATOMIC); } else if (req->type == SCTP_PARAM_RESET_ADD_OUT_STREAMS) { struct sctp_strreset_addstrm *addstrm; + const struct sctp_sched_ops *sched; __u16 number; addstrm = (struct sctp_strreset_addstrm *)req; @@ -1048,7 +1049,10 @@ struct sctp_chunk *sctp_process_strreset_resp( for (i = number; i < stream->outcnt; i++) SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; } else { - sctp_stream_shrink_out(stream, number); + sched = sctp_sched_ops_from_stream(stream); + sched->unsched_all(stream); + sctp_stream_outq_migrate(stream, NULL, number); + sched->sched_all(stream); stream->outcnt = number; } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index ecf09941a0f3c..f69bb9f6e6416 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1969,9 +1969,10 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, if (err < 0) return err; - err = -EOPNOTSUPP; - if (msg->msg_flags&MSG_OOB) + if (msg->msg_flags & MSG_OOB) { + err = -EOPNOTSUPP; goto out; + } if (msg->msg_namelen) { err = unix_validate_addr(sunaddr, msg->msg_namelen); @@ -1986,10 +1987,11 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, goto out; } else { sunaddr = NULL; - err = -ENOTCONN; other = unix_peer_get(sk); - if (!other) + if (!other) { + err = -ENOTCONN; goto out; + } } if ((test_bit(SOCK_PASSCRED, &sock->flags) || @@ -2000,9 +2002,10 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, goto out; } - err = -EMSGSIZE; - if (len > READ_ONCE(sk->sk_sndbuf) - 32) + if (len > READ_ONCE(sk->sk_sndbuf) - 32) { + err = -EMSGSIZE; goto out; + } if (len > SKB_MAX_ALLOC) { data_len = min_t(size_t, @@ -2034,9 +2037,10 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, restart: if (!other) { - err = -ECONNRESET; - if (sunaddr == NULL) + if (!sunaddr) { + err = -ECONNRESET; goto out_free; + } other = unix_find_other(sock_net(sk), sunaddr, msg->msg_namelen, sk->sk_type); @@ -2056,9 +2060,11 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, sk_locked = 0; unix_state_lock(other); restart_locked: - err = -EPERM; - if (!unix_may_send(sk, other)) + + if (!unix_may_send(sk, other)) { + err = -EPERM; goto out_unlock; + } if (unlikely(sock_flag(other, SOCK_DEAD))) { /* @@ -2071,7 +2077,6 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, if (!sk_locked) unix_state_lock(sk); - err = 0; if (sk->sk_type == SOCK_SEQPACKET) { /* We are here only when racing with unix_release_sock() * is clearing @other. Never change state to TCP_CLOSE @@ -2099,9 +2104,10 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, goto restart; } - err = -EPIPE; - if (other->sk_shutdown & RCV_SHUTDOWN) + if (other->sk_shutdown & RCV_SHUTDOWN) { + err = -EPIPE; goto out_unlock; + } if (sk->sk_type != SOCK_SEQPACKET) { err = security_unix_may_send(sk->sk_socket, other->sk_socket); diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index ebf95d48e86c1..6b96a952c88f2 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb, ka->encap_sport, sock_net_uid(net, NULL)); rt = ip_route_output_key(net, &fl4); - if (IS_ERR(rt)) + if (IS_ERR(rt)) { + kfree_skb(skb); return PTR_ERR(rt); + } skb_dst_set(skb, &rt->dst); @@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb, dst = ipv6_stub->ipv6_dst_lookup_flow(net, sk, &fl6, NULL); if (IS_ERR(dst)) { local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock); + kfree_skb(skb); return PTR_ERR(dst); } @@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka) sizeof(struct ipv6hdr)) + sizeof(struct udphdr); const u8 nat_ka_payload = 0xFF; - int err = -EAFNOSUPPORT; struct sk_buff *skb; struct udphdr *uh; @@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka) switch (ka->family) { case AF_INET: - err = nat_keepalive_send_ipv4(skb, ka); + nat_keepalive_send_ipv4(skb, ka); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - err = nat_keepalive_send_ipv6(skb, ka, uh); + nat_keepalive_send_ipv6(skb, ka, uh); break; #endif - } - if (err) + default: kfree_skb(skb); + break; + } } struct nat_keepalive_work_ctx { diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index 71bfbb6760f7e..15469d664f5f9 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -816,16 +816,22 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) } if (skb->ip_summed == CHECKSUM_PARTIAL) { - err = skb_checksum_help(skb); - if (err) { - XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); - kfree_skb(skb); - return err; + if (skb->protocol == htons(ETH_P_IP)) { + err = ip_check_csum_start(skb); + if (err) + goto error; } + err = skb_checksum_help(skb); + if (err) + goto error; } out: return xfrm_output2(net, sk, skb); +error: + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); + kfree_skb(skb); + return err; } EXPORT_SYMBOL_GPL(xfrm_output); diff --git a/tools/hv/hv_kvp_daemon.c b/tools/hv/hv_kvp_daemon.c index ae57bf69ad4af..902513e36a36d 100644 --- a/tools/hv/hv_kvp_daemon.c +++ b/tools/hv/hv_kvp_daemon.c @@ -83,6 +83,7 @@ enum { }; static int in_hand_shake; +static int debug; static char *os_name = ""; static char *os_major = ""; @@ -183,6 +184,20 @@ static void kvp_update_file(int pool) kvp_release_lock(pool); } +static void kvp_dump_initial_pools(int pool) +{ + int i; + + syslog(LOG_DEBUG, "===Start dumping the contents of pool %d ===\n", + pool); + + for (i = 0; i < kvp_file_info[pool].num_records; i++) + syslog(LOG_DEBUG, "pool: %d, %d/%d key=%s val=%s\n", + pool, i + 1, kvp_file_info[pool].num_records, + kvp_file_info[pool].records[i].key, + kvp_file_info[pool].records[i].value); +} + static void kvp_update_mem_state(int pool) { FILE *filep; @@ -270,6 +285,8 @@ static int kvp_file_init(void) return 1; kvp_file_info[i].num_records = 0; kvp_update_mem_state(i); + if (debug) + kvp_dump_initial_pools(i); } return 0; @@ -297,6 +314,9 @@ static int kvp_key_delete(int pool, const __u8 *key, int key_size) * Found a match; just move the remaining * entries up. */ + if (debug) + syslog(LOG_DEBUG, "%s: deleting the KVP: pool=%d key=%s val=%s", + __func__, pool, record[i].key, record[i].value); if (i == (num_records - 1)) { kvp_file_info[pool].num_records--; kvp_update_file(pool); @@ -315,20 +335,36 @@ static int kvp_key_delete(int pool, const __u8 *key, int key_size) kvp_update_file(pool); return 0; } + + if (debug) + syslog(LOG_DEBUG, "%s: could not delete KVP: pool=%d key=%s. Record not found", + __func__, pool, key); + return 1; } static int kvp_key_add_or_modify(int pool, const __u8 *key, int key_size, const __u8 *value, int value_size) { - int i; - int num_records; struct kvp_record *record; + int num_records; int num_blocks; + int i; + + if (debug) + syslog(LOG_DEBUG, "%s: got a KVP: pool=%d key=%s val=%s", + __func__, pool, key, value); if ((key_size > HV_KVP_EXCHANGE_MAX_KEY_SIZE) || - (value_size > HV_KVP_EXCHANGE_MAX_VALUE_SIZE)) + (value_size > HV_KVP_EXCHANGE_MAX_VALUE_SIZE)) { + syslog(LOG_ERR, "%s: Too long key or value: key=%s, val=%s", + __func__, key, value); + + if (debug) + syslog(LOG_DEBUG, "%s: Too long key or value: pool=%d, key=%s, val=%s", + __func__, pool, key, value); return 1; + } /* * First update the in-memory state. @@ -348,6 +384,9 @@ static int kvp_key_add_or_modify(int pool, const __u8 *key, int key_size, */ memcpy(record[i].value, value, value_size); kvp_update_file(pool); + if (debug) + syslog(LOG_DEBUG, "%s: updated: pool=%d key=%s val=%s", + __func__, pool, key, value); return 0; } @@ -359,8 +398,10 @@ static int kvp_key_add_or_modify(int pool, const __u8 *key, int key_size, record = realloc(record, sizeof(struct kvp_record) * ENTRIES_PER_BLOCK * (num_blocks + 1)); - if (record == NULL) + if (!record) { + syslog(LOG_ERR, "%s: Memory alloc failure", __func__); return 1; + } kvp_file_info[pool].num_blocks++; } @@ -368,6 +409,11 @@ static int kvp_key_add_or_modify(int pool, const __u8 *key, int key_size, memcpy(record[i].key, key, key_size); kvp_file_info[pool].records = record; kvp_file_info[pool].num_records++; + + if (debug) + syslog(LOG_DEBUG, "%s: added: pool=%d key=%s val=%s", + __func__, pool, key, value); + kvp_update_file(pool); return 0; } @@ -1661,6 +1707,7 @@ void print_usage(char *argv[]) fprintf(stderr, "Usage: %s [options]\n" "Options are:\n" " -n, --no-daemon stay in foreground, don't daemonize\n" + " -d, --debug Enable debug logs(syslog debug by default)\n" " -h, --help print this help\n", argv[0]); } @@ -1682,10 +1729,11 @@ int main(int argc, char *argv[]) static struct option long_options[] = { {"help", no_argument, 0, 'h' }, {"no-daemon", no_argument, 0, 'n' }, + {"debug", no_argument, 0, 'd' }, {0, 0, 0, 0 } }; - while ((opt = getopt_long(argc, argv, "hn", long_options, + while ((opt = getopt_long(argc, argv, "hnd", long_options, &long_index)) != -1) { switch (opt) { case 'n': @@ -1694,6 +1742,9 @@ int main(int argc, char *argv[]) case 'h': print_usage(argv); exit(0); + case 'd': + debug = 1; + break; default: print_usage(argv); exit(EXIT_FAILURE); @@ -1716,6 +1767,9 @@ int main(int argc, char *argv[]) */ kvp_get_domain_name(full_domain_name, sizeof(full_domain_name)); + if (debug) + syslog(LOG_INFO, "Logging debug info in syslog(debug)"); + if (kvp_file_init()) { syslog(LOG_ERR, "Failed to initialize the pools"); exit(EXIT_FAILURE); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 2207e51e4d079..d48ab0b95dec9 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -32,6 +32,7 @@ TARGETS += filesystems TARGETS += filesystems/binderfs TARGETS += filesystems/epoll TARGETS += filesystems/fat +TARGETS += filesystems/fuse TARGETS += filesystems/overlayfs TARGETS += filesystems/statmount TARGETS += firmware diff --git a/tools/testing/selftests/filesystems/fuse/.gitignore b/tools/testing/selftests/filesystems/fuse/.gitignore new file mode 100644 index 0000000000000..323aa31624df3 --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +fuse_mnt +fuse_uring_test +fusectl_test diff --git a/tools/testing/selftests/filesystems/fuse/Makefile b/tools/testing/selftests/filesystems/fuse/Makefile new file mode 100644 index 0000000000000..ca4f296c2555d --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/Makefile @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +CFLAGS += -Wall -O2 -g $(KHDR_INCLUDES) + +TEST_GEN_PROGS := fusectl_test +TEST_GEN_FILES := fuse_mnt + +# fuse_uring_test requires liburing; add it only when the library is present. +URING_CFLAGS := $(shell pkg-config liburing --cflags 2>/dev/null) +URING_LDLIBS := $(shell pkg-config liburing --libs 2>/dev/null) +ifeq ($(URING_LDLIBS),) +$(warning liburing not found; fuse_uring_test will be skipped) +else +TEST_GEN_PROGS += fuse_uring_test +endif + +# fuse_acl_cache_test requires libfuse3; add it only when the library is present. +ACL_CFLAGS := $(shell pkg-config fuse3 --cflags 2>/dev/null) +ACL_LDLIBS := $(shell pkg-config fuse3 --libs 2>/dev/null) +ifneq ($(ACL_CFLAGS),) +TEST_GEN_PROGS += fuse_acl_cache_test +endif + +include ../../lib.mk + +VAR_CFLAGS := $(shell pkg-config fuse --cflags 2>/dev/null) +ifeq ($(VAR_CFLAGS),) +# libfuse2-devel not found; try fuse3 which provides v2 compat headers +VAR_CFLAGS := $(shell pkg-config fuse3 --cflags 2>/dev/null) +ifeq ($(VAR_CFLAGS),) +VAR_CFLAGS := -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse +endif +endif + +VAR_LDLIBS := $(shell pkg-config fuse --libs 2>/dev/null) +ifeq ($(VAR_LDLIBS),) +VAR_LDLIBS := $(shell pkg-config fuse3 --libs 2>/dev/null) +ifeq ($(VAR_LDLIBS),) +VAR_LDLIBS := -lfuse -pthread +endif +endif + +$(OUTPUT)/fuse_mnt: CFLAGS += $(VAR_CFLAGS) +$(OUTPUT)/fuse_mnt: LDLIBS += $(VAR_LDLIBS) + +$(OUTPUT)/fuse_acl_cache_test: CFLAGS += $(ACL_CFLAGS) +$(OUTPUT)/fuse_acl_cache_test: LDLIBS += $(ACL_LDLIBS) + +$(OUTPUT)/fuse_uring_test: CFLAGS += $(URING_CFLAGS) -Wextra -Wno-unused-parameter -pthread +$(OUTPUT)/fuse_uring_test: LDLIBS += $(URING_LDLIBS) -lpthread diff --git a/tools/testing/selftests/filesystems/fuse/config b/tools/testing/selftests/filesystems/fuse/config new file mode 100644 index 0000000000000..c4b20b2c2d037 --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/config @@ -0,0 +1,3 @@ +CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y +CONFIG_IO_URING=y diff --git a/tools/testing/selftests/filesystems/fuse/fuse_acl_cache_test.c b/tools/testing/selftests/filesystems/fuse/fuse_acl_cache_test.c new file mode 100644 index 0000000000000..5d8f28271b1ac --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/fuse_acl_cache_test.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test: FUSE ACL caching bug triggered by AT_STATX_FORCE_SYNC + * + * A FUSE mount that does not negotiate FUSE_POSIX_ACL initialises every inode + * with i_acl = i_default_acl = ACL_DONT_CACHE. When a fresh stat is needed + * (e.g. AT_STATX_FORCE_SYNC), fuse_update_get_attr() calls + * forget_all_cached_acls() before issuing FUSE_GETATTR. On an unfixed kernel, + * __forget_cached_acl() replaces ACL_DONT_CACHE with ACL_NOT_CACHED, + * inadvertently enabling the kernel ACL cache for that inode. The next + * getxattr populates the cache. Because fuse_set_acl() skips + * forget_all_cached_acls() for !fc->posix_acl mounts, any subsequent change to + * the ACL leaves the stale kernel entry in place, and the next getxattr returns + * wrong data without ever reaching the FUSE daemon. + * + * Fix (fs/posix_acl.c): __forget_cached_acl() returns early when *p is + * ACL_DONT_CACHE, preserving the "never cache" invariant for the inode's + * lifetime. + * + * Test outline: + * 1. Mount a minimal FUSE fs (no FUSE_POSIX_ACL negotiated). + * 2. lgetxattr -> daemon called, ACL_A returned, NOT cached (ACL_DONT_CACHE). + * 3. statx(AT_STATX_FORCE_SYNC) -> forget_all_cached_acls() called. + * Buggy: ACL_DONT_CACHE -> ACL_NOT_CACHED (cache enabled). + * Fixed: ACL_DONT_CACHE preserved. + * 4. lgetxattr -> daemon called, ACL_A returned. + * Buggy: result now cached (ACL_NOT_CACHED -> cached). + * Fixed: result still not cached. + * 5. Daemon switches to ACL_B internally (different size). + * 6. lgetxattr -> should return ACL_B (44 bytes). + * Buggy: cache hit, returns stale ACL_A (28 bytes). FAIL. + * Fixed: no cache, daemon called, returns ACL_B (44 bytes). PASS. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FUSE_USE_VERSION 31 +#include + +#include "../../kselftest_harness.h" + +/* ---- ACL binary encoding ------------------------------------------------ */ +/* + * POSIX ACL v2 xattr format (little-endian): + * header: u32 version (= 0x00000002) + * entry: u16 tag | u16 perm | u32 id + * + * Entries must appear in tag-ascending order; named USER/GROUP entries + * require a MASK entry. Both ACLs pass posix_acl_from_xattr() validation. + */ + +/* ACL_A: 3 entries (USER_OBJ:rwx, GROUP_OBJ:r-x, OTHER:r-x) = 28 bytes */ +static const uint8_t acl_a[] = { + 0x02, 0x00, 0x00, 0x00, /* v2 header */ + 0x01, 0x00, 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, /* USER_OBJ rwx */ + 0x04, 0x00, 0x05, 0x00, 0xff, 0xff, 0xff, 0xff, /* GROUP_OBJ r-x */ + 0x20, 0x00, 0x05, 0x00, 0xff, 0xff, 0xff, 0xff, /* OTHER r-x */ +}; + +/* + * ACL_B: 5 entries — adds USER uid=1 and MASK = 44 bytes. + * A named USER entry requires a MASK; all tags in ascending order. + */ +static const uint8_t acl_b[] = { + 0x02, 0x00, 0x00, 0x00, /* v2 header */ + 0x01, 0x00, 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, /* USER_OBJ rwx */ + 0x02, 0x00, 0x07, 0x00, 0x01, 0x00, 0x00, 0x00, /* USER uid=1 rwx */ + 0x04, 0x00, 0x05, 0x00, 0xff, 0xff, 0xff, 0xff, /* GROUP_OBJ r-x */ + 0x10, 0x00, 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, /* MASK rwx */ + 0x20, 0x00, 0x05, 0x00, 0xff, 0xff, 0xff, 0xff, /* OTHER r-x */ +}; + +/* ---- Shared state (daemon thread <-> test thread) ----------------------- */ + +#define FILE_INO 2 +#define FILE_NAME "testfile" + +struct daemon_state { + pthread_mutex_t lock; + const uint8_t *acl; + size_t acl_size; + int getxattr_count; +}; + +/* + * Global: callbacks are stateless fns so we use a single global. + * Safe because only one test instance runs at a time. + */ +static struct daemon_state g_ds = { + .lock = PTHREAD_MUTEX_INITIALIZER, +}; + +/* ---- FUSE lowlevel callbacks -------------------------------------------- */ + +static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char *name) +{ + if (parent != FUSE_ROOT_ID || strcmp(name, FILE_NAME)) { + fuse_reply_err(req, ENOENT); + return; + } + struct fuse_entry_param e = {}; + + /* + * Long attr/entry timeouts so that normal stat() calls do not + * expire and trigger forget_all_cached_acls() on their own; + * only the explicit AT_STATX_FORCE_SYNC should trigger it. + */ + e.ino = FILE_INO; + e.generation = 1; + e.attr_timeout = 10.0; + e.entry_timeout = 10.0; + e.attr.st_ino = FILE_INO; + e.attr.st_mode = S_IFREG | 0644; + e.attr.st_nlink = 1; + fuse_reply_entry(req, &e); +} + +static void fs_getattr(fuse_req_t req, fuse_ino_t ino, + struct fuse_file_info *fi) +{ + struct stat st = {}; + + (void)fi; + if (ino == FUSE_ROOT_ID) { + st.st_ino = FUSE_ROOT_ID; + st.st_mode = S_IFDIR | 0755; + st.st_nlink = 2; + } else if (ino == FILE_INO) { + st.st_ino = FILE_INO; + st.st_mode = S_IFREG | 0644; + st.st_nlink = 1; + } else { + fuse_reply_err(req, ENOENT); + return; + } + fuse_reply_attr(req, &st, 10); +} + +static void fs_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, + size_t size) +{ + if (ino != FILE_INO || + strcmp(name, "system.posix_acl_access") != 0) { + fuse_reply_err(req, ENODATA); + return; + } + + pthread_mutex_lock(&g_ds.lock); + const uint8_t *acl = g_ds.acl; + size_t acl_size = g_ds.acl_size; + g_ds.getxattr_count++; + pthread_mutex_unlock(&g_ds.lock); + + if (size == 0) + fuse_reply_xattr(req, acl_size); + else if (size < acl_size) + fuse_reply_err(req, ERANGE); + else + fuse_reply_buf(req, (const char *)acl, acl_size); +} + +static const struct fuse_lowlevel_ops fs_ops = { + .lookup = fs_lookup, + .getattr = fs_getattr, + .getxattr = fs_getxattr, +}; + +/* ---- Daemon thread ------------------------------------------------------- */ + +static void *run_daemon(void *arg) +{ + fuse_session_loop((struct fuse_session *)arg); + return NULL; +} + +/* ---- kselftest harness --------------------------------------------------- */ + +FIXTURE(acl_cache) { + struct fuse_session *se; + char mountpoint[PATH_MAX]; + char file_path[PATH_MAX]; + pthread_t thread; +}; + +FIXTURE_SETUP(acl_cache) +{ + char *fuse_argv[] = { "fuse_acl_cache_test", NULL }; + struct fuse_args args = FUSE_ARGS_INIT(1, fuse_argv); + + g_ds.acl = acl_a; + g_ds.acl_size = sizeof(acl_a); + g_ds.getxattr_count = 0; + + strcpy(self->mountpoint, "/tmp/acl_cache_test_XXXXXX"); + if (!mkdtemp(self->mountpoint)) + SKIP(return, "mkdtemp: %s", strerror(errno)); + + snprintf(self->file_path, sizeof(self->file_path), + "%s/" FILE_NAME, self->mountpoint); + + self->se = fuse_session_new(&args, &fs_ops, sizeof(fs_ops), NULL); + if (!self->se) { + rmdir(self->mountpoint); + SKIP(return, "fuse_session_new failed"); + } + + if (fuse_session_mount(self->se, self->mountpoint)) { + fuse_session_destroy(self->se); + rmdir(self->mountpoint); + SKIP(return, "fuse_session_mount failed " + "(missing fusermount3 or insufficient privileges)"); + } + + if (pthread_create(&self->thread, NULL, run_daemon, self->se)) { + fuse_session_unmount(self->se); + fuse_session_destroy(self->se); + rmdir(self->mountpoint); + SKIP(return, "pthread_create: %s", strerror(errno)); + } + + fuse_opt_free_args(&args); +} + +FIXTURE_TEARDOWN(acl_cache) +{ + fuse_session_exit(self->se); + fuse_session_unmount(self->se); + pthread_join(self->thread, NULL); + fuse_session_destroy(self->se); + rmdir(self->mountpoint); +} + +static int do_force_statx(const char *path) +{ + struct statx stx; + + return statx(AT_FDCWD, path, AT_STATX_FORCE_SYNC, STATX_BASIC_STATS, + &stx); +} + +TEST_F(acl_cache, stale_after_force_sync) +{ + char buf[512]; + ssize_t sz; + int count; + + /* + * Step 1: two getxattr calls before any statx(FORCE_SYNC). + * i_acl == ACL_DONT_CACHE. __get_acl's cmpxchg(p, ACL_NOT_CACHED, + * sentinel) finds *p != ACL_NOT_CACHED on every call, so the sentinel + * is never placed and the result is never cached. Both calls must + * reach the daemon, proving ACL_DONT_CACHE suppresses caching. + */ + sz = lgetxattr(self->file_path, "system.posix_acl_access", + buf, sizeof(buf)); + ASSERT_EQ(sz, (ssize_t)sizeof(acl_a)); + + sz = lgetxattr(self->file_path, "system.posix_acl_access", + buf, sizeof(buf)); + ASSERT_EQ(sz, (ssize_t)sizeof(acl_a)); + + pthread_mutex_lock(&g_ds.lock); + count = g_ds.getxattr_count; + pthread_mutex_unlock(&g_ds.lock); + + ASSERT_EQ(count, 2); + TH_LOG("step 1 OK: both pre-trigger getxattrs reached daemon (count=%d), " + "ACL_DONT_CACHE is working", count); + + /* + * Step 2: statx(AT_STATX_FORCE_SYNC). + * fuse_update_get_attr() calls forget_all_cached_acls() before sending + * FUSE_GETATTR. + * Buggy kernel: ACL_DONT_CACHE -> ACL_NOT_CACHED (cache enabled) + * Fixed kernel: ACL_DONT_CACHE preserved (no effect) + */ + ASSERT_EQ(do_force_statx(self->file_path), 0); + TH_LOG("step 2 OK: statx(AT_STATX_FORCE_SYNC) succeeded"); + + /* + * Step 3: getxattr — cache population attempt after the trigger. + * Buggy: *p == ACL_NOT_CACHED -> sentinel placed -> fuse_get_inode_acl + * called -> ACL_A parsed and stored in the kernel cache. + * Fixed: *p == ACL_DONT_CACHE -> sentinel placement skipped -> + * fuse_get_inode_acl called but result not cached. + * Either way the correct ACL_A is returned here. + */ + sz = lgetxattr(self->file_path, "system.posix_acl_access", + buf, sizeof(buf)); + ASSERT_EQ(sz, (ssize_t)sizeof(acl_a)); + + pthread_mutex_lock(&g_ds.lock); + count = g_ds.getxattr_count; + pthread_mutex_unlock(&g_ds.lock); + + ASSERT_EQ(count, 3); + TH_LOG("step 3 OK: post-trigger getxattr reached daemon (count=%d), " + "returned correct ACL_A (%zd bytes)", count, sz); + + /* + * Step 4: switch daemon to ACL_B (different size: 44 vs 28 bytes). + * Simulates an ACL change that fuse_set_acl() would NOT invalidate for + * !fc->posix_acl mounts (it skips forget_all_cached_acls in that case). + * On a fixed kernel the ACL was never cached, so this is moot. + */ + pthread_mutex_lock(&g_ds.lock); + g_ds.acl = acl_b; + g_ds.acl_size = sizeof(acl_b); + pthread_mutex_unlock(&g_ds.lock); + TH_LOG("step 4: daemon switched to ACL_B (%zu bytes)", sizeof(acl_b)); + + /* + * Step 5: getxattr — the decisive check. + * Buggy kernel: cache hit -> stale ACL_A (28 bytes), count stays 3. + * Fixed kernel: no cache -> daemon called -> ACL_B (44 bytes), count 4. + */ + sz = lgetxattr(self->file_path, "system.posix_acl_access", + buf, sizeof(buf)); + + pthread_mutex_lock(&g_ds.lock); + count = g_ds.getxattr_count; + pthread_mutex_unlock(&g_ds.lock); + + if (sz == (ssize_t)sizeof(acl_a)) + TH_LOG("step 5 BUG: stale ACL_A (%zd bytes) from kernel cache " + "(count=%d); ACL_DONT_CACHE corrupted by " + "forget_all_cached_acls()", sz, count); + else + TH_LOG("step 5 OK: daemon reached (count=%d), " + "fresh ACL_B (%zd bytes)", count, sz); + + EXPECT_EQ(sz, (ssize_t)sizeof(acl_b)); + EXPECT_EQ(count, 4); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/filesystems/fuse/fuse_mnt.c b/tools/testing/selftests/filesystems/fuse/fuse_mnt.c new file mode 100644 index 0000000000000..44a2b5e0fd1dd --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/fuse_mnt.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fusectl test file-system + * Creates a simple FUSE filesystem with a single read-write file (/test) + */ + +#define FUSE_USE_VERSION 31 + +#include +#include +#include +#include +#include +#include +#include + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + +static char *content; +static size_t content_size = 0; +static const char test_path[] = "/test"; + +static int test_getattr(const char *path, struct stat *st, + struct fuse_file_info *fi) +{ + (void)fi; + memset(st, 0, sizeof(*st)); + + if (!strcmp(path, "/")) { + st->st_mode = S_IFDIR | 0755; + st->st_nlink = 2; + return 0; + } + + if (!strcmp(path, test_path)) { + st->st_mode = S_IFREG | 0664; + st->st_nlink = 1; + st->st_size = content_size; + return 0; + } + + return -ENOENT; +} + +static int test_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi, + enum fuse_readdir_flags flags) +{ + (void)flags; + + if (strcmp(path, "/")) + return -ENOENT; + + filler(buf, ".", NULL, 0, 0); + filler(buf, "..", NULL, 0, 0); + filler(buf, test_path + 1, NULL, 0, 0); + + return 0; +} + +static int test_open(const char *path, struct fuse_file_info *fi) +{ + if (strcmp(path, test_path)) + return -ENOENT; + + return 0; +} + +static int test_read(const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *fi) +{ + if (strcmp(path, test_path) != 0) + return -ENOENT; + + if (!content || content_size == 0) + return 0; + + if (offset >= content_size) + return 0; + + if (offset + size > content_size) + size = content_size - offset; + + memcpy(buf, content + offset, size); + + return size; +} + +static int test_write(const char *path, const char *buf, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + size_t new_size; + + if (strcmp(path, test_path) != 0) + return -ENOENT; + + if(offset > content_size) + return -EINVAL; + + new_size = MAX(offset + size, content_size); + + if (new_size > content_size) + content = realloc(content, new_size); + + content_size = new_size; + + if (!content) + return -ENOMEM; + + memcpy(content + offset, buf, size); + + return size; +} + +static int test_truncate(const char *path, off_t size, + struct fuse_file_info *fi) +{ + (void)fi; + if (strcmp(path, test_path) != 0) + return -ENOENT; + + if (size == 0) { + free(content); + content = NULL; + content_size = 0; + return 0; + } + + content = realloc(content, size); + + if (!content) + return -ENOMEM; + + if (size > content_size) + memset(content + content_size, 0, size - content_size); + + content_size = size; + return 0; +} + +static struct fuse_operations memfd_ops = { + .getattr = test_getattr, + .readdir = test_readdir, + .open = test_open, + .read = test_read, + .write = test_write, + .truncate = test_truncate, +}; + +int main(int argc, char *argv[]) +{ + return fuse_main(argc, argv, &memfd_ops, NULL); +} diff --git a/tools/testing/selftests/filesystems/fuse/fuse_uring_test.c b/tools/testing/selftests/filesystems/fuse/fuse_uring_test.c new file mode 100644 index 0000000000000..d733d80b63a4a --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/fuse_uring_test.c @@ -0,0 +1,1442 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE io-uring selftest + * + * Validates FUSE request dispatch over io_uring by running a minimal + * in-memory filesystem daemon in a thread and exercising it from the + * test thread. The daemon negotiates FUSE_OVER_IO_URING during + * FUSE_INIT, registers ring entries via FUSE_IO_URING_CMD_REGISTER, + * and handles requests through FUSE_IO_URING_CMD_COMMIT_AND_FETCH. + * + * An atomic counter tracks requests served through io_uring so tests + * can verify the kernel is actually using the io_uring path. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* linux/fuse.h and liburing.h after glibc headers to avoid redefinition */ +#include +#include + +#include "../../kselftest_harness.h" + +/* + * FUSE io-uring UAPI definitions. Remove this block once linux/fuse.h + * includes the io-uring structures (added upstream in v6.14). + */ + +#ifndef FUSE_OVER_IO_URING +#define FUSE_OVER_IO_URING (1ULL << 41) +#endif + +#ifndef FUSE_URING_IN_OUT_HEADER_SZ +#define FUSE_URING_IN_OUT_HEADER_SZ 128 +#define FUSE_URING_OP_IN_OUT_SZ 128 + +struct fuse_uring_ent_in_out { + uint64_t flags; + uint64_t commit_id; + uint32_t payload_sz; + uint32_t padding; + uint64_t reserved; +}; + +struct fuse_uring_req_header { + char in_out[FUSE_URING_IN_OUT_HEADER_SZ]; + char op_in[FUSE_URING_OP_IN_OUT_SZ]; + struct fuse_uring_ent_in_out ring_ent_in_out; +}; + +enum fuse_uring_cmd { + FUSE_IO_URING_CMD_REGISTER = 1, + FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, +}; + +struct fuse_uring_cmd_req { + uint64_t flags; + uint64_t commit_id; + uint16_t qid; + uint8_t padding[6]; +}; +#endif /* FUSE_URING_IN_OUT_HEADER_SZ */ + +/* ---- constants --------------------------------------------------------- */ + +#define HELLO_CONTENT "Hello from FUSE io-uring!\n" +#define HELLO_LEN (sizeof(HELLO_CONTENT) - 1) +#define TESTFILE_MAX (256 * 1024) +#define INODE_ROOT 1 +#define INODE_HELLO 2 +#define INODE_TESTFILE 3 +#define DIRENT_NAME_OFF 24 /* offsetof(struct fuse_dirent, name) */ +#define DIRENT_ALIGN(x) (((x) + 7) & ~(size_t)7) +#define MP_LEN 128 /* max mountpoint path length */ +#define PATH_LEN (MP_LEN + 32) + +/* ---- daemon state (shared between daemon thread and test thread) ------- */ + +struct daemon_state { + int fuse_fd; + struct io_uring ring; + int ring_ready; + int nr_queues; + size_t max_payload_sz; + + /* per-queue entry */ + struct { + struct fuse_uring_req_header *hdr; + void *payload; + struct iovec iov[2]; + } *entries; + + /* synchronisation */ + pthread_mutex_t lock; + pthread_cond_t cond; + atomic_int ready; + atomic_int exiting; + + /* io-uring path verification */ + atomic_int uring_reqs; + + /* in-memory filesystem */ + char *testfile; + size_t testfile_sz; +}; + +/* ---- FUSE request handling --------------------------------------------- */ + +static size_t add_dirent(void *buf, size_t bufsz, size_t off, + uint64_t ino, unsigned type, + const char *name, uint64_t next_off) +{ + size_t nlen = strlen(name); + size_t elen = DIRENT_ALIGN(DIRENT_NAME_OFF + nlen); + struct fuse_dirent *d; + + if (off + elen > bufsz) + return 0; + d = (struct fuse_dirent *)((char *)buf + off); + d->ino = ino; + d->off = next_off; + d->namelen = nlen; + d->type = type; + memcpy(d->name, name, nlen); + if (elen > DIRENT_NAME_OFF + nlen) + memset(d->name + nlen, 0, elen - DIRENT_NAME_OFF - nlen); + return elen; +} + +static void fill_attr(struct fuse_attr *a, uint64_t ino, size_t tf_sz) +{ + memset(a, 0, sizeof(*a)); + a->ino = ino; + a->blksize = 4096; + switch (ino) { + case INODE_ROOT: + a->mode = S_IFDIR | 0755; + a->nlink = 2; + break; + case INODE_HELLO: + a->mode = S_IFREG | 0444; + a->nlink = 1; + a->size = HELLO_LEN; + break; + case INODE_TESTFILE: + a->mode = S_IFREG | 0644; + a->nlink = 1; + a->size = tf_sz; + break; + } +} + +/* + * Handle one FUSE request. Returns payload bytes (>=0) or -errno. + */ +static int handle_request(struct daemon_state *ds, + struct fuse_uring_req_header *hdr, void *payload, + uint32_t payload_sz) +{ + struct fuse_in_header *ih = (void *)hdr->in_out; + + switch (ih->opcode) { + case FUSE_GETATTR: { + struct fuse_attr_out *ao = (void *)payload; + + if (ih->nodeid < INODE_ROOT || ih->nodeid > INODE_TESTFILE) + return -ENOENT; + memset(ao, 0, sizeof(*ao)); + ao->attr_valid = 1; + fill_attr(&ao->attr, ih->nodeid, ds->testfile_sz); + return sizeof(*ao); + } + case FUSE_LOOKUP: { + const char *name = (const char *)payload; + struct fuse_entry_out *eo; + uint64_t ino = 0; + + if (ih->nodeid != INODE_ROOT) + return -ENOENT; + if (payload_sz >= 5 && !memcmp(name, "hello", 5) && + (payload_sz == 5 || name[5] == '\0')) + ino = INODE_HELLO; + else if (payload_sz >= 8 && !memcmp(name, "testfile", 8) && + (payload_sz == 8 || name[8] == '\0')) + ino = INODE_TESTFILE; + else + return -ENOENT; + eo = (void *)payload; + memset(eo, 0, sizeof(*eo)); + eo->nodeid = ino; + eo->generation = 1; + eo->entry_valid = 1; + eo->attr_valid = 1; + fill_attr(&eo->attr, ino, ds->testfile_sz); + return sizeof(*eo); + } + case FUSE_OPEN: + case FUSE_OPENDIR: { + struct fuse_open_out *oo = (void *)payload; + + memset(oo, 0, sizeof(*oo)); + return sizeof(*oo); + } + case FUSE_READ: { + struct fuse_read_in *ri = (void *)hdr->op_in; + const char *src; + size_t src_sz, sz; + + if (ih->nodeid == INODE_HELLO) { + src = HELLO_CONTENT; + src_sz = HELLO_LEN; + } else if (ih->nodeid == INODE_TESTFILE) { + src = ds->testfile; + src_sz = ds->testfile_sz; + } else { + return -ENOENT; + } + if (ri->offset >= src_sz) + return 0; + sz = src_sz - ri->offset; + if (sz > ri->size) + sz = ri->size; + memcpy(payload, src + ri->offset, sz); + return (int)sz; + } + case FUSE_WRITE: { + struct fuse_write_in *wi = (void *)hdr->op_in; + struct fuse_write_out *wo; + size_t sz = wi->size, end; + + if (ih->nodeid != INODE_TESTFILE) + return -EACCES; + if (sz > payload_sz) + sz = payload_sz; + end = wi->offset + sz; + if (end > TESTFILE_MAX) + return -ENOSPC; + memcpy(ds->testfile + wi->offset, payload, sz); + if (end > ds->testfile_sz) + ds->testfile_sz = end; + wo = (void *)payload; + memset(wo, 0, sizeof(*wo)); + wo->size = sz; + return sizeof(*wo); + } + case FUSE_READDIR: { + struct fuse_read_in *ri = (void *)hdr->op_in; + size_t bufsz = ri->size, w = 0, n; + + if (ih->nodeid != INODE_ROOT) + return -ENOENT; + if (bufsz > ds->max_payload_sz) + bufsz = ds->max_payload_sz; + if (ri->offset < 1) { + n = add_dirent(payload, bufsz, w, INODE_ROOT, DT_DIR, ".", 1); + if (n) w += n; + } + if (ri->offset < 2) { + n = add_dirent(payload, bufsz, w, INODE_ROOT, DT_DIR, "..", 2); + if (n) w += n; + } + if (ri->offset < 3) { + n = add_dirent(payload, bufsz, w, INODE_HELLO, DT_REG, "hello", 3); + if (n) w += n; + } + if (ri->offset < 4) { + n = add_dirent(payload, bufsz, w, INODE_TESTFILE, DT_REG, "testfile", 4); + if (n) w += n; + } + return (int)w; + } + case FUSE_STATFS: { + struct fuse_statfs_out *so = (void *)payload; + + memset(so, 0, sizeof(*so)); + so->st.bsize = 4096; + so->st.frsize = 4096; + so->st.namelen = 255; + so->st.blocks = 1024; + so->st.bfree = 512; + so->st.bavail = 512; + return sizeof(*so); + } + case FUSE_RELEASE: + case FUSE_RELEASEDIR: + case FUSE_FLUSH: + case FUSE_FSYNC: + case FUSE_FSYNCDIR: + return 0; + case FUSE_DESTROY: + ds->exiting = 1; + return 0; + default: + return -ENOSYS; + } +} + +/* ---- io_uring SQE helpers ---------------------------------------------- */ + +static int sqe_register(struct io_uring *ring, int fd, + struct iovec *iov, int qid, void *user_data) +{ + struct io_uring_sqe *sqe = io_uring_get_sqe(ring); + struct fuse_uring_cmd_req *cr; + + if (!sqe) + return -ENOSPC; + io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, NULL, 0, 0); + sqe->cmd_op = FUSE_IO_URING_CMD_REGISTER; + sqe->addr = (unsigned long)iov; + sqe->len = 2; + sqe->user_data = (uint64_t)(uintptr_t)user_data; + cr = (void *)sqe->cmd; + memset(cr, 0, sizeof(*cr)); + cr->qid = qid; + return 0; +} + +static int sqe_commit_fetch(struct io_uring *ring, int fd, + struct fuse_uring_req_header *hdr, + uint64_t commit_id, int qid, + int32_t error, uint32_t payload_sz, + void *user_data) +{ + struct io_uring_sqe *sqe = io_uring_get_sqe(ring); + struct fuse_uring_cmd_req *cr; + struct fuse_out_header *oh = (void *)hdr->in_out; + + if (!sqe) + return -ENOSPC; + + oh->len = sizeof(*oh); + oh->error = error; + oh->unique = commit_id; + hdr->ring_ent_in_out.commit_id = commit_id; + hdr->ring_ent_in_out.payload_sz = payload_sz; + + io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, NULL, 0, 0); + sqe->cmd_op = FUSE_IO_URING_CMD_COMMIT_AND_FETCH; + sqe->user_data = (uint64_t)(uintptr_t)user_data; + cr = (void *)sqe->cmd; + memset(cr, 0, sizeof(*cr)); + cr->commit_id = commit_id; + cr->qid = qid; + return 0; +} + +/* ---- daemon thread ----------------------------------------------------- */ + +static int daemon_init(struct daemon_state *ds) +{ + char *buf; + char reply[sizeof(struct fuse_out_header) + sizeof(struct fuse_init_out)]; + struct fuse_in_header *ih; + struct fuse_init_in *ii; + struct fuse_out_header *oh; + struct fuse_init_out *io; + ssize_t n; + + buf = malloc(FUSE_MIN_READ_BUFFER); + if (!buf) + return -1; + + n = read(ds->fuse_fd, buf, FUSE_MIN_READ_BUFFER); + if (n < (ssize_t)sizeof(struct fuse_in_header)) { + free(buf); + return -1; + } + ih = (void *)buf; + if (ih->opcode != FUSE_INIT) { + free(buf); + return -1; + } + ii = (void *)(buf + sizeof(*ih)); + + if (!(((uint64_t)ii->flags | ((uint64_t)ii->flags2 << 32)) & FUSE_OVER_IO_URING)) { + free(buf); + return -1; + } + + memset(reply, 0, sizeof(reply)); + oh = (void *)reply; + io = (void *)(reply + sizeof(*oh)); + oh->len = sizeof(reply); + oh->unique = ih->unique; + io->major = FUSE_KERNEL_VERSION; + io->minor = FUSE_KERNEL_MINOR_VERSION; + io->max_readahead = ii->max_readahead; + free(buf); + + io->max_write = TESTFILE_MAX; + io->max_pages = TESTFILE_MAX / 4096; + io->max_background = 16; + io->congestion_threshold = 12; + io->time_gran = 1; + io->flags = FUSE_BIG_WRITES | FUSE_INIT_EXT; + io->flags2 = (uint32_t)(FUSE_OVER_IO_URING >> 32); + + n = write(ds->fuse_fd, reply, sizeof(reply)); + if (n != sizeof(reply)) + return -1; + + ds->max_payload_sz = io->max_write; + if (ds->max_payload_sz < FUSE_MIN_READ_BUFFER) + ds->max_payload_sz = FUSE_MIN_READ_BUFFER; + return 0; +} + +static int daemon_setup_ring(struct daemon_state *ds) +{ + struct io_uring_params params = { .flags = IORING_SETUP_SQE128 }; + int i, ret; + + ret = io_uring_queue_init_params(ds->nr_queues * 2 + 4, + &ds->ring, ¶ms); + if (ret < 0) + return ret; + ds->ring_ready = 1; + + ds->testfile = calloc(1, TESTFILE_MAX); + if (!ds->testfile) + return -ENOMEM; + + ds->entries = calloc(ds->nr_queues, sizeof(*ds->entries)); + if (!ds->entries) + return -ENOMEM; + + for (i = 0; i < ds->nr_queues; i++) { + if (posix_memalign((void **)&ds->entries[i].hdr, 4096, + sizeof(*ds->entries[i].hdr))) + return -ENOMEM; + if (posix_memalign(&ds->entries[i].payload, 4096, + ds->max_payload_sz)) + return -ENOMEM; + memset(ds->entries[i].hdr, 0, sizeof(*ds->entries[i].hdr)); + memset(ds->entries[i].payload, 0, ds->max_payload_sz); + ds->entries[i].iov[0].iov_base = ds->entries[i].hdr; + ds->entries[i].iov[0].iov_len = sizeof(*ds->entries[i].hdr); + ds->entries[i].iov[1].iov_base = ds->entries[i].payload; + ds->entries[i].iov[1].iov_len = ds->max_payload_sz; + + ret = sqe_register(&ds->ring, ds->fuse_fd, + ds->entries[i].iov, i, + (void *)(uintptr_t)i); + if (ret) + return ret; + } + return io_uring_submit(&ds->ring); +} + +static void *daemon_loop(void *arg) +{ + struct daemon_state *ds = arg; + struct io_uring_cqe *cqe; + + if (daemon_init(ds)) + goto out; + if (daemon_setup_ring(ds) < 0) + goto out; + + /* signal test thread that we're ready */ + pthread_mutex_lock(&ds->lock); + ds->ready = 1; + pthread_cond_signal(&ds->cond); + pthread_mutex_unlock(&ds->lock); + + while (!ds->exiting) { + if (io_uring_wait_cqe(&ds->ring, &cqe)) + break; + do { + int qid = (int)(uintptr_t)cqe->user_data; + int ret; + + if (cqe->res < 0) { + io_uring_cqe_seen(&ds->ring, cqe); + if (cqe->res == -ENOTCONN || + cqe->res == -ECONNABORTED) + goto out; + continue; + } + + atomic_fetch_add(&ds->uring_reqs, 1); + + ret = handle_request(ds, ds->entries[qid].hdr, + ds->entries[qid].payload, + ds->entries[qid].hdr->ring_ent_in_out.payload_sz); + + if (ret < 0) { + if (sqe_commit_fetch(&ds->ring, ds->fuse_fd, + ds->entries[qid].hdr, + ds->entries[qid].hdr->ring_ent_in_out.commit_id, + qid, ret, 0, + (void *)(uintptr_t)qid)) + goto out; + } else { + if (sqe_commit_fetch(&ds->ring, ds->fuse_fd, + ds->entries[qid].hdr, + ds->entries[qid].hdr->ring_ent_in_out.commit_id, + qid, 0, ret, + (void *)(uintptr_t)qid)) + goto out; + } + + io_uring_cqe_seen(&ds->ring, cqe); + } while (io_uring_peek_cqe(&ds->ring, &cqe) == 0); + + io_uring_submit(&ds->ring); + } +out: + return NULL; +} + +/* ---- daemon lifecycle (used by fixtures) ------------------------------- */ + +static int daemon_start(struct daemon_state *ds, const char *mountpoint) +{ + char opts[256]; + + memset(ds, 0, sizeof(*ds)); + pthread_mutex_init(&ds->lock, NULL); + pthread_cond_init(&ds->cond, NULL); + ds->nr_queues = sysconf(_SC_NPROCESSORS_CONF); + if (ds->nr_queues > 64) + ds->nr_queues = 64; + + ds->fuse_fd = open("/dev/fuse", O_RDWR | O_CLOEXEC); + if (ds->fuse_fd < 0) + return -errno; + + snprintf(opts, sizeof(opts), + "fd=%d,rootmode=40000,user_id=%u,group_id=%u", + ds->fuse_fd, getuid(), getgid()); + if (mount("fuse_uring_test", mountpoint, "fuse", + MS_NOSUID | MS_NODEV, opts)) { + close(ds->fuse_fd); + ds->fuse_fd = -1; + return -errno; + } + + return 0; +} + +static int daemon_wait_ready(struct daemon_state *ds, const char *mountpoint, + pthread_t *thread) +{ + struct timespec ts; + int err; + + if (pthread_create(thread, NULL, daemon_loop, ds)) + return -errno; + + pthread_mutex_lock(&ds->lock); + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 10; + err = 0; + while (!ds->ready && !err) + err = pthread_cond_timedwait(&ds->cond, &ds->lock, &ts); + pthread_mutex_unlock(&ds->lock); + + if (!ds->ready) { + /* Thread was created but daemon failed; clean up */ + ds->exiting = 1; + umount2(mountpoint, MNT_DETACH); + pthread_join(*thread, NULL); + close(ds->fuse_fd); + ds->fuse_fd = -1; + return -ETIMEDOUT; + } + return 0; +} + +static void daemon_stop(struct daemon_state *ds, const char *mountpoint, + pthread_t thread) +{ + int i; + + umount2(mountpoint, MNT_DETACH); + pthread_join(thread, NULL); + if (ds->ring_ready) + io_uring_queue_exit(&ds->ring); + if (ds->entries) { + for (i = 0; i < ds->nr_queues; i++) { + free(ds->entries[i].hdr); + free(ds->entries[i].payload); + } + free(ds->entries); + } + free(ds->testfile); + if (ds->fuse_fd >= 0) + close(ds->fuse_fd); + pthread_mutex_destroy(&ds->lock); + pthread_cond_destroy(&ds->cond); +} + +/* ---- prerequisites and kernel parameter management --------------------- */ + +static int orig_uring_disabled = -1; /* -1 = not saved */ +static int orig_enable_uring = -1; /* -1 = not saved */ + +static int read_sysctl_int(const char *path) +{ + FILE *f; + int val = -1; + + f = fopen(path, "r"); + if (f) { + if (fscanf(f, "%d", &val) != 1) + val = -1; + fclose(f); + } + return val; +} + +static void write_sysctl(const char *path, const char *val) +{ + FILE *f; + + f = fopen(path, "w"); + if (f) { + fputs(val, f); + fclose(f); + } +} + +static int ensure_io_uring_enabled(void) +{ + int val; + + /* enable io_uring if kernel restricts it */ + val = read_sysctl_int("/proc/sys/kernel/io_uring_disabled"); + if (val > 0) { + if (orig_uring_disabled < 0) + orig_uring_disabled = val; + write_sysctl("/proc/sys/kernel/io_uring_disabled", "0"); + if (read_sysctl_int("/proc/sys/kernel/io_uring_disabled") != 0) + return -1; + } + + return 0; +} + +static int ensure_fuse_uring_enabled(void) +{ + FILE *f; + char val; + + if (access("/dev/fuse", R_OK | W_OK)) + return -1; + + f = fopen("/sys/module/fuse/parameters/enable_uring", "r"); + if (!f) + return -1; + val = fgetc(f); + fclose(f); + + if (val != 'Y') { + if (orig_enable_uring < 0) + orig_enable_uring = 0; + write_sysctl("/sys/module/fuse/parameters/enable_uring", "1"); + /* re-check */ + f = fopen("/sys/module/fuse/parameters/enable_uring", "r"); + if (!f) + return -1; + val = fgetc(f); + fclose(f); + if (val != 'Y') + return -1; + } + + return 0; +} + +static void restore_kernel_params(void) +{ + if (orig_enable_uring >= 0) { + write_sysctl("/sys/module/fuse/parameters/enable_uring", + orig_enable_uring ? "1" : "0"); + orig_enable_uring = -1; + } + if (orig_uring_disabled >= 0) { + char buf[16]; + + snprintf(buf, sizeof(buf), "%d", orig_uring_disabled); + write_sysctl("/proc/sys/kernel/io_uring_disabled", buf); + orig_uring_disabled = -1; + } +} + +/* ---- post-test health helpers ------------------------------------------ */ + +/* + * Count entries under /sys/fs/fuse/connections — each corresponds to + * an active fuse_conn. Used to detect connection leaks after teardown. + */ +static int count_fuse_connections(void) +{ + DIR *d = opendir("/sys/fs/fuse/connections"); + struct dirent *de; + int n = 0; + + if (!d) + return -1; + while ((de = readdir(d)) != NULL) { + if (de->d_name[0] == '.') + continue; + n++; + } + closedir(d); + return n; +} + +/* + * Time-bounded umount. Retries every 50 ms up to deadline_ms total. + * Returns 0 on success (or already unmounted), -1 on timeout. + */ +static int umount_with_deadline(const char *mp, unsigned int deadline_ms) +{ + unsigned int slept_ms = 0; + + while (slept_ms <= deadline_ms) { + int rc = umount2(mp, MNT_DETACH); + + if (rc == 0) + return 0; + if (errno == EINVAL || errno == ENOENT) + return 0; /* not mounted — treat as success */ + usleep(50000); + slept_ms += 50; + } + return -1; +} + +/* ---- kselftest fixtures ------------------------------------------------ */ + +FIXTURE(fuse_uring) { + struct daemon_state ds; + char mountpoint[MP_LEN]; + pthread_t thread; + int thread_started; + int conn_baseline; +}; + +FIXTURE_SETUP(fuse_uring) +{ + static int atexit_registered; + + if (getuid() != 0) + SKIP(return, "must run as root"); + if (!atexit_registered) { + atexit(restore_kernel_params); + atexit_registered = 1; + } + if (ensure_io_uring_enabled()) + SKIP(return, "cannot enable io_uring"); + if (ensure_fuse_uring_enabled()) + SKIP(return, "FUSE io-uring not available"); + + strcpy(self->mountpoint, "/tmp/fuse_uring_XXXXXX"); + if (!mkdtemp(self->mountpoint)) + SKIP(return, "mkdtemp: %s", strerror(errno)); + + if (daemon_start(&self->ds, self->mountpoint)) { + rmdir(self->mountpoint); + SKIP(return, "daemon_start: %s", strerror(errno)); + } + if (daemon_wait_ready(&self->ds, self->mountpoint, &self->thread)) { + rmdir(self->mountpoint); + SKIP(return, "daemon did not become ready"); + } + self->thread_started = 1; + self->conn_baseline = count_fuse_connections(); + if (self->conn_baseline < 0) + self->conn_baseline = 0; +} + +FIXTURE_TEARDOWN(fuse_uring) +{ + if (!self->thread_started) + return; + daemon_stop(&self->ds, self->mountpoint, self->thread); + rmdir(self->mountpoint); +} + +/* ---- tests ------------------------------------------------------------- */ + +TEST_F(fuse_uring, read_file) +{ + char buf[256]; + char path[PATH_LEN]; + int fd, n; + + snprintf(path, sizeof(path), "%s/hello", self->mountpoint); + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + n = read(fd, buf, sizeof(buf)); + close(fd); + ASSERT_EQ(n, (int)HELLO_LEN); + buf[n] = '\0'; + ASSERT_STREQ(buf, HELLO_CONTENT); +} + +TEST_F(fuse_uring, write_and_readback) +{ + char path[PATH_LEN]; + const char *msg = "io-uring write test"; + char buf[256]; + int fd, n; + + snprintf(path, sizeof(path), "%s/testfile", self->mountpoint); + + fd = open(path, O_WRONLY); + ASSERT_GE(fd, 0); + n = write(fd, msg, strlen(msg)); + close(fd); + ASSERT_EQ(n, (int)strlen(msg)); + + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + n = read(fd, buf, sizeof(buf)); + close(fd); + ASSERT_EQ(n, (int)strlen(msg)); + buf[n] = '\0'; + ASSERT_STREQ(buf, msg); +} + +TEST_F(fuse_uring, readdir) +{ + DIR *d; + struct dirent *de; + int found_hello = 0, found_testfile = 0; + + d = opendir(self->mountpoint); + ASSERT_NE(d, NULL); + while ((de = readdir(d)) != NULL) { + if (!strcmp(de->d_name, "hello")) + found_hello = 1; + if (!strcmp(de->d_name, "testfile")) + found_testfile = 1; + } + closedir(d); + ASSERT_EQ(found_hello, 1); + ASSERT_EQ(found_testfile, 1); +} + +TEST_F(fuse_uring, stat_files) +{ + struct stat st; + char path[PATH_LEN]; + + ASSERT_EQ(stat(self->mountpoint, &st), 0); + ASSERT_TRUE(S_ISDIR(st.st_mode)); + + snprintf(path, sizeof(path), "%s/hello", self->mountpoint); + ASSERT_EQ(stat(path, &st), 0); + ASSERT_TRUE(S_ISREG(st.st_mode)); + ASSERT_EQ(st.st_size, (off_t)HELLO_LEN); +} + +struct conc_ctx { + const char *mountpoint; + int id; +}; + +static void *concurrent_worker(void *arg) +{ + struct conc_ctx *ctx = arg; + char path[PATH_LEN]; + int i; + + for (i = 0; i < 10; i++) { + char buf[128]; + int fd; + + /* readers hit /hello */ + snprintf(path, sizeof(path), "%s/hello", ctx->mountpoint); + fd = open(path, O_RDONLY); + if (fd >= 0) { + read(fd, buf, sizeof(buf)); + close(fd); + } + + /* writers hit /testfile */ + snprintf(path, sizeof(path), "%s/testfile", ctx->mountpoint); + fd = open(path, O_WRONLY); + if (fd >= 0) { + snprintf(buf, sizeof(buf), "thread%d iter%d", ctx->id, i); + write(fd, buf, strlen(buf)); + close(fd); + } + } + return NULL; +} + +#define CONC_THREADS 4 + +TEST_F(fuse_uring, concurrent_io) +{ + pthread_t threads[CONC_THREADS]; + struct conc_ctx ctxs[CONC_THREADS]; + int i; + + for (i = 0; i < CONC_THREADS; i++) { + ctxs[i].mountpoint = self->mountpoint; + ctxs[i].id = i; + pthread_create(&threads[i], NULL, concurrent_worker, &ctxs[i]); + } + for (i = 0; i < CONC_THREADS; i++) + pthread_join(threads[i], NULL); + + /* verify filesystem still works after concurrent access */ + { + char path[PATH_LEN]; + char buf[128]; + int fd, n; + + snprintf(path, sizeof(path), "%s/hello", self->mountpoint); + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + n = read(fd, buf, sizeof(buf)); + close(fd); + ASSERT_EQ(n, (int)HELLO_LEN); + } +} + +/* + * Per-operation io_uring path verification. Reset the daemon's + * request counter before each distinct operation type and verify it + * incremented afterwards. If the kernel silently falls back to + * /dev/fuse for any opcode, the counter stays at zero for that op. + */ +TEST_F(fuse_uring, requests_via_uring) +{ + char path[PATH_LEN]; + char buf[256]; + int fd, n, before, after; + DIR *d; + struct dirent *de; + struct stat st; + + /* LOOKUP + OPEN + READ + RELEASE (via open+read+close) */ + atomic_store(&self->ds.uring_reqs, 0); + snprintf(path, sizeof(path), "%s/hello", self->mountpoint); + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + n = read(fd, buf, sizeof(buf)); + close(fd); + ASSERT_EQ(n, (int)HELLO_LEN); + after = atomic_load(&self->ds.uring_reqs); + ASSERT_GT(after, 0); + TH_LOG("read path: %d io_uring requests (LOOKUP+OPEN+READ+RELEASE)", + after); + + /* WRITE (via open+write+close on testfile) */ + before = atomic_load(&self->ds.uring_reqs); + snprintf(path, sizeof(path), "%s/testfile", self->mountpoint); + fd = open(path, O_WRONLY); + ASSERT_GE(fd, 0); + n = write(fd, "verify", 6); + close(fd); + ASSERT_EQ(n, 6); + after = atomic_load(&self->ds.uring_reqs); + ASSERT_GT(after, before); + TH_LOG("write path: %d new io_uring requests", after - before); + + /* READDIR (via opendir+readdir+closedir) */ + before = atomic_load(&self->ds.uring_reqs); + d = opendir(self->mountpoint); + ASSERT_NE(d, NULL); + while ((de = readdir(d)) != NULL) + ; + closedir(d); + after = atomic_load(&self->ds.uring_reqs); + ASSERT_GT(after, before); + TH_LOG("readdir path: %d new io_uring requests", after - before); + + /* GETATTR (via stat) */ + before = atomic_load(&self->ds.uring_reqs); + snprintf(path, sizeof(path), "%s/hello", self->mountpoint); + ASSERT_EQ(stat(path, &st), 0); + after = atomic_load(&self->ds.uring_reqs); + ASSERT_GT(after, before); + TH_LOG("stat path: %d new io_uring requests", after - before); + + TH_LOG("total io_uring requests: %d", after); +} + +/* + * Sustained I/O: write 256KB in 4KB chunks with a known pattern, then + * read the entire file back and verify every byte. Exercises payload + * buffer reuse across many commit/fetch cycles and catches corruption + * in the io_uring request pipeline. + */ +TEST_F(fuse_uring, sustained_io) +{ + char path[PATH_LEN]; + char wbuf[4096], rbuf[4096]; + int fd, total, off, n, i; + int write_sz = TESTFILE_MAX; + + snprintf(path, sizeof(path), "%s/testfile", self->mountpoint); + + /* write phase: fill file with pattern */ + fd = open(path, O_WRONLY); + ASSERT_GE(fd, 0); + for (off = 0; off < write_sz; off += sizeof(wbuf)) { + /* fill with offset-dependent pattern */ + for (i = 0; i < (int)sizeof(wbuf); i++) + wbuf[i] = (char)((off + i) & 0xff); + n = write(fd, wbuf, sizeof(wbuf)); + ASSERT_EQ(n, (int)sizeof(wbuf)); + } + close(fd); + + /* read phase: verify every byte */ + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + total = 0; + while (total < write_sz) { + n = read(fd, rbuf, sizeof(rbuf)); + ASSERT_GT(n, 0); + for (i = 0; i < n; i++) { + if (rbuf[i] != (char)((total + i) & 0xff)) { + TH_LOG("data mismatch at offset %d: " + "expected 0x%02x got 0x%02x", + total + i, + (total + i) & 0xff, + (unsigned char)rbuf[i]); + ASSERT_EQ(rbuf[i], + (char)((total + i) & 0xff)); + } + } + total += n; + } + close(fd); + ASSERT_EQ(total, write_sz); + TH_LOG("sustained I/O: wrote and verified %d bytes in 4KB chunks", + write_sz); +} + +/* + * Background requests: fill a file, then read it sequentially in small + * chunks to trigger kernel readahead. Readahead generates background + * FUSE requests that flow through fuse_uring_queue_bq_req() and + * fuse_uring_flush_bg() — the code path fixed by 31da059891bd (bg dispatch + * ordering). Verify that all data arrives correctly and that the + * io_uring request count exceeds the minimum expected from foreground + * requests alone, indicating background requests were dispatched. + */ +TEST_F(fuse_uring, background_requests) +{ + char path[PATH_LEN]; + char wbuf[4096], rbuf[512]; + int fd, n, i, off, total; + int before, after, read_calls; + int write_sz = TESTFILE_MAX; + + snprintf(path, sizeof(path), "%s/testfile", self->mountpoint); + + /* fill the file with a known pattern */ + fd = open(path, O_WRONLY); + ASSERT_GE(fd, 0); + for (off = 0; off < write_sz; off += sizeof(wbuf)) { + for (i = 0; i < (int)sizeof(wbuf); i++) + wbuf[i] = (char)((off + i) % 251); + n = write(fd, wbuf, sizeof(wbuf)); + ASSERT_EQ(n, (int)sizeof(wbuf)); + } + close(fd); + + /* + * Sequential read in small chunks (512B) to maximise the + * kernel's readahead window. The kernel will issue background + * FUSE_READ requests ahead of what we ask for. + */ + before = atomic_load(&self->ds.uring_reqs); + fd = open(path, O_RDONLY); + ASSERT_GE(fd, 0); + + total = 0; + read_calls = 0; + while (total < write_sz) { + n = read(fd, rbuf, sizeof(rbuf)); + ASSERT_GT(n, 0); + /* verify pattern */ + for (i = 0; i < n; i++) { + if (rbuf[i] != (char)((total + i) % 251)) { + TH_LOG("bg read mismatch at offset %d", + total + i); + ASSERT_EQ(rbuf[i], + (char)((total + i) % 251)); + } + } + total += n; + read_calls++; + } + close(fd); + after = atomic_load(&self->ds.uring_reqs); + + ASSERT_EQ(total, write_sz); + + /* + * The io_uring request count should reflect both the foreground + * reads and any background readahead the kernel issued. At + * minimum we expect LOOKUP + OPEN + reads + RELEASE. + */ + TH_LOG("background requests: %d read() calls for %d bytes, " + "%d io_uring requests (delta from before)", + read_calls, write_sz, after - before); + ASSERT_GT(after - before, 0); +} + +/* + * Crash recovery: fork a daemon process, drive active I/O against it + * from multiple threads, then SIGKILL the daemon while requests are + * in flight. This exercises the teardown paths that race with + * io_uring task work (the bug class fixed by bea4fe98204b, 952b5d36f6a2, + * 198f45eeb9f7). + */ + +static atomic_int crash_io_running; + +static void *crash_io_worker(void *arg) +{ + const char *mp = arg; + char path[PATH_LEN]; + char buf[128]; + + snprintf(path, sizeof(path), "%s/hello", mp); + while (crash_io_running) { + int fd = open(path, O_RDONLY); + + if (fd >= 0) { + read(fd, buf, sizeof(buf)); + close(fd); + } + snprintf(path, sizeof(path), "%s/testfile", mp); + fd = open(path, O_WRONLY); + if (fd >= 0) { + write(fd, "crash", 5); + close(fd); + } + snprintf(path, sizeof(path), "%s/hello", mp); + } + return NULL; +} + +#define CRASH_IO_THREADS 4 + +TEST_F(fuse_uring, crash_recovery) +{ + char crash_mp[MP_LEN]; + pid_t pid; + int status; + + strcpy(crash_mp, "/tmp/fuse_crash_XXXXXX"); + if (!mkdtemp(crash_mp)) + SKIP(return, "mkdtemp: %s", strerror(errno)); + + pid = fork(); + ASSERT_GE(pid, 0); + + if (pid == 0) { + /* child: run a daemon, handle requests until killed */ + struct daemon_state cds; + pthread_t ct; + + if (daemon_start(&cds, crash_mp)) + _exit(1); + if (daemon_wait_ready(&cds, crash_mp, &ct)) + _exit(1); + pthread_join(ct, NULL); + _exit(0); + } + + /* parent: wait for child's filesystem to appear */ + { + char path[PATH_LEN]; + int fd, i; + + snprintf(path, sizeof(path), "%s/hello", crash_mp); + for (i = 0; i < 50; i++) { + fd = open(path, O_RDONLY); + if (fd >= 0) { + close(fd); + break; + } + usleep(100000); + } + } + + /* start I/O threads hammering the filesystem */ + { + pthread_t io_threads[CRASH_IO_THREADS]; + int i; + + crash_io_running = 1; + for (i = 0; i < CRASH_IO_THREADS; i++) + pthread_create(&io_threads[i], NULL, + crash_io_worker, crash_mp); + + /* let I/O build up, then kill daemon mid-flight */ + usleep(200000); + kill(pid, SIGKILL); + waitpid(pid, &status, 0); + + /* stop I/O threads (they'll get errors now) */ + crash_io_running = 0; + for (i = 0; i < CRASH_IO_THREADS; i++) + pthread_join(io_threads[i], NULL); + } + + /* + * Bounded umount: if async teardown is stuck, this will + * time out rather than silently leaking the connection. + */ + { + int rc, final_conns; + + rc = umount_with_deadline(crash_mp, 6000); + ASSERT_EQ(rc, 0) { + TH_LOG("crash_mp umount timed out after 6s " + "(async teardown may be stuck)"); + } + rmdir(crash_mp); + + /* grace period for delayed_work to drain */ + usleep(200000); + + final_conns = count_fuse_connections(); + TH_LOG("crash_recovery: conns %d -> %d (baseline %d)", + self->conn_baseline, final_conns, + self->conn_baseline); + ASSERT_LE(final_conns, self->conn_baseline + 1); + } + + /* + * The kernel should be healthy. Verify by doing an operation + * on the main fixture's filesystem. + */ + { + struct stat st; + + ASSERT_EQ(stat(self->mountpoint, &st), 0); + TH_LOG("kernel healthy after daemon crash with active I/O"); + } +} + +/* + * Abort before ring ready: negotiate FUSE_OVER_IO_URING in FUSE_INIT + * but never register any io_uring entries, then abort the connection. + * Exercises the deadlock fix (d55011469b41) where blocked allocators would + * hang forever waiting for a ring that never becomes ready, and the + * abort-during-creation fix (c146284c4355) where rings created after abort + * would leak. + */ +TEST_F(fuse_uring, abort_before_ring_ready) +{ + char abort_mp[MP_LEN]; + pid_t pid; + int status; + struct stat st; + + strcpy(abort_mp, "/tmp/fuse_abort_XXXXXX"); + if (!mkdtemp(abort_mp)) + SKIP(return, "mkdtemp: %s", strerror(errno)); + + pid = fork(); + ASSERT_GE(pid, 0); + + if (pid == 0) { + /* + * Child: open /dev/fuse, mount, do FUSE_INIT with + * FUSE_OVER_IO_URING, but never register io_uring + * entries. The ring never becomes ready. Sleep + * until killed — the kernel must handle teardown of + * an io_uring-negotiated connection where the ring + * was never initialised without deadlocking. + */ + struct daemon_state ds; + + memset(&ds, 0, sizeof(ds)); + if (daemon_start(&ds, abort_mp)) + _exit(1); + if (daemon_init(&ds)) + _exit(1); + pause(); + _exit(0); + } + + /* + * Give the child time to complete FUSE_INIT, then kill it. + * This tears down the FUSE connection while fc->io_uring is + * set but the ring was never readied. On a buggy kernel this + * deadlocks in fuse_block_alloc() or leaks resources. + */ + usleep(500000); + kill(pid, SIGKILL); + waitpid(pid, &status, 0); + + { + int rc; + + rc = umount_with_deadline(abort_mp, 6000); + ASSERT_EQ(rc, 0) { + TH_LOG("abort_mp umount timed out after 6s"); + } + rmdir(abort_mp); + } + + /* verify kernel is still healthy */ + ASSERT_EQ(stat(self->mountpoint, &st), 0); + TH_LOG("kernel healthy after teardown of uninitialized io_uring ring"); +} + +#define REG_RACE_CHILDREN 16 + +/* + * Regression test for the race classes fixed by: + * 952b5d36f6a2 fuse-uring: fix race between registration and + * connection abortion + * b70a3aca1693 fuse-uring: Avoid queue->stopped races and set/read + * that value under lock + * d351da750669 fuse-uring: Avoid use-after-free in + * fuse_uring_async_stop_queues (CVE-2026-64261) + * + * Forks N children performing FUSE_INIT with FUSE_OVER_IO_URING + * negotiated but delaying entry registration. Parent kills them + * at staggered offsets — some during FUSE_INIT, others after + * negotiation while in pause(). The crucial window is between + * fuse_uring_create() running under fc->lock and fuse_abort_conn() + * clearing fc->connected. + * + * Delays are deterministic (child_index * 2.5ms) for reproducibility. + * Seed logged via TH_LOG for reruns. + */ +TEST_F(fuse_uring, teardown_under_concurrent_registration) +{ + char mp_bufs[REG_RACE_CHILDREN][MP_LEN]; + pid_t pids[REG_RACE_CHILDREN]; + int i, status, initial_conns, final_conns; + unsigned int seed = (unsigned int)getpid(); + + srandom(seed); + TH_LOG("registration-race seed=%u", seed); + + initial_conns = count_fuse_connections(); + if (initial_conns < 0) + initial_conns = 0; + + for (i = 0; i < REG_RACE_CHILDREN; i++) { + snprintf(mp_bufs[i], sizeof(mp_bufs[i]), + "/tmp/fuse_regrace_%d_XXXXXX", i); + if (!mkdtemp(mp_bufs[i])) { + int j; + + for (j = 0; j < i; j++) { + kill(pids[j], SIGKILL); + waitpid(pids[j], &status, 0); + umount_with_deadline(mp_bufs[j], 1000); + rmdir(mp_bufs[j]); + } + SKIP(return, "mkdtemp: %s", strerror(errno)); + } + + pids[i] = fork(); + ASSERT_GE(pids[i], 0); + + if (pids[i] == 0) { + struct daemon_state ds; + + memset(&ds, 0, sizeof(ds)); + if (daemon_start(&ds, mp_bufs[i])) + _exit(1); + if (daemon_init(&ds)) + _exit(1); + /* + * Deliberately skip ring registration. + * Enter pause() with fc->io_uring=1 but ring + * never initialized. + */ + usleep(i * 2500); /* deterministic stagger */ + pause(); + _exit(0); + } + } + + /* Stage 1: kill even-indexed children early */ + usleep(10000); + for (i = 0; i < REG_RACE_CHILDREN; i += 2) + kill(pids[i], SIGKILL); + + /* Stage 2: kill odd-indexed children slightly later */ + usleep(20000); + for (i = 1; i < REG_RACE_CHILDREN; i += 2) + kill(pids[i], SIGKILL); + + for (i = 0; i < REG_RACE_CHILDREN; i++) + waitpid(pids[i], &status, 0); + + /* Bounded umount for all children */ + { + int leaked = 0; + + for (i = 0; i < REG_RACE_CHILDREN; i++) { + if (umount_with_deadline(mp_bufs[i], 6000) != 0) + leaked++; + rmdir(mp_bufs[i]); + } + ASSERT_EQ(leaked, 0) { + TH_LOG("%d child mounts would not release", leaked); + } + } + + /* Wait for async teardown delayed_work to drain */ + usleep(6000000); + + final_conns = count_fuse_connections(); + TH_LOG("registration race: %d children, conns %d -> %d", + REG_RACE_CHILDREN, initial_conns, final_conns); + ASSERT_EQ(final_conns, initial_conns); + + /* Main fixture mount must still be healthy */ + { + struct stat st; + + ASSERT_EQ(stat(self->mountpoint, &st), 0); + } +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/filesystems/fuse/fusectl_test.c b/tools/testing/selftests/filesystems/fuse/fusectl_test.c new file mode 100644 index 0000000000000..8d124d1cacb26 --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/fusectl_test.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (c) 2025 Chen Linxuan + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest_harness.h" + +#define FUSECTL_MOUNTPOINT "/sys/fs/fuse/connections" +#define FUSE_MOUNTPOINT "/tmp/fuse_mnt_XXXXXX" +#define FUSE_DEVICE "/dev/fuse" +#define FUSECTL_TEST_VALUE "1" + +static void write_file(struct __test_metadata *const _metadata, + const char *path, const char *val) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(val); + + ASSERT_GE(fd, 0); + ASSERT_EQ(write(fd, val, len), len); + ASSERT_EQ(close(fd), 0); +} + +FIXTURE(fusectl){ + char fuse_mountpoint[sizeof(FUSE_MOUNTPOINT)]; + int connection; +}; + +FIXTURE_SETUP(fusectl) +{ + const char *fuse_mnt_prog = "./fuse_mnt"; + int status, pid; + struct stat statbuf; + uid_t uid = getuid(); + gid_t gid = getgid(); + char buf[32]; + + /* Setup userns */ + ASSERT_EQ(unshare(CLONE_NEWNS|CLONE_NEWUSER), 0); + sprintf(buf, "0 %d 1", uid); + write_file(_metadata, "/proc/self/uid_map", buf); + write_file(_metadata, "/proc/self/setgroups", "deny"); + sprintf(buf, "0 %d 1", gid); + write_file(_metadata, "/proc/self/gid_map", buf); + ASSERT_EQ(mount("", "/", NULL, MS_REC|MS_PRIVATE, NULL), 0); + + strcpy(self->fuse_mountpoint, FUSE_MOUNTPOINT); + + if (!mkdtemp(self->fuse_mountpoint)) + SKIP(return, + "Failed to create FUSE mountpoint %s", + strerror(errno)); + + if (access(FUSECTL_MOUNTPOINT, F_OK)) + SKIP(return, + "FUSE control filesystem not mounted"); + + pid = fork(); + if (pid < 0) + SKIP(return, + "Failed to fork FUSE daemon process: %s", + strerror(errno)); + + if (pid == 0) { + execlp(fuse_mnt_prog, fuse_mnt_prog, self->fuse_mountpoint, NULL); + exit(errno); + } + + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + SKIP(return, + "Failed to start FUSE daemon %s", + strerror(WEXITSTATUS(status))); + } + + if (stat(self->fuse_mountpoint, &statbuf)) + SKIP(return, + "Failed to stat FUSE mountpoint %s", + strerror(errno)); + + self->connection = statbuf.st_dev; +} + +FIXTURE_TEARDOWN(fusectl) +{ + umount2(self->fuse_mountpoint, MNT_DETACH); + rmdir(self->fuse_mountpoint); +} + +TEST_F(fusectl, abort) +{ + char path_buf[PATH_MAX]; + int abort_fd, test_fd, ret; + + sprintf(path_buf, "/sys/fs/fuse/connections/%d/abort", self->connection); + + ASSERT_EQ(0, access(path_buf, F_OK)); + + abort_fd = open(path_buf, O_WRONLY); + ASSERT_GE(abort_fd, 0); + + sprintf(path_buf, "%s/test", self->fuse_mountpoint); + + test_fd = open(path_buf, O_RDWR); + ASSERT_GE(test_fd, 0); + + ret = read(test_fd, path_buf, sizeof(path_buf)); + ASSERT_EQ(ret, 0); + + ret = write(test_fd, "test", sizeof("test")); + ASSERT_EQ(ret, sizeof("test")); + + ret = lseek(test_fd, 0, SEEK_SET); + ASSERT_GE(ret, 0); + + ret = write(abort_fd, FUSECTL_TEST_VALUE, sizeof(FUSECTL_TEST_VALUE)); + ASSERT_GT(ret, 0); + + close(abort_fd); + + ret = read(test_fd, path_buf, sizeof(path_buf)); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, ENOTCONN); +} + +TEST_HARNESS_MAIN