diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79e2f945ba29..1fb4045f2805 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -74,7 +74,7 @@ jobs: go-version: ["1.25", "1.26"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index f7c994e4b6c4..343404a00c4e 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -30,7 +30,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -467,7 +467,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -553,7 +553,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - name: consider (pre-job) debugging @@ -612,7 +612,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -672,7 +672,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -738,7 +738,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -798,7 +798,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -863,7 +863,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -975,7 +975,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1035,7 +1035,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1106,7 +1106,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1210,7 +1210,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1272,7 +1272,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1344,7 +1344,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1441,7 +1441,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1518,7 +1518,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1583,7 +1583,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1874,7 +1874,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -1982,7 +1982,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -2050,7 +2050,7 @@ jobs: ceph-image: ${{ fromJson(inputs.ceph_images) }} kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -2088,7 +2088,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -2177,7 +2177,7 @@ jobs: kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/checkmake.yaml b/.github/workflows/checkmake.yaml index 09466e1258fe..4d508be2a2a9 100644 --- a/.github/workflows/checkmake.yaml +++ b/.github/workflows/checkmake.yaml @@ -18,7 +18,7 @@ jobs: name: CheckMake runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Run CheckMake uses: Uno-Takashi/checkmake-action@bc11ee86274ceaf5710dbcd80871d7cd7ecc78ce # v2 with: diff --git a/.github/workflows/codegen.yml b/.github/workflows/codegen.yml index bdb3552146c1..64ce7b338624 100644 --- a/.github/workflows/codegen.yml +++ b/.github/workflows/codegen.yml @@ -30,7 +30,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/codespell.yaml b/.github/workflows/codespell.yaml index 8af5904c4403..04362d5ef79d 100644 --- a/.github/workflows/codespell.yaml +++ b/.github/workflows/codespell.yaml @@ -24,7 +24,7 @@ jobs: name: codespell runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - name: codespell @@ -53,7 +53,7 @@ jobs: name: misspell runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - name: misspell diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 89e4f0fc4021..c95b1234af33 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -28,7 +28,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - uses: wagoid/commitlint-github-action@b948419dd99f3fd78a6548d48f94e3df7f6bf3ed # v6.2.1 diff --git a/.github/workflows/crds-gen.yml b/.github/workflows/crds-gen.yml index f95ab8e0e4f5..61dc63dacb4b 100644 --- a/.github/workflows/crds-gen.yml +++ b/.github/workflows/crds-gen.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index 585ea04ca2a2..0073a256f9ae 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -155,7 +155,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -235,7 +235,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -275,7 +275,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 16b0fd57b15a..4e178ed858e2 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -24,7 +24,7 @@ jobs: name: docs-check runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index f14ee847ed19..30e5c315e936 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -24,7 +24,7 @@ jobs: name: golangci-lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -57,7 +57,7 @@ jobs: name: kube-api-lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 760047c8c8cf..67df7852dd3e 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml index d0b5b2905515..1db4325028b6 100644 --- a/.github/workflows/integration-test-helm-suite.yaml +++ b/.github/workflows/integration-test-helm-suite.yaml @@ -38,7 +38,7 @@ jobs: kubernetes-version: ["v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml index 48482eeede37..2e14f8d06919 100644 --- a/.github/workflows/integration-test-keystone-auth-suite.yaml +++ b/.github/workflows/integration-test-keystone-auth-suite.yaml @@ -37,7 +37,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml index 5fe8e33bbeb9..1c9c0d40d843 100644 --- a/.github/workflows/integration-test-mgr-suite.yaml +++ b/.github/workflows/integration-test-mgr-suite.yaml @@ -36,7 +36,7 @@ jobs: kubernetes-version: ["v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml index 8d9395bd8c7f..b0bb93483557 100644 --- a/.github/workflows/integration-test-multi-cluster-suite.yaml +++ b/.github/workflows/integration-test-multi-cluster-suite.yaml @@ -37,7 +37,7 @@ jobs: kubernetes-version: ["v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml index 6cc3c186a2b1..4a2cadcdf03e 100644 --- a/.github/workflows/integration-test-object-suite.yaml +++ b/.github/workflows/integration-test-object-suite.yaml @@ -37,7 +37,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml index 44f97d3b2295..05efefe2c137 100644 --- a/.github/workflows/integration-test-smoke-suite.yaml +++ b/.github/workflows/integration-test-smoke-suite.yaml @@ -37,7 +37,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml index 058b5215f9c9..695f1b531913 100644 --- a/.github/workflows/integration-test-upgrade-suite.yaml +++ b/.github/workflows/integration-test-upgrade-suite.yaml @@ -37,7 +37,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -86,7 +86,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml index 91658c8618f6..cab97193fe7d 100644 --- a/.github/workflows/integration-tests-on-release.yaml +++ b/.github/workflows/integration-tests-on-release.yaml @@ -30,7 +30,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.32.13", "v1.33.12", "1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -69,7 +69,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.32.13", "v1.33.12", "1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -110,7 +110,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.32.13", "v1.33.12", "1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -148,7 +148,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.32.13", "v1.33.12", "1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -186,7 +186,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.32.13", "v1.33.12", "1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -225,7 +225,7 @@ jobs: kubernetes-version: ["v1.31.14", "v1.35.5"] steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index 27b70df8b263..edc322564de6 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -23,7 +23,7 @@ jobs: yaml-linter: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 @@ -33,7 +33,7 @@ jobs: pylint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/mod-check.yml b/.github/workflows/mod-check.yml index 19073c12e024..8475d01f631c 100644 --- a/.github/workflows/mod-check.yml +++ b/.github/workflows/mod-check.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml index 6552be4cba9c..18dd735738c9 100644 --- a/.github/workflows/multus.yaml +++ b/.github/workflows/multus.yaml @@ -36,7 +36,7 @@ jobs: NUMBER_OF_COMPUTE_NODES: 5 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/push-build.yaml b/.github/workflows/push-build.yaml index 55574cefb18e..58c692d10620 100644 --- a/.github/workflows/push-build.yaml +++ b/.github/workflows/push-build.yaml @@ -24,7 +24,7 @@ jobs: id-token: write # for cosign OIDC keyless signing steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rbac-gen.yaml b/.github/workflows/rbac-gen.yaml index d4d36e317ed7..111c59313791 100644 --- a/.github/workflows/rbac-gen.yaml +++ b/.github/workflows/rbac-gen.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 8b9fe2d814b4..9eca17cd5466 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -28,7 +28,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false @@ -64,6 +64,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@87557b9c84dde89fdd9b10e88954ac2f4248e463 # v4.36.1 + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: sarif_file: results.sarif diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 77dde304c070..f325e7d273f2 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -24,6 +24,6 @@ jobs: name: Shellcheck runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Lint shell scripts run: make lint.shell diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml index 0a606883e1e9..14ccf65f2ac6 100644 --- a/.github/workflows/snyk.yaml +++ b/.github/workflows/snyk.yaml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 39386cd88caa..ac829152cd6b 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -37,7 +37,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" steps: - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index d9c7ea56c850..5b9f7ade4d7c 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -811,6 +811,28 @@ spec: This configuration will split the replication of volumes across unique racks in the data center setup. +## OSD Device Class via Node Label + +The CRUSH device class for all OSDs on a node can be set using the node label `osd.rook.io/device-class`. This label can be applied retroactively on nodes with provisioned OSDs, or on new nodes about to be added to the cluster. + +For example, if a node has `osd.rook.io/device-class: fast`, OSDs on the node will be assigned the `fast` device class. + +This can help group nodes of the same performance tier together in one pool. + +`allowDeviceClassUpdate: true` must be set in the CephCluster storage spec for the device class to be updated on existing OSDs. Without this setting, the node label will only affect newly provisioned OSDs. + +!!! warning + Changing an OSD's device class will cause Ceph to rebalance data for any pools whose CRUSH rules target that device class. Plan device class changes of existing nodes and OSDs carefully to avoid unexpected data movement. + +### Resolving device classes + +The device class for a node is determined as follows: + +* If the per-node `deviceClass` in the CephCluster CR **and** the `osd.rook.io/device-class` node label are both set, the operator will log an error and **skip** that node. The user must remove one to resolve the conflict. +* If only the CR `deviceClass` is set, it is used. +* If only the node label is set, it is used. +* If neither is set, Ceph auto-detects the class (defaults to `hdd`, `ssd`, or `nvme` based on hardware). + ## Deleting a CephCluster During deletion of a CephCluster resource, Rook protects against accidental or premature destruction diff --git a/Documentation/Helm-Charts/csi-drivers-chart.md b/Documentation/Helm-Charts/csi-drivers-chart.md index 7a09c0761784..659ea5799424 100644 --- a/Documentation/Helm-Charts/csi-drivers-chart.md +++ b/Documentation/Helm-Charts/csi-drivers-chart.md @@ -16,13 +16,15 @@ The `helm install` command deploys the drivers in the default configuration from Ceph-CSI publishes the drivers chart from the `ceph-csi-operator` Helm repository. -!!! important - Install this chart with the recommended values.yaml. The drivers will fail if only configured with the chart defaults. +**IMPORTANT** + +1. Install this chart with the recommended values.yaml. The drivers will fail if only configured with the chart defaults. +2. If installing in another namespace, replace all instances of `rook-ceph` in the values.yaml with the required namespace. ```console helm repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator helm install ceph-csi-drivers --namespace rook-ceph ceph-csi-operator/ceph-csi-drivers \ - -f https://raw.githubusercontent.com/rook/rook/master/deploy/charts/rook-ceph/ceph-csi-drivers/values.yaml + -f https://raw.githubusercontent.com/rook/rook/master/deploy/charts/ceph-csi-drivers/values.yaml ``` ## Custom settings diff --git a/deploy/charts/ceph-csi-drivers/values.yaml b/deploy/charts/ceph-csi-drivers/values.yaml index 1fd5ae6af7b1..1a06620775e4 100644 --- a/deploy/charts/ceph-csi-drivers/values.yaml +++ b/deploy/charts/ceph-csi-drivers/values.yaml @@ -20,13 +20,13 @@ operatorConfig: drivers: rbd: enabled: true - name: rook-ceph.rbd.csi.ceph.com + name: rook-ceph.rbd.csi.ceph.com # csi-provisioner-name cephfs: enabled: true - name: rook-ceph.cephfs.csi.ceph.com + name: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name nfs: enabled: false - name: rook-ceph.nfs.csi.ceph.com + name: rook-ceph.nfs.csi.ceph.com # csi-provisioner-name nvmeof: enabled: false - name: rook-ceph.nvmeof.csi.ceph.com + name: rook-ceph.nvmeof.csi.ceph.com # csi-provisioner-name diff --git a/go.mod b/go.mod index f32bec985fca..1694c8eff819 100644 --- a/go.mod +++ b/go.mod @@ -14,12 +14,12 @@ replace ( require ( github.com/IBM/keyprotect-go-client v0.16.0 - github.com/aws/aws-sdk-go-v2 v1.41.9 - github.com/aws/aws-sdk-go-v2/config v1.32.20 - github.com/aws/aws-sdk-go-v2/credentials v1.19.19 - github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 - github.com/aws/aws-sdk-go-v2/service/sns v1.39.19 - github.com/aws/smithy-go v1.26.0 + github.com/aws/aws-sdk-go-v2 v1.41.12 + github.com/aws/aws-sdk-go-v2/config v1.32.23 + github.com/aws/aws-sdk-go-v2/credentials v1.19.22 + github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2 + github.com/aws/aws-sdk-go-v2/service/sns v1.40.0 + github.com/aws/smithy-go v1.27.1 github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/ceph/ceph-csi-operator/api v0.0.0-20260527105556-70b2f7cfddf0 github.com/ceph/ceph-csi/api v0.0.0-20241216133622-88b7e0d6684f @@ -45,8 +45,8 @@ require ( github.com/stretchr/testify v1.11.1 github.com/sykesm/zap-logfmt v0.0.4 go.uber.org/zap v1.27.1 - golang.org/x/sync v0.20.0 - gopkg.in/ini.v1 v1.67.2 + golang.org/x/sync v0.21.0 + gopkg.in/ini.v1 v1.67.3 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.35.4 k8s.io/apiextensions-apiserver v0.35.4 @@ -71,19 +71,19 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/ansel1/merry v1.8.1 // indirect github.com/ansel1/merry/v2 v2.2.2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index 798ac436719b..7e3b331ed999 100644 --- a/go.sum +++ b/go.sum @@ -488,44 +488,44 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4= -github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= -github.com/aws/aws-sdk-go-v2/config v1.32.20 h1:8VMDnWc/kEzxsI/1ngGM9mG81a8IGmIHD8KLcYGwagc= -github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM= -github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU= -github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 h1:0w6dCiO8iez+YKwRhRBlL1CH/E3GTfdkuzrwj1by8vo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= -github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= -github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 h1:1VwbP3qMNfxUDEXWki4rCE5iA+44VA1lokTz9HasGzw= -github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.19 h1:FFhX5wY9zHX1IzSsqHlcd9TZgejkF5+F/SpvWZcdS+k= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.19/go.mod h1:1L0Y96eKbF+uIfA/m6JagGDBprXP8Bzz7fUjjmVCI7A= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 h1:N6pIsdFOW1Kd9S4KyFKXdGRBojPPxkP32+uHFWLv4Hc= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 h1:hc+lBYiiTr8Zk4MTzIsQ92MeDWCIDvWGmzKUWOaBcOg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 h1:ErklX/7uhSbkAAeyQD/Y1OoQ9hO3SJXQNEgksORW3Js= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8= -github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s= -github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= +github.com/aws/aws-sdk-go-v2/config v1.32.23 h1:PYDobtcsJXK6bQe9I8RQk6s19Bz3xa3xRU08Hy1Em3Y= +github.com/aws/aws-sdk-go-v2/config v1.32.23/go.mod h1:QID4dqUQVgEOYPKsPWd1sNWCCR2c5g7o3jeEtIXPOZU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.22 h1:SHfH6wyPsEgG7fVsi5rQxWEt7tuIcN2PGhb1mTFv6tE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.22/go.mod h1:54nO8lKD4aQPOntM/VTWjnR+DYzTwx0YkSMZMhAgewQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 h1:b+kcDejJrXc30zU/w8Tc9klISwaO5wh+6T0sMBdDoHM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28/go.mod h1:LnI62O9GnSv6GcuLXxOYqlq0C8EmxMcgnF6m7LdYuOY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 h1:VkE9FuzTQVjBBrnj4+oCdxCLFIz7aqLYKUCjtvxVcOs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29/go.mod h1:H32Z2Qth9b+9LqjyBsCnozMQ8H2N7YBUDVXwbs0iggg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21 h1:FsZxbPiVgEHYofziwfylouMki8b1Z7mI4CMU/7bhwBA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21/go.mod h1:Mmm30OV+JLXYQUcbSd84THnv3P5JtjhVDujLwMqRG0U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 h1:axj4mEDletwKmTm/9jR+DkIMmCfcn5vE4jBMAAN+3Vg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28/go.mod h1:3Aaz69M0jqfSHLKqxgolgUBFT4hpwSNc7DzC95orEi8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28 h1:li8rTZAAb22g4UsxbjwMdaNVWbgVcDzPqI7nDTI+mF4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28/go.mod h1:/brXioSGIMEdcBFoubpSdmighSVp6poP+mma/wB7iHA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2 h1:b4ikkRk22T4xYkEgaWc3Voe+3xbt5YbbFhNehOWyUiY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2/go.mod h1:Gp7eHZ0NZ8ZK5RXpoIUp/C8OeAmJqpCgdwEK1D/QOek= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 h1:YcpVyIPLCbiypN6KSphijN5fC7DDjX114SqA7prnnxg= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.4/go.mod h1:5ZICS++oFTRPfa1GsBqFDWX/8WamZ/QQOcCzIuU/zLw= +github.com/aws/aws-sdk-go-v2/service/sns v1.40.0 h1:mAf3EuBF24vGz5IWttC8A6zX/q+5wqwAFeRhB3Nmpik= +github.com/aws/aws-sdk-go-v2/service/sns v1.40.0/go.mod h1:xiP2M3/oc7h8JyhNS4gy/whFAb9NRug4UrEfg91xumY= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.2 h1:ySNWu7TPmj5fKFIa1GYvX+Ddxd5ccruqC20aMNuyWDM= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.2/go.mod h1:A+U9luAOwFeB1kseyWCITVg7/NntoPebCFR9pQ4ch9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5 h1:KSzGGqfk39O+WU3OEyYbx6F7sLDQCqxlOJ+2IksfK6U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5/go.mod h1:ATs88lXDeQB6CZOgQ5BIl9JbYS+EsCWUSDyff6L/oVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 h1:RTO7mmGyedgnNmcPh3yQizNfc6GKoV5iqfdJavuf9vw= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.2/go.mod h1:fBhUZXDin9YYqhcpOMjIcpdik25rVwWyxLdPH1RZd9s= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A= github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -1572,8 +1572,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2079,8 +2079,8 @@ gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaD gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= -gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod index e2b9b2f55e45..f6a3f2c31bd0 100644 --- a/pkg/apis/go.mod +++ b/pkg/apis/go.mod @@ -80,6 +80,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum index 73d9e2efa4aa..bacf1dc18979 100644 --- a/pkg/apis/go.sum +++ b/pkg/apis/go.sum @@ -985,8 +985,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/daemon/ceph/client/deviceclass_pdb.go b/pkg/daemon/ceph/client/deviceclass_pdb.go new file mode 100644 index 000000000000..5fe561959176 --- /dev/null +++ b/pkg/daemon/ceph/client/deviceclass_pdb.go @@ -0,0 +1,229 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "encoding/json" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" +) + +// DeviceClassInfo describes a device class whose pools are confined to a single +// CRUSH device-class shadow tree. +type DeviceClassInfo struct { + // FailureDomainType is the CRUSH failure domain type (host, zone, ...) that + // the class's rules select with their chooseleaf/choose step. + FailureDomainType string + // Pools are the RADOS pool names confined to this device class. + Pools []string +} + +// DeviceClassPDBLayout is the per-device-class view of the cluster derived from +// the live CRUSH map joined to the in-use pools. It is the source of truth for +// the class-aware OSD PodDisruptionBudget path: the CRUSH map, not the CRs, +// decides whether each pool keeps its data inside a single device class. +type DeviceClassPDBLayout struct { + // Classes maps each device class to its pools and failure domain — the device classes + // the in-use pools partition into. It is populated only when every in-use pool maps to + // a single device class; if any pool's data spans device classes (or maps to no + // resolvable failure domain), the pools do not partition by class and Classes is empty. + Classes map[string]*DeviceClassInfo + // FailureDomainTypes holds the failure-domain type each in-use pool's CRUSH rule + // selects. The operator reduces these to the cluster-wide failure domain (the finest + // in the CRUSH hierarchy) for the cluster-wide OSD PDB group. + FailureDomainTypes []string +} + +// poolLsDetailEntry is the slice of an `osd pool ls detail` entry the layout +// needs: the pool name and the numeric id of the crush rule it references. +type poolLsDetailEntry struct { + PoolName string `json:"pool_name"` + CrushRule int `json:"crush_rule"` +} + +// GetDeviceClassPDBLayout reads the live CRUSH map and pool list and reports the device +// classes the in-use pools partition into (empty unless every pool maps to a single +// device class), plus each in-use pool's failure-domain type for the cluster-wide failure +// domain. This single read backs the eligibility gate, the per-class failure-domain types, +// and the per-class pool mapping so they cannot disagree. The whole read is two ceph calls +// regardless of pool count. +func GetDeviceClassPDBLayout(context *clusterd.Context, clusterInfo *ClusterInfo) (*DeviceClassPDBLayout, error) { + crushMap, err := GetCrushMap(context, clusterInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to get crush map") + } + + rulesByID := make(map[int]ruleSpec, len(crushMap.Rules)) + for _, rule := range crushMap.Rules { + rulesByID[rule.ID] = rule + } + + args := []string{"osd", "pool", "ls", "detail"} + buf, err := NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return nil, errors.Wrap(err, "failed to list pool details") + } + var pools []poolLsDetailEntry + if err := json.Unmarshal(buf, &pools); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal osd pool ls detail response") + } + + layout := &DeviceClassPDBLayout{Classes: map[string]*DeviceClassInfo{}} + poolsPartitionByClass := true + for _, pool := range pools { + rule, ok := rulesByID[pool.CrushRule] + if !ok { + // The rule the pool references is missing from the crush map. Fail closed: + // the pool escapes the per-class partition, so protect cluster-wide. + logger.Warningf("crush rule %d for pool %q not found in crush map; the pools do not partition by device class", pool.CrushRule, pool.PoolName) + poolsPartitionByClass = false + continue + } + // Collect the outermost selected level of every in-use pool (whether or not it + // maps to a single class) so the operator can derive the cluster-wide failure domain. + if fd := ruleFailureDomain(rule); fd != "" { + layout.FailureDomainTypes = append(layout.FailureDomainTypes, fd) + } + class, fdType, confined := ruleDeviceClass(rule) + if !confined { + poolsPartitionByClass = false + continue + } + info, ok := layout.Classes[class] + if !ok { + info = &DeviceClassInfo{FailureDomainType: fdType} + layout.Classes[class] = info + } + info.Pools = append(info.Pools, pool.PoolName) + } + // Per-class protection is valid only if every in-use pool maps to a single device + // class. If any pool spans classes, the pools do not partition by class: clear the + // set so the cluster-wide group is used. + if !poolsPartitionByClass { + layout.Classes = nil + } + return layout, nil +} + +// ruleDeviceClass returns the single device class a CRUSH rule confines its data +// to, the failure domain type the rule selects, and whether the rule is confined +// to exactly one device class with a resolvable failure domain. A rule that takes +// a plain root (item_name without a "~class" suffix), spans multiple device +// classes (e.g. a hybrid rule with several take steps), or has no chooseleaf type +// is not confined. The failure domain comes from ruleFailureDomain (outermost-wins). +func ruleDeviceClass(rule ruleSpec) (string, string, bool) { + classes := map[string]struct{}{} + plainRoot := false + for _, step := range rule.Steps { + if step.Operation == "take" { + // item_name is "~" for a device-class shadow tree, or a + // plain "" when the rule spans all classes. + if idx := strings.LastIndex(step.ItemName, "~"); idx >= 0 { + classes[step.ItemName[idx+1:]] = struct{}{} + } else { + plainRoot = true + } + } + } + fdType := ruleFailureDomain(rule) + if plainRoot || len(classes) != 1 || fdType == "" { + return "", "", false + } + var class string + for c := range classes { + class = c + } + return class, fdType, true +} + +// ruleFailureDomain returns the failure-domain type a CRUSH rule selects: the type of its +// outermost choose/chooseleaf step (a two-step rule's inner chooseleaf does not override +// the outer level). Returns "" when the rule selects no level. +func ruleFailureDomain(rule ruleSpec) string { + fdType := "" + for _, step := range rule.Steps { + switch { + case strings.HasPrefix(step.Operation, "chooseleaf") && step.Type != "" && fdType == "": + fdType = step.Type + case strings.HasPrefix(step.Operation, "choose") && step.Type != "" && fdType == "": + fdType = step.Type + } + } + return fdType +} + +type pgLsByPoolResponse struct { + PgStats []struct { + State string `json:"state"` + } `json:"pg_stats"` +} + +// IsDeviceClassClean reports whether every PG of the given pools matches +// pgHealthyRegex (the same regex IsClusterClean uses). Unlike the cluster-wide +// IsClusterClean, it scopes the check to one device class's pools so a class that +// has finished rebalancing returns to normal protection without waiting for a +// slower class. An empty pgHealthyRegex uses the default healthy-PG regex. +func IsDeviceClassClean(context *clusterd.Context, clusterInfo *ClusterInfo, pools []string, pgHealthyRegex string) (string, bool, error) { + regexCompiled := defaultPgHealthyRegexCompiled + if pgHealthyRegex != "" { + var err error + regexCompiled, err = regexp.Compile(pgHealthyRegex) + if err != nil { + return "unable to compile pgHealthyRegex", false, err + } + } + + totalPGs := 0 + uncleanStates := map[string]int{} + for _, pool := range pools { + args := []string{"pg", "ls-by-pool", pool} + buf, err := NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return "unable to get PG health", false, errors.Wrapf(err, "failed to get pg status for pool %q", pool) + } + var resp pgLsByPoolResponse + if err := json.Unmarshal(buf, &resp); err != nil { + return "unable to parse PG health", false, errors.Wrapf(err, "failed to unmarshal pg ls-by-pool response for pool %q", pool) + } + for _, pg := range resp.PgStats { + totalPGs++ + if !regexCompiled.MatchString(pg.State) { + uncleanStates[pg.State]++ + } + } + } + + if totalPGs == 0 { + return "device class has no PGs", true, nil + } + if len(uncleanStates) == 0 { + return "all PGs for device class are clean", true, nil + } + return "device class PGs are not clean: " + statesSummary(uncleanStates), false, nil +} + +func statesSummary(states map[string]int) string { + parts := make([]string, 0, len(states)) + for state, count := range states { + parts = append(parts, state+"="+strconv.Itoa(count)) + } + return strings.Join(parts, ",") +} diff --git a/pkg/daemon/ceph/client/deviceclass_pdb_test.go b/pkg/daemon/ceph/client/deviceclass_pdb_test.go new file mode 100644 index 000000000000..48c229b6b2fa --- /dev/null +++ b/pkg/daemon/ceph/client/deviceclass_pdb_test.go @@ -0,0 +1,279 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +// crushDumpFor builds an `osd crush dump` payload with one rule per entry, and +// returns the rule name to rule id mapping the pool mock needs. Each entry maps +// a rule name to its take item_name (e.g. "default~ssd" or the plain "default") +// and its chooseleaf failure domain type. +func crushDumpFor(rules map[string][2]string) (string, map[string]int) { + ruleJSON := "" + ruleIDs := map[string]int{} + first := true + id := 0 + for name, spec := range rules { + if !first { + ruleJSON += "," + } + first = false + ruleJSON += fmt.Sprintf(`{"rule_id":%d,"rule_name":%q,"type":1,"steps":[`+ + `{"op":"take","item_name":%q},`+ + `{"op":"chooseleaf_firstn","num":0,"type":%q},`+ + `{"op":"emit"}]}`, id, name, spec[0], spec[1]) + ruleIDs[name] = id + id++ + } + return fmt.Sprintf(`{"devices":[],"types":[],"buckets":[],"rules":[%s]}`, ruleJSON), ruleIDs +} + +// poolToRule maps each pool name to its crush rule name; a rule name absent from +// ruleIDs gets a dangling rule id, modeling a pool whose rule is missing. +func deviceClassExecutor(crushDump string, ruleIDs map[string]int, poolToRule map[string]string) *exectest.MockExecutor { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + switch { + case args[0] == "osd" && args[1] == "crush" && args[2] == "dump": + return crushDump, nil + case args[0] == "osd" && args[1] == "pool" && args[2] == "ls" && args[3] == "detail": + out := "[" + first := true + num := 1 + for pool, rule := range poolToRule { + if !first { + out += "," + } + first = false + id, ok := ruleIDs[rule] + if !ok { + id = 9999 + } + out += fmt.Sprintf(`{"pool_id":%d,"pool_name":%q,"crush_rule":%d}`, num, pool, id) + num++ + } + return out + "]", nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + return executor +} + +func TestRuleDeviceClass(t *testing.T) { + tests := []struct { + name string + rule ruleSpec + wantClass string + wantFD string + confined bool + }{ + { + name: "class-confined ssd host rule", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "take", ItemName: "default~ssd"}, + {Operation: "chooseleaf_firstn", Type: "host"}, + {Operation: "emit"}, + }}, + wantClass: "ssd", wantFD: "host", confined: true, + }, + { + name: "plain root rule spans classes", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "take", ItemName: "default"}, + {Operation: "chooseleaf_firstn", Type: "host"}, + }}, + confined: false, + }, + { + name: "hybrid rule with two classes spans classes", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "take", ItemName: "default~hdd"}, + {Operation: "chooseleaf_firstn", Type: "host"}, + {Operation: "emit"}, + {Operation: "take", ItemName: "default~ssd"}, + {Operation: "chooseleaf_firstn", Type: "host"}, + {Operation: "emit"}, + }}, + confined: false, + }, + { + name: "two-step class-confined rule: outermost choose wins over inner chooseleaf", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "take", ItemName: "default~ssd"}, + {Operation: "choose_firstn", Type: "zone"}, + {Operation: "chooseleaf_firstn", Type: "host"}, + {Operation: "emit"}, + }}, + wantClass: "ssd", wantFD: "zone", confined: true, + }, + { + name: "EC rule with chooseleaf_indep zone", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "set_chooseleaf_tries", Type: ""}, + {Operation: "take", ItemName: "default~hdd"}, + {Operation: "chooseleaf_indep", Type: "zone"}, + {Operation: "emit"}, + }}, + wantClass: "hdd", wantFD: "zone", confined: true, + }, + { + name: "confined rule without a failure domain is not confined", + rule: ruleSpec{Steps: []stepSpec{ + {Operation: "take", ItemName: "default~ssd"}, + {Operation: "emit"}, + }}, + confined: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + class, fd, confined := ruleDeviceClass(tc.rule) + assert.Equal(t, tc.confined, confined) + if confined { + assert.Equal(t, tc.wantClass, class) + assert.Equal(t, tc.wantFD, fd) + } + }) + } +} + +func TestGetDeviceClassPDBLayout(t *testing.T) { + t.Run("two confined classes is eligible", func(t *testing.T) { + crush, ruleIDs := crushDumpFor(map[string][2]string{ + "ssd_rule": {"default~ssd", "host"}, + "hdd_rule": {"default~hdd", "zone"}, + }) + executor := deviceClassExecutor(crush, ruleIDs, map[string]string{ + "ssd-pool": "ssd_rule", + "hdd-pool": "hdd_rule", + "hdd-pool2": "hdd_rule", + }) + layout, err := GetDeviceClassPDBLayout(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Len(t, layout.Classes, 2) + assert.Equal(t, "host", layout.Classes["ssd"].FailureDomainType) + assert.Equal(t, "zone", layout.Classes["hdd"].FailureDomainType) + assert.ElementsMatch(t, []string{"ssd-pool"}, layout.Classes["ssd"].Pools) + assert.ElementsMatch(t, []string{"hdd-pool", "hdd-pool2"}, layout.Classes["hdd"].Pools) + // one entry per in-use pool's outermost selected level, for the global FD + assert.ElementsMatch(t, []string{"host", "zone", "zone"}, layout.FailureDomainTypes) + }) + + t.Run("a pool spanning classes clears the partition", func(t *testing.T) { + crush, ruleIDs := crushDumpFor(map[string][2]string{ + "ssd_rule": {"default~ssd", "host"}, + "replicated": {"default", "host"}, + "hdd_rule": {"default~hdd", "host"}, + }) + executor := deviceClassExecutor(crush, ruleIDs, map[string]string{ + "ssd-pool": "ssd_rule", + "hdd-pool": "hdd_rule", + ".mgr": "replicated", // spans all classes + }) + layout, err := GetDeviceClassPDBLayout(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Empty(t, layout.Classes) + }) + + t.Run("single class is not enough on its own", func(t *testing.T) { + crush, ruleIDs := crushDumpFor(map[string][2]string{ + "ssd_rule": {"default~ssd", "host"}, + }) + executor := deviceClassExecutor(crush, ruleIDs, map[string]string{ + "ssd-pool": "ssd_rule", + }) + layout, err := GetDeviceClassPDBLayout(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Len(t, layout.Classes, 1) + }) + + t.Run("missing crush rule fails closed, clearing the partition", func(t *testing.T) { + crush, ruleIDs := crushDumpFor(map[string][2]string{ + "ssd_rule": {"default~ssd", "host"}, + }) + executor := deviceClassExecutor(crush, ruleIDs, map[string]string{ + "ssd-pool": "ssd_rule", + "orphaned-pool": "missing_rule", + }) + layout, err := GetDeviceClassPDBLayout(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Empty(t, layout.Classes) + }) +} + +func TestIsDeviceClassClean(t *testing.T) { + pgResp := func(states ...string) string { + out := `{"pg_stats":[` + for i, s := range states { + if i > 0 { + out += "," + } + out += fmt.Sprintf(`{"state":%q}`, s) + } + return out + "]}" + } + + t.Run("all pgs clean", func(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if args[0] == "pg" && args[1] == "ls-by-pool" { + return pgResp("active+clean", "active+clean"), nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + _, clean, err := IsDeviceClassClean(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster"), []string{"ssd-pool"}, "") + assert.NoError(t, err) + assert.True(t, clean) + }) + + t.Run("a non-clean pg makes the class unclean", func(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if args[0] == "pg" && args[1] == "ls-by-pool" { + if args[2] == "ssd-pool-a" { + return pgResp("active+clean"), nil + } + return pgResp("active+clean", "active+recovering"), nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + _, clean, err := IsDeviceClassClean(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster"), []string{"ssd-pool-a", "ssd-pool-b"}, "") + assert.NoError(t, err) + assert.False(t, clean) + }) + + t.Run("no pgs counts as clean", func(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if args[0] == "pg" && args[1] == "ls-by-pool" { + return `{"pg_stats":[]}`, nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + _, clean, err := IsDeviceClassClean(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster"), []string{"empty-pool"}, "") + assert.NoError(t, err) + assert.True(t, clean) + }) +} diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 23b781eeef77..1986ed2c6d6d 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -580,17 +580,6 @@ func (c *Cluster) EnableServiceMonitor() error { return nil } -// IsModuleInSpec returns whether a module is present in the CephCluster manager spec -func IsModuleInSpec(modules []cephv1.Module, moduleName string) bool { - for _, v := range modules { - if v.Name == moduleName { - return true - } - } - - return false -} - // ApplyMonitoringLabels function adds the name of the resource that manages // cephcluster, as a label on the ceph metrics func applyMonitoringLabels(c *Cluster, serviceMonitor *monitoringv1.ServiceMonitor) { diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go index 80b839305dbc..38d1ce13f861 100644 --- a/pkg/operator/ceph/cluster/osd/create.go +++ b/pkg/operator/ceph/cluster/osd/create.go @@ -329,6 +329,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov // create the job that prepares osds on the node storeConfig := osdconfig.ToStoreConfig(n.Config) metadataDevice := osdconfig.MetadataDevice(n.Config) + deviceClass, err := c.resolveDeviceClass(storeConfig.DeviceClass, n.Name) + if err != nil { + log.NamespacedError(c.clusterInfo.Namespace, logger, "%v", err) + continue + } + storeConfig.DeviceClass = deviceClass osdProps := osdProperties{ crushHostname: n.Name, devices: n.Devices, diff --git a/pkg/operator/ceph/cluster/osd/create_test.go b/pkg/operator/ceph/cluster/osd/create_test.go index a857b5c1cb11..76d1673ac837 100644 --- a/pkg/operator/ceph/cluster/osd/create_test.go +++ b/pkg/operator/ceph/cluster/osd/create_test.go @@ -639,6 +639,225 @@ func Test_startProvisioningOverNodes(t *testing.T) { }) } +func Test_startProvisioningOverNodes_deviceClassNodeLabel(t *testing.T) { + namespace := "rook-ceph" + dataDirHostPath := "/var/lib/mycluster" + useAllDevices := true + + clusterInfo := &cephclient.ClusterInfo{ + Namespace: namespace, + CephVersion: cephver.Squid, + } + clusterInfo.SetName("mycluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + clusterInfo.Context = context.TODO() + + getDeviceClassEnvFromJobs := func(clientset *fake.Clientset) map[string]string { + jobs, err := clientset.BatchV1().Jobs(namespace).List(context.TODO(), metav1.ListOptions{}) + assert.NoError(t, err) + result := map[string]string{} + for _, job := range jobs.Items { + for _, container := range job.Spec.Template.Spec.Containers { + for _, env := range container.Env { + if env.Name == CrushDeviceClassVarName { + nodeName := strings.TrimPrefix(job.Name, "rook-ceph-osd-prepare-") + result[nodeName] = env.Value + } + } + } + } + return result + } + + t.Run("node label sets device class when CR has none", func(t *testing.T) { + clientset := fake.NewClientset() + // Create a node with the deviceclass label + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{ + corev1.LabelHostname: "node0", + NodeDeviceClassLabelKey: "ssd", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + assert.NoError(t, err) + + spec := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + UseAllNodes: false, + Nodes: []cephv1.Node{{Name: "node0"}}, + Selection: cephv1.Selection{UseAllDevices: &useAllDevices}, + }, + DataDirHostPath: dataDirHostPath, + } + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, spec, "rook/rook:master") + config := c.newProvisionConfig() + errs := newProvisionErrors() + + prepareJobsRun, err := c.startProvisioningOverNodes(config, errs) + assert.NoError(t, err) + assert.Zero(t, errs.len()) + assert.Equal(t, 1, prepareJobsRun.Len()) + + deviceClasses := getDeviceClassEnvFromJobs(clientset) + assert.Equal(t, "ssd", deviceClasses["node0"]) + }) + + t.Run("conflict between CR and node label skips the node", func(t *testing.T) { + clientset := fake.NewClientset() + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{ + corev1.LabelHostname: "node0", + NodeDeviceClassLabelKey: "ssd", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + assert.NoError(t, err) + + spec := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + UseAllNodes: false, + Nodes: []cephv1.Node{{ + Name: "node0", + Config: map[string]string{"deviceClass": "hdd"}, + }}, + Selection: cephv1.Selection{UseAllDevices: &useAllDevices}, + }, + DataDirHostPath: dataDirHostPath, + } + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, spec, "rook/rook:master") + config := c.newProvisionConfig() + errs := newProvisionErrors() + + prepareJobsRun, err := c.startProvisioningOverNodes(config, errs) + assert.NoError(t, err) + assert.Zero(t, errs.len()) + assert.Zero(t, prepareJobsRun.Len()) + }) + + t.Run("no label and no CR config results in empty device class", func(t *testing.T) { + clientset := fake.NewClientset() + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{ + corev1.LabelHostname: "node0", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + assert.NoError(t, err) + + spec := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + UseAllNodes: false, + Nodes: []cephv1.Node{{Name: "node0"}}, + Selection: cephv1.Selection{UseAllDevices: &useAllDevices}, + }, + DataDirHostPath: dataDirHostPath, + } + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, spec, "rook/rook:master") + config := c.newProvisionConfig() + errs := newProvisionErrors() + + prepareJobsRun, err := c.startProvisioningOverNodes(config, errs) + assert.NoError(t, err) + assert.Zero(t, errs.len()) + assert.Equal(t, 1, prepareJobsRun.Len()) + + deviceClasses := getDeviceClassEnvFromJobs(clientset) + assert.Equal(t, "", deviceClasses["node0"]) + }) + + t.Run("conflict skips one node but other nodes still provision", func(t *testing.T) { + clientset := fake.NewClientset() + // node0 has both CR config and label (conflict) + node0 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{ + corev1.LabelHostname: "node0", + NodeDeviceClassLabelKey: "ssd", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + } + // node1 has only label (no conflict) + node1 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + corev1.LabelHostname: "node1", + NodeDeviceClassLabelKey: "fast", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node0, metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{}) + assert.NoError(t, err) + + spec := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + UseAllNodes: false, + Nodes: []cephv1.Node{ + {Name: "node0", Config: map[string]string{"deviceClass": "hdd"}}, + {Name: "node1"}, + }, + Selection: cephv1.Selection{UseAllDevices: &useAllDevices}, + }, + DataDirHostPath: dataDirHostPath, + } + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, spec, "rook/rook:master") + config := c.newProvisionConfig() + errs := newProvisionErrors() + + prepareJobsRun, err := c.startProvisioningOverNodes(config, errs) + assert.NoError(t, err) + assert.Zero(t, errs.len()) + // Only node1 should have a prepare job (node0 skipped due to conflict) + assert.Equal(t, 1, prepareJobsRun.Len()) + + deviceClasses := getDeviceClassEnvFromJobs(clientset) + assert.Equal(t, "fast", deviceClasses["node1"]) + _, node0HasJob := deviceClasses["node0"] + assert.False(t, node0HasJob) + }) +} + func newDummyPVC(name, namespace string, capacity string, storageClassName string) cephv1.VolumeClaimTemplate { volMode := corev1.PersistentVolumeBlock return cephv1.VolumeClaimTemplate{ diff --git a/pkg/operator/ceph/cluster/osd/labels.go b/pkg/operator/ceph/cluster/osd/labels.go index 5e70c6c98a1f..1aee947fbcf2 100644 --- a/pkg/operator/ceph/cluster/osd/labels.go +++ b/pkg/operator/ceph/cluster/osd/labels.go @@ -36,6 +36,8 @@ const ( OSDOverPVCLabelKey = "ceph.rook.io/pvc" // TopologyLocationLabel is the crush location label added to OSD deployments TopologyLocationLabel = "topology-location-%s" + // DeviceClassLabelKey is the label on OSD pods holding the OSD's device class + DeviceClassLabelKey = deviceClass // CephImageLabelKey is the ceph image version label added to PVC CephImageLabelKey = "ceph.rook.io/cephImageAtCreation" // RookImageLabelKey is the rook image version label added to PVC diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 8f744026ff0d..1ee2638457b5 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -80,6 +80,10 @@ const ( deviceType = "device-type" encrypted = "encrypted" + // NodeDeviceClassLabelKey is the node label the operator reads to assign a device class + // to all OSDs provisioned on that node. + NodeDeviceClassLabelKey = "osd.rook.io/device-class" + // CephxStatus is applied to each OSD deployment as value of this annotation key cephxStatusAnnotationKey = "cephx-status" ) @@ -590,6 +594,11 @@ func (c *Cluster) getOSDPropsForNode(nodeName, deviceClass string) (osdPropertie storeConfig := osdconfig.ToStoreConfig(n.Config) metadataDevice := osdconfig.MetadataDevice(n.Config) + deviceClass, err := c.resolveDeviceClass(storeConfig.DeviceClass, n.Name) + if err != nil { + return osdProperties{}, err + } + storeConfig.DeviceClass = deviceClass osdProps := osdProperties{ crushHostname: n.Name, devices: n.Devices, @@ -987,10 +996,13 @@ func getNode(ctx context.Context, clientset kubernetes.Interface, nodeName strin // try to find by the node by matching the provided nodeName node, err = clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if kerrors.IsNotFound(err) { - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%q=%q", k8sutil.LabelHostname(), nodeName)} + listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.LabelHostname(), nodeName)} nodeList, err := clientset.CoreV1().Nodes().List(ctx, listOpts) - if err != nil || len(nodeList.Items) < 1 { - return nil, errors.Wrapf(err, "could not find node %q hostname label", nodeName) + if err != nil { + return nil, errors.Wrapf(err, "could not find node %q by hostname label", nodeName) + } + if len(nodeList.Items) != 1 { + return nil, errors.Errorf("could not find node %q by hostname label", nodeName) } return &nodeList.Items[0], nil } else if err != nil { @@ -1000,6 +1012,29 @@ func getNode(ctx context.Context, clientset kubernetes.Interface, nodeName strin return node, nil } +// resolveDeviceClass determines the device class for a node by checking the CR config +// and the osd.rook.io/device-class node label. If both are set, it returns an error +// to avoid ambiguity — the user must choose one or the other. +func (c *Cluster) resolveDeviceClass(crDeviceClass, nodeName string) (string, error) { + node, err := getNode(c.clusterInfo.Context, c.context.Clientset, nodeName) + if err != nil { + return "", errors.Wrapf(err, "failed to get node %q to read device class label", nodeName) + } + labelClass := node.Labels[NodeDeviceClassLabelKey] + if crDeviceClass != "" && labelClass != "" { + return "", errors.Errorf("node %q has both CR device class %q and node label %q set; remove one to resolve the conflict", nodeName, crDeviceClass, labelClass) + } + if crDeviceClass != "" { + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "using device class %q from CR config for node %q", crDeviceClass, nodeName) + return crDeviceClass, nil + } + if labelClass != "" { + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "using device class %q from node label for node %q", labelClass, nodeName) + return labelClass, nil + } + return "", nil +} + func updateLocationWithNodeLabels(location *[]string, nodeLabels map[string]string) string { topology, topologyAffinity := topology.ExtractOSDTopologyFromLabels(nodeLabels) diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index e54113294336..90fd1ebdbb8a 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -1495,3 +1495,90 @@ func TestPerDeviceClassForOSD(t *testing.T) { }) } } + +func TestResolveDeviceClass(t *testing.T) { + namespace := "rook-ceph" + clusterInfo := &cephclient.ClusterInfo{ + Namespace: namespace, + CephVersion: cephver.Squid, + Context: context.TODO(), + } + clusterInfo.SetName("mycluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + + createNode := func(clientset *fake.Clientset, name string, labels map[string]string) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + assert.NoError(t, err) + } + + t.Run("only node label set", func(t *testing.T) { + clientset := fake.NewClientset() + createNode(clientset, "node0", map[string]string{ + corev1.LabelHostname: "node0", + NodeDeviceClassLabelKey: "ssd", + }) + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master") + + result, err := c.resolveDeviceClass("", "node0") + assert.NoError(t, err) + assert.Equal(t, "ssd", result) + }) + + t.Run("only CR device class set", func(t *testing.T) { + clientset := fake.NewClientset() + createNode(clientset, "node0", map[string]string{ + corev1.LabelHostname: "node0", + }) + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master") + + result, err := c.resolveDeviceClass("hdd", "node0") + assert.NoError(t, err) + assert.Equal(t, "hdd", result) + }) + + t.Run("both set returns conflict error", func(t *testing.T) { + clientset := fake.NewClientset() + createNode(clientset, "node0", map[string]string{ + corev1.LabelHostname: "node0", + NodeDeviceClassLabelKey: "ssd", + }) + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master") + + result, err := c.resolveDeviceClass("hdd", "node0") + assert.Error(t, err) + assert.Contains(t, err.Error(), "conflict") + assert.Equal(t, "", result) + }) + + t.Run("neither set returns empty", func(t *testing.T) { + clientset := fake.NewClientset() + createNode(clientset, "node0", map[string]string{ + corev1.LabelHostname: "node0", + }) + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master") + + result, err := c.resolveDeviceClass("", "node0") + assert.NoError(t, err) + assert.Equal(t, "", result) + }) + + t.Run("node not found returns error", func(t *testing.T) { + clientset := fake.NewClientset() + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master") + + result, err := c.resolveDeviceClass("", "nonexistent") + assert.Error(t, err) + assert.Equal(t, "", result) + }) +} diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index f3acacb18ce1..d4a43bbd8899 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -18,6 +18,7 @@ limitations under the License. package osd import ( + "context" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -32,7 +33,9 @@ import ( "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) @@ -796,6 +799,17 @@ func getDummyDeploymentOnPVC(clientset *fake.Clientset, c *Cluster, pvcName stri // WARNING! modifies c.ValidStorage func getDummyDeploymentOnNode(clientset *fake.Clientset, c *Cluster, nodeName string, osdID int) *appsv1.Deployment { + // Ensure the node exists in the fake clientset so resolveDeviceClass can look it up + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{corev1.LabelHostname: nodeName}, + }, + } + if _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}); err != nil && !kerrors.IsAlreadyExists(err) { + panic(err) + } + osd := &OSDInfo{ ID: osdID, UUID: "some-uuid", @@ -812,6 +826,96 @@ func getDummyDeploymentOnNode(clientset *fake.Clientset, c *Cluster, nodeName st return d } +func TestDeploymentOnNode_DeviceClassFromLabel(t *testing.T) { + clientset := fake.NewClientset() + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{corev1.LabelHostname: "node0", NodeDeviceClassLabelKey: "fast"}, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + assert.NoError(t, err) + + clusterInfo := &cephclient.ClusterInfo{ + Namespace: "ns", + CephVersion: cephver.Squid, + } + clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + clusterInfo.Context = context.TODO() + + spec := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + Nodes: []cephv1.Node{{Name: "node0"}}, + AllowDeviceClassUpdate: true, + }, + DataDirHostPath: "/var/lib/rook", + } + ctx := &clusterd.Context{Clientset: clientset} + c := New(ctx, clusterInfo, spec, "rook/rook:master") + c.ValidStorage.Nodes = []cephv1.Node{{Name: "node0"}} + + osd := &OSDInfo{ID: 0, UUID: "u", BlockPath: "/dev/vda", CVMode: "raw", Store: "bluestore"} + config := c.newProvisionConfig() + + t.Run("node label device class flows into deployment", func(t *testing.T) { + d, err := deploymentOnNode(c, osd, "node0", config) + assert.NoError(t, err) + // Check ROOK_OSD_DEVICE_CLASS env var + for _, container := range d.Spec.Template.Spec.Containers { + for _, env := range container.Env { + if env.Name == "ROOK_OSD_DEVICE_CLASS" { + assert.Equal(t, "fast", env.Value) + return + } + } + } + t.Fatal("ROOK_OSD_DEVICE_CLASS env var not found") + }) + + t.Run("conflict between CR and label returns error", func(t *testing.T) { + specWithCR := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + Nodes: []cephv1.Node{{Name: "node0", Config: map[string]string{"deviceClass": "hdd"}}}, + }, + DataDirHostPath: "/var/lib/rook", + } + c2 := New(ctx, clusterInfo, specWithCR, "rook/rook:master") + c2.ValidStorage.Nodes = []cephv1.Node{{Name: "node0", Config: map[string]string{"deviceClass": "hdd"}}} + + _, err := deploymentOnNode(c2, osd, "node0", c2.newProvisionConfig()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "conflict") + }) + + t.Run("AllowDeviceClassUpdate false does not change existing OSD class", func(t *testing.T) { + specNoUpdate := cephv1.ClusterSpec{ + Storage: cephv1.StorageScopeSpec{ + Nodes: []cephv1.Node{{Name: "node0"}}, + AllowDeviceClassUpdate: false, + }, + DataDirHostPath: "/var/lib/rook", + } + c3 := New(ctx, clusterInfo, specNoUpdate, "rook/rook:master") + c3.ValidStorage.Nodes = []cephv1.Node{{Name: "node0"}} + + osdWithClass := &OSDInfo{ID: 0, UUID: "u", BlockPath: "/dev/vda", CVMode: "raw", Store: "bluestore", DeviceClass: "hdd"} + d, err := deploymentOnNode(c3, osdWithClass, "node0", c3.newProvisionConfig()) + assert.NoError(t, err) + // OSD should keep "hdd" even though node label says "fast" + for _, container := range d.Spec.Template.Spec.Containers { + for _, env := range container.Env { + if env.Name == "ROOK_OSD_DEVICE_CLASS" { + assert.Equal(t, "hdd", env.Value) + return + } + } + } + t.Fatal("ROOK_OSD_DEVICE_CLASS env var not found") + }) +} + func TestOSDPlacement(t *testing.T) { clientset := fake.NewClientset() clusterInfo := &cephclient.ClusterInfo{ diff --git a/pkg/operator/ceph/cluster/osd/update_test.go b/pkg/operator/ceph/cluster/osd/update_test.go index 8288555af60e..5306809616be 100644 --- a/pkg/operator/ceph/cluster/osd/update_test.go +++ b/pkg/operator/ceph/cluster/osd/update_test.go @@ -546,6 +546,79 @@ func Test_updateExistingOSDs(t *testing.T) { assert.Equal(t, 1, osdIDUpdated) updateConfig.osdsToSkipReconcile.Delete("0") }) + + t.Run("conflict between CR and node label skips conflicting OSD but updates others", func(t *testing.T) { + clientset = fake.NewClientset() + + // node0 has no label (no conflict) + node0 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{corev1.LabelHostname: "node0"}, + }, + } + _, err := clientset.CoreV1().Nodes().Create(context.TODO(), node0, metav1.CreateOptions{}) + assert.NoError(t, err) + + // node1 has BOTH label and CR config (conflict) + node1 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{corev1.LabelHostname: "node1", NodeDeviceClassLabelKey: "ssd"}, + }, + } + _, err = clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{}) + assert.NoError(t, err) + + updateQueue = newUpdateQueueWithIDs(0, 1) + existingDeployments = newExistenceListWithIDs(0, 1) + + ctx = &clusterd.Context{ + Clientset: clientset, + Executor: executor, + } + clusterInfo := &cephclient.ClusterInfo{ + Namespace: namespace, + CephVersion: cephver.Squid, + Context: context.TODO(), + } + clusterInfo.SetName("mycluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + spec := cephv1.ClusterSpec{ + ContinueUpgradeAfterChecksEvenIfNotHealthy: forceUpgradeIfUnhealthy, + UpgradeOSDRequiresHealthyPGs: requiresHealthyPGs, + Storage: cephv1.StorageScopeSpec{ + Nodes: []cephv1.Node{ + {Name: "node0"}, + {Name: "node1", Config: map[string]string{"deviceClass": "hdd"}}, + }, + }, + } + c = New(ctx, clusterInfo, spec, "rook/rook:master") + c.ValidStorage.Nodes = spec.Storage.Nodes + config := c.newProvisionConfig() + updateConfig = c.newUpdateConfig(config, updateQueue, existingDeployments, sets.New[string]()) + + addDeploymentOnNode("node0", 0) + // Create node1's deployment manually since getDummyDeploymentOnNode would panic on conflict. + // In production, this deployment would have been created before the label was added. + d1 := getDummyDeploymentOnNode(clientset, c, "node0", 1) // use node0 to avoid panic + d1.Name = "rook-ceph-osd-1" + d1.Spec.Template.Spec.NodeSelector = map[string]string{corev1.LabelHostname: "node1"} + _, err = clientset.AppsV1().Deployments(namespace).Create(context.TODO(), d1, metav1.CreateOptions{}) + assert.NoError(t, err) + + osdToBeQueried = 0 + returnOkToStopIDs = []int{0, 1} + deploymentsUpdated = []string{} + errs = newProvisionErrors() + + updateConfig.updateExistingOSDs(errs) + // OSD 1 on node1 should fail due to conflict + assert.Equal(t, 1, errs.len()) + // OSD 0 on node0 should have been updated successfully + assert.Contains(t, deploymentsUpdated, "rook-ceph-osd-0") + }) } func Test_getOSDUpdateInfo(t *testing.T) { diff --git a/pkg/operator/ceph/disruption/clusterdisruption/add.go b/pkg/operator/ceph/disruption/clusterdisruption/add.go index 0c2cb270f1a1..6e62b66f16d7 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/add.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/add.go @@ -19,6 +19,7 @@ package clusterdisruption import ( ctx "context" "reflect" + "strings" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" @@ -69,8 +70,11 @@ func pdbPredicate[T *policyv1.PodDisruptionBudget]() predicate.TypedFuncs[T] { UpdateFunc: func(e event.TypedUpdateEvent[T]) bool { pdb := (*policyv1.PodDisruptionBudget)(e.ObjectNew) - // reconcile for the main PDB update event when first OSD goes down, that is, when `DisruptionsAllowed` gets updated to 0. - return pdb.Name == osdPDBAppName && pdb.Spec.MaxUnavailable.IntVal == 1 && pdb.Status.DisruptionsAllowed == 0 + // Reconcile when the first OSD of a group goes down. ExpectedPods>0 excludes a + // default PDB that selects no OSDs: it also reports DisruptionsAllowed=0, but + // that is not a drain. + isOSDDefaultPDB := pdb.Name == osdPDBAppName || strings.HasPrefix(pdb.Name, osdPDBAppName+"-") + return isOSDDefaultPDB && pdb.Spec.MaxUnavailable.IntVal == 1 && pdb.Status.DisruptionsAllowed == 0 && pdb.Status.ExpectedPods > 0 }, DeleteFunc: func(e event.TypedDeleteEvent[T]) bool { // Do not reconcile when PDB is deleted diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd.go b/pkg/operator/ceph/disruption/clusterdisruption/osd.go index 9b260bb0f886..0d265395d6af 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd.go @@ -55,6 +55,67 @@ const ( nooutFlag = "noout" ) +// pdbDrainKeys names the ConfigMap keys that hold one group's drain state. The +// cluster-wide group uses the bare baseline keys; a per-class group uses the +// dc..* keys. +type pdbDrainKeys struct { + draining string + setNoOut string + duration string +} + +// globalDrainKeys are the drain-state ConfigMap keys for the cluster-wide group. +var globalDrainKeys = pdbDrainKeys{ + draining: drainingFailureDomainKey, + setNoOut: setNoOut, + duration: drainingFailureDomainDurationKey, +} + +// groupDrainState is one group's view of OSD failure domains, the group-scoped +// analog of the tuple the baseline computed for the whole cluster. +type groupDrainState struct { + allFailureDomains []string + nodeDrainFailureDomains []string + osdDownFailureDomains []string + downOSDs []int +} + +// pdbGroup is one set of OSDs reconciled together. A cluster gets either a single +// cluster-wide group (deviceClass "") or one group per device class; both are this same +// struct, driven by the same reconcile. A group's PDB naming, ConfigMap keys, and +// PG-health source are all derived from deviceClass (see the methods below), so the +// reconcile never branches on it. +type pdbGroup struct { + // deviceClass is "" for the cluster-wide group, else the device class this group covers. + deviceClass string + // failureDomainType is the CRUSH failure domain type for this group's PDBs. + failureDomainType string + // pools are the RADOS pools backing this group's per-class PG-health check; nil for + // the cluster-wide group, which checks PG health cluster-wide. + pools []string + // state is the enumerated failure-domain state, filled by populateOSDFailureDomains. + state groupDrainState + // excludeOSDs are down OSDs to exclude from the group's default PDB, computed + // by updateDrainState. + excludeOSDs []int + // degraded is set by populateOSDFailureDomains when this group's failure-domain type + // does not resolve to a label on one of its OSDs. A degraded group stays idle + // (default-only PDB, no blocking, no noout, no drain-state writes) so it neither + // aborts the whole reconcile nor leaves its OSDs uncovered. + degraded bool +} + +// defaultPDB builds this group's maxUnavailable=1 default PDB. +func (g *pdbGroup) defaultPDB(namespace string) *policyv1.PodDisruptionBudget { + return defaultOSDPDB(namespace, g.defaultPDBName(), g.deviceClassIn(), nil, g.excludeOSDs) +} + +// blockingPDB builds this group's maxUnavailable=0 blocking PDB for a failure domain. +func (g *pdbGroup) blockingPDB(namespace, failureDomainName string) *policyv1.PodDisruptionBudget { + topologyLabel := fmt.Sprintf(osd.TopologyLocationLabel, g.failureDomainType) + return blockingOSDPDB(namespace, g.blockingPDBName(failureDomainName), g.deviceClass, topologyLabel, failureDomainName) +} + func (r *ReconcileClusterDisruption) createPDB(pdb client.Object) error { err := r.client.Create(r.context.OpManagerContext, pdb) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -71,152 +132,136 @@ func (r *ReconcileClusterDisruption) deletePDB(pdb client.Object) error { return nil } -// createDefaultPDBforOSD creates a single PDB for all OSDs with maxUnavailable=1 -// This allows all OSDs in a single failure domain to go down. -func (r *ReconcileClusterDisruption) createDefaultPDBforOSD(namespace string, excludeOSDs []int) error { - cephCluster, ok := r.clusterMap.GetCluster(namespace) +// applyPDB creates or updates a PDB in place, setting the CephCluster owner +// reference. It is the "apply" half of the apply-then-prune contract. +func (r *ReconcileClusterDisruption) applyPDB(pdb *policyv1.PodDisruptionBudget) error { + cephCluster, ok := r.clusterMap.GetCluster(pdb.Namespace) if !ok { - return errors.Errorf("failed to find the namespace %q in the clustermap", namespace) + return errors.Errorf("failed to find the namespace %q in the clustermap", pdb.Namespace) + } + ownerInfo := k8sutil.NewOwnerInfo(cephCluster, r.scheme) + if err := ownerInfo.SetControllerReference(pdb); err != nil { + return errors.Wrapf(err, "failed to set owner reference on pdb %q", pdb.Name) } - pdbRequest := types.NamespacedName{Name: osdPDBAppName, Namespace: namespace} - objectMeta := metav1.ObjectMeta{ - Name: osdPDBAppName, - Namespace: namespace, + + existing := &policyv1.PodDisruptionBudget{} + err := r.client.Get(r.context.OpManagerContext, types.NamespacedName{Name: pdb.Name, Namespace: pdb.Namespace}, existing) + if apierrors.IsNotFound(err) { + logger.Infof("creating osd pdb %q", pdb.Name) + return r.createPDB(pdb) } + if err != nil { + return errors.Wrapf(err, "failed to get pdb %q", pdb.Name) + } + existing.Spec = pdb.Spec + return r.client.Update(r.context.OpManagerContext, existing) +} + +// defaultOSDPDB builds a maxUnavailable=1 OSD PDB. deviceClassIn scopes it to a device +// class; deviceClassNotIn scopes it to OSDs outside the given classes; excludeOSDs drops +// specific OSD ids from the selector. +func defaultOSDPDB(namespace, name string, deviceClassIn, deviceClassNotIn []string, excludeOSDs []int) *policyv1.PodDisruptionBudget { matchExpressions := []metav1.LabelSelectorRequirement{ - // require the pod to be an OSD pod { Key: k8sutil.AppAttr, Operator: metav1.LabelSelectorOpIn, Values: []string{osdPDBAppName}, }, } + if len(deviceClassIn) > 0 { + matchExpressions = append(matchExpressions, metav1.LabelSelectorRequirement{ + Key: osd.DeviceClassLabelKey, + Operator: metav1.LabelSelectorOpIn, + Values: deviceClassIn, + }) + } + if len(deviceClassNotIn) > 0 { + matchExpressions = append(matchExpressions, metav1.LabelSelectorRequirement{ + Key: osd.DeviceClassLabelKey, + Operator: metav1.LabelSelectorOpNotIn, + Values: deviceClassNotIn, + }) + } if len(excludeOSDs) > 0 { - excludeOSDsValues := make([]string, len(excludeOSDs)) - for i, excludeOSD := range excludeOSDs { - excludeOSDsValues[i] = strconv.Itoa(excludeOSD) + values := make([]string, len(excludeOSDs)) + for i, id := range excludeOSDs { + values[i] = strconv.Itoa(id) } matchExpressions = append(matchExpressions, metav1.LabelSelectorRequirement{ - // don't consider pods for excluded OSD IDs Key: osdPDBOsdIdLabel, Operator: metav1.LabelSelectorOpNotIn, - Values: excludeOSDsValues, + Values: values, }) } - - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: objectMeta, + maxUnavailable := intstr.FromInt32(1) + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: policyv1.PodDisruptionBudgetSpec{ - MaxUnavailable: &intstr.IntOrString{IntVal: 1}, - Selector: &metav1.LabelSelector{ - MatchExpressions: matchExpressions, - }, + MaxUnavailable: &maxUnavailable, + Selector: &metav1.LabelSelector{MatchExpressions: matchExpressions}, }, } - ownerInfo := k8sutil.NewOwnerInfo(cephCluster, r.scheme) - err := ownerInfo.SetControllerReference(pdb) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to pdb %v", pdb) - } - - existingPDB := &policyv1.PodDisruptionBudget{} - err = r.client.Get(r.context.OpManagerContext, pdbRequest, existingPDB) - if err != nil { - if apierrors.IsNotFound(err) { - logger.Info("all PGs are active+clean. Restoring default OSD pdb settings") - logger.Infof("creating the default pdb %q with maxUnavailable=1 for all osd", osdPDBAppName) - return r.createPDB(pdb) - } - return errors.Wrapf(err, "failed to get pdb %q", pdb.Name) - } - - existingPDB.Spec = pdb.Spec - err = r.client.Update(r.context.OpManagerContext, existingPDB) - if err != nil { - return errors.Wrapf(err, "failed to update existing pdb %q", existingPDB.Name) - } - return nil -} - -func (r *ReconcileClusterDisruption) deleteDefaultPDBforOSD(namespace string) error { - pdbRequest := types.NamespacedName{Name: osdPDBAppName, Namespace: namespace} - objectMeta := metav1.ObjectMeta{ - Name: osdPDBAppName, - Namespace: namespace, - } - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: objectMeta, - } - err := r.client.Get(r.context.OpManagerContext, pdbRequest, &policyv1.PodDisruptionBudget{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return errors.Wrapf(err, "failed to get pdb %q", pdb.Name) - } - logger.Infof("deleting the default pdb %q with maxUnavailable=1 for all osd", osdPDBAppName) - return r.deletePDB(pdb) } -// createBlockingPDBForOSD creates individual blocking PDBs (maxUnavailable=0) for all the OSDs in -// failure domains that are not draining -func (r *ReconcileClusterDisruption) createBlockingPDBForOSD(namespace, failureDomainType, failureDomainName string) error { - cephCluster, ok := r.clusterMap.GetCluster(namespace) - if !ok { - return errors.Errorf("failed to find the namespace %q in the clustermap", namespace) - } - - pdbName := getPDBName(failureDomainType, failureDomainName) - pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace} - objectMeta := metav1.ObjectMeta{ - Name: pdbName, - Namespace: namespace, - } +// blockingOSDPDB builds a maxUnavailable=0 blocking PDB for one non-draining +// failure domain. A non-empty deviceClass adds the per-class "In" clause; the +// cluster-wide group passes "" and gets the baseline topology-only selector. +func blockingOSDPDB(namespace, name, deviceClass, topologyLabel, failureDomainName string) *policyv1.PodDisruptionBudget { + maxUnavailable := intstr.FromInt32(0) selector := &metav1.LabelSelector{ - MatchLabels: map[string]string{fmt.Sprintf(osd.TopologyLocationLabel, failureDomainType): failureDomainName}, + MatchLabels: map[string]string{topologyLabel: failureDomainName}, + } + if deviceClass != "" { + selector.MatchExpressions = []metav1.LabelSelectorRequirement{ + { + Key: osd.DeviceClassLabelKey, + Operator: metav1.LabelSelectorOpIn, + Values: []string{deviceClass}, + }, + } } - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: objectMeta, + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: policyv1.PodDisruptionBudgetSpec{ - MaxUnavailable: &intstr.IntOrString{IntVal: 0}, + MaxUnavailable: &maxUnavailable, Selector: selector, }, } - ownerInfo := k8sutil.NewOwnerInfo(cephCluster, r.scheme) - err := ownerInfo.SetControllerReference(pdb) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to pdb %v", pdb) - } - err = r.client.Get(r.context.OpManagerContext, pdbRequest, &policyv1.PodDisruptionBudget{}) - if err != nil { - if apierrors.IsNotFound(err) { - logger.Infof("creating temporary blocking pdb %q with maxUnavailable=0 for %q failure domain %q", pdbName, failureDomainType, failureDomainName) - return r.createPDB(pdb) +} + +func getPDBName(failureDomainType, failureDomainName string) string { + return k8sutil.TruncateNodeName(fmt.Sprintf("%s-%s-%s", osdPDBAppName, failureDomainType, "%s"), failureDomainName) +} + +// listOSDPDBs returns every PDB targeting OSDs — rook-ceph-osd and rook-ceph-osd-* (the +// defaults and blocking PDBs) — and not the rgw/mds PDBs. +func (r *ReconcileClusterDisruption) listOSDPDBs(namespace string) ([]policyv1.PodDisruptionBudget, error) { + pdbList := &policyv1.PodDisruptionBudgetList{} + if err := r.client.List(r.context.OpManagerContext, pdbList, client.InNamespace(namespace)); err != nil { + return nil, errors.Wrap(err, "failed to list pod disruption budgets") + } + osdPDBs := make([]policyv1.PodDisruptionBudget, 0, len(pdbList.Items)) + for i := range pdbList.Items { + name := pdbList.Items[i].Name + if name == osdPDBAppName || strings.HasPrefix(name, osdPDBAppName+"-") { + osdPDBs = append(osdPDBs, pdbList.Items[i]) } - return errors.Wrapf(err, "failed to get pdb %q", pdb.Name) } - return nil + return osdPDBs, nil } -func (r *ReconcileClusterDisruption) deleteBlockingPDBForOSD(namespace, failureDomainType, failureDomainName string) error { - pdbName := getPDBName(failureDomainType, failureDomainName) - pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace} - objectMeta := metav1.ObjectMeta{ - Name: pdbName, - Namespace: namespace, - } - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: objectMeta, +// pdbHasDeviceClassSelector reports whether a PDB carries a device-class selector +// clause, the marker that distinguishes class-scoped OSD PDBs from a classless one. +func pdbHasDeviceClassSelector(pdb *policyv1.PodDisruptionBudget) bool { + if pdb.Spec.Selector == nil { + return false } - err := r.client.Get(r.context.OpManagerContext, pdbRequest, &policyv1.PodDisruptionBudget{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil + for _, expr := range pdb.Spec.Selector.MatchExpressions { + if expr.Key == osd.DeviceClassLabelKey { + return true } - return errors.Wrapf(err, "failed to get pdb %q", pdb.Name) } - logger.Infof("deleting temporary blocking pdb with %q with maxUnavailable=0 for %q failure domain %q", pdbName, failureDomainType, failureDomainName) - return r.deletePDB(pdb) + return false } func (r *ReconcileClusterDisruption) initializePDBState(request reconcile.Request) (*corev1.ConfigMap, error) { @@ -241,271 +286,442 @@ func (r *ReconcileClusterDisruption) initializePDBState(request reconcile.Reques } else if err != nil { return pdbStateMap, errors.Wrapf(err, "failed to get the pdbStateMap %s", pdbStateMapRequest) } + // A ConfigMap whose drain-state keys were all deleted is stored with no data + // field and reloads with a nil Data map; guarantee it is writable. + if pdbStateMap.Data == nil { + pdbStateMap.Data = map[string]string{} + } return pdbStateMap, nil } +// reconcilePDBsForOSDs updates each group's drain state, then applies the whole desired +// OSD-PDB set before pruning any extra OSD PDB, so a pod is never matched by zero PDBs. func (r *ReconcileClusterDisruption) reconcilePDBsForOSDs( clusterInfo *cephclient.ClusterInfo, request reconcile.Request, pdbStateMap *corev1.ConfigMap, - failureDomainType string, - allFailureDomains, - osdDownFailureDomains, - nodeDrainFailureDomains []string, - downOSDs []int, + groups []*pdbGroup, pgHealthyRegex string, ) (reconcile.Result, error) { - pgHealthMsg, pgClean, err := cephclient.IsClusterClean(r.context.ClusterdContext, clusterInfo, pgHealthyRegex) - if err != nil { - // If the error contains that message, this means the cluster is not up and running - // No monitors are present and thus no ceph configuration has been created - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Debugf("ceph %q cluster not ready, cannot check status yet.", request.Namespace) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + namespace := clusterInfo.Namespace + + // When some group is scoped to a device class, computeDesiredPDBs also emits the + // rook-ceph-osd default (NotIn those classes) for OSDs of no managed class. + hasClassGroups := false + for _, g := range groups { + if g.deviceClass != "" { + hasClassGroups = true + break } - logger.Debugf("ceph %q cluster failed to check cluster health. %v", request.Namespace, err) - return opcontroller.WaitForRequeueIfCephClusterNotReady, nil } - osdDown := len(downOSDs) > 0 - // OSDs which should be excluded from the default PDB. This is done when there are no active drains and all PGs are - // active+clean, but there are some down OSDs. In that case we exclude the down OSDs from the PDB otherwise all drains - // in the cluster would be blocked until the down OSDs came back. - excludeOSDs := make([]int, 0) - - // switch block to update the PDB state config map based on the PG status and running OSDs - switch { - case !osdDown && pgClean: - logger.Infof("OSDs are up and PGs are clean. PG status: %q", pgHealthMsg) - resetPDBConfig(pdbStateMap) - case osdDown && pgClean: - logger.Infof("OSD(s) %v are down but PGs are clean. PG Status: %q", downOSDs, pgHealthMsg) - // In case of a node drain event, the OSD pods can get drained rapidly and it would take some time for rook to fetch - // the correct PG status. So wait for 60 seconds when OSD is down and node drain event is detected - if len(nodeDrainFailureDomains) > 0 { - lastNodeDrainTimeStamp, err := getLastNodeDrainTimeStamp(pdbStateMap, drainingFailureDomainDurationKey) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to get last node drain timestamp from the configmap %q", pdbStateMap.Name) - } - if time.Since(lastNodeDrainTimeStamp) < 60*time.Second { - logger.Infof("node drain is detected. Requeue to ensure that correct PG status is read.") - } else { - excludeOSDs = slices.Clone(downOSDs) + // per-group drain-state update + for _, g := range groups { + if g.degraded { + // FD unresolvable on some OSD: keep the group idle so its default PDB + // covers all its OSDs. Clear any prior drain state so computeDesiredPDBs + // takes the idle branch and updateNoout/requeue treat it as quiescent. + resetPDBConfig(pdbStateMap, g.keys()) + g.excludeOSDs = nil + continue + } + pgHealthMsg, pgClean, err := r.groupPGsClean(clusterInfo, g, pgHealthyRegex) + if err != nil { + // If the error contains that message, this means the cluster is not up and running + // No monitors are present and thus no ceph configuration has been created + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Debugf("ceph %q cluster not ready, cannot check status yet.", request.Namespace) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil } - } else { - excludeOSDs = slices.Clone(downOSDs) - resetPDBConfig(pdbStateMap) + logger.Debugf("ceph %q cluster failed to check cluster health. %v", request.Namespace, err) + return opcontroller.WaitForRequeueIfCephClusterNotReady, nil + } + if err := r.updateDrainState(pdbStateMap, g, pgClean, pgHealthMsg); err != nil { + return reconcile.Result{}, err } - case osdDown && !pgClean: - setPDBConfig(pdbStateMap, osdDownFailureDomains, nodeDrainFailureDomains) - logger.Infof("OSD(s) %v are down and PGs are not clean. PGs Status: %q", downOSDs, pgHealthMsg) - - // no-op. Wait for the PGs to become healthy from the previous node drain event - case !osdDown && !pgClean && len(pdbStateMap.Data[drainingFailureDomainKey]) > 1: - logger.Infof("OSDs are up but PGs are not clean from previous drain event. PGs Status: %q", pgHealthMsg) } - // handle drains based on the PDB config map - if pdbStateMap.Data[drainingFailureDomainKey] != "" { - logger.Infof("OSD failure Domains : %q", allFailureDomains) - logger.Infof("Draining Failure Domain: %q", pdbStateMap.Data[drainingFailureDomainKey]) - logger.Infof("Set noout on draining Failure Domain: %q", pdbStateMap.Data[setNoOut]) - // delete default OSD pdb and create blocking OSD pdbs - err := r.handleActiveDrains(allFailureDomains, pdbStateMap.Data[drainingFailureDomainKey], failureDomainType, clusterInfo.Namespace) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to handle active drains") + desired := computeDesiredPDBs(namespace, groups, pdbStateMap) + + // Apply every desired PDB before pruning, so a pod is never matched by zero PDBs. + // The rook-ceph-osd default is applied LAST: its NotIn selector grows when a class + // appears, narrowing coverage, so the per-class defaults that take it over must exist + // first. Other selectors only ever add coverage, so their order does not matter. + for name := range desired { + if hasClassGroups && name == osdPDBAppName { + continue } - } else if pdbStateMap.Data[drainingFailureDomainKey] == "" { - // delete all blocking OSD pdb and restore the default OSD pdb - err = r.handleInactiveDrains(allFailureDomains, failureDomainType, clusterInfo.Namespace, excludeOSDs) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to handle inactive drains") + if err := r.applyPDB(desired[name]); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to apply osd pdb %q", name) + } + } + if hasClassGroups { + if err := r.applyPDB(desired[osdPDBAppName]); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to apply osd pdb %q", osdPDBAppName) } } - err = r.updateNoout(clusterInfo, pdbStateMap, allFailureDomains) + // prune existing OSD PDBs not in the desired set + existing, err := r.listOSDPDBs(namespace) if err != nil { + return reconcile.Result{}, err + } + for i := range existing { + if _, ok := desired[existing[i].Name]; ok { + continue + } + logger.Infof("pruning osd pdb %q not in the desired set", existing[i].Name) + if err := r.deletePDB(&existing[i]); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to prune osd pdb %q", existing[i].Name) + } + } + + // reconcile noout across the union of all groups' draining failure domains + if err := r.updateNoout(clusterInfo, pdbStateMap, groups); err != nil { logger.Errorf("failed to update maintenance noout in cluster %q. %v", request, err) } - // update PDB configmap - err = r.client.Update(clusterInfo.Context, pdbStateMap) - if err != nil { + // drop drain-state keys of classes no longer present (a cluster-wide group clears + // all dc.* keys, so its ConfigMap state matches the baseline) + gcStaleDrainKeys(pdbStateMap, groups) + + if err := r.client.Update(clusterInfo.Context, pdbStateMap); err != nil { if errors.Is(err, context.Canceled) { return reconcile.Result{}, nil } return reconcile.Result{}, errors.Wrapf(err, "failed to update configMap %q in cluster %q", pdbStateMapName, request) } - return r.requeuePDBController(request) + return r.requeuePDBController(pdbStateMap, groups), nil } -func (r *ReconcileClusterDisruption) handleActiveDrains(allFailureDomains []string, drainingFailureDomain, - failureDomainType, namespace string, -) error { - for _, failureDomainName := range allFailureDomains { - // create blocking PDB for failure domains not currently draining - if failureDomainName != drainingFailureDomain { - err := r.createBlockingPDBForOSD(namespace, failureDomainType, failureDomainName) - if err != nil { - return errors.Wrapf(err, "failed to create blocking pdb for %q failure domain %q", failureDomainType, failureDomainName) - } - } else { - err := r.deleteBlockingPDBForOSD(namespace, failureDomainType, failureDomainName) - if err != nil { - return errors.Wrapf(err, "failed to delete blocking pdb for %q failure domain %q. %v", failureDomainType, failureDomainName, err) +// computeDesiredPDBs returns the OSD PDBs every group wants this reconcile, keyed by +// name. Pure over the groups' filled state and the drain-state ConfigMap, so it is +// unit tested directly. +func computeDesiredPDBs(namespace string, groups []*pdbGroup, pdbStateMap *corev1.ConfigMap) map[string]*policyv1.PodDisruptionBudget { + desired := map[string]*policyv1.PodDisruptionBudget{} + var classes []string + for _, g := range groups { + if g.deviceClass != "" { + classes = append(classes, g.deviceClass) + } + drainingFD := pdbStateMap.Data[g.keys().draining] + if drainingFD == "" { + // idle group: one maxUnavailable=1 default lets a single failure domain drain + desired[g.defaultPDBName()] = g.defaultPDB(namespace) + continue + } + // draining group: pin every failure domain except the draining one with a maxUnavailable=0 PDB + for _, fd := range g.state.allFailureDomains { + if fd == drainingFD { + continue } + desired[g.blockingPDBName(fd)] = g.blockingPDB(namespace, fd) } } + // With device-class groups present, the rook-ceph-osd default (NotIn those classes) + // protects OSDs that no class group selects. + if len(classes) > 0 { + slices.Sort(classes) + desired[osdPDBAppName] = defaultOSDPDB(namespace, osdPDBAppName, nil, classes, nil) + } + return desired +} - // delete the default PDB for OSD - // This will allow all OSDs in the currently drained failure domain to be removed. - logger.Debug("deleting default pdb with maxUnavailable=1 for all osd") - err := r.deleteDefaultPDBforOSD(namespace) - if err != nil { - return errors.Wrap(err, "failed to delete the default osd pdb") +// updateDrainState updates a single group's drain-state keys from its PG health +// and failure-domain state, and records the OSDs to exclude from its default PDB. +func (r *ReconcileClusterDisruption) updateDrainState(pdbStateMap *corev1.ConfigMap, g *pdbGroup, pgClean bool, pgHealthMsg string) error { + osdDown := len(g.state.downOSDs) > 0 + // OSDs to exclude from the group default PDB: when there are no active drains and PGs are + // clean but some OSDs are down, exclude them so drains elsewhere are not blocked while they recover. + g.excludeOSDs = make([]int, 0) + + name := g.defaultPDBName() + keys := g.keys() + switch { + case !osdDown && pgClean: + logger.Infof("group %q: OSDs are up and PGs are clean. PG status: %q", name, pgHealthMsg) + resetPDBConfig(pdbStateMap, keys) + case osdDown && pgClean: + logger.Infof("group %q: OSD(s) %v are down but PGs are clean. PG Status: %q", name, g.state.downOSDs, pgHealthMsg) + // In case of a node drain event, the OSD pods can get drained rapidly and it would take some time for rook to fetch + // the correct PG status. So wait for 60 seconds when OSD is down and node drain event is detected + if len(g.state.nodeDrainFailureDomains) > 0 { + lastNodeDrainTimeStamp, err := getLastNodeDrainTimeStamp(pdbStateMap, keys.duration) + if err != nil { + return errors.Wrapf(err, "failed to get last node drain timestamp for group %q", name) + } + if time.Since(lastNodeDrainTimeStamp) < 60*time.Second { + logger.Infof("group %q: node drain is detected. Requeue to ensure that correct PG status is read.", name) + } else { + g.excludeOSDs = slices.Clone(g.state.downOSDs) + } + } else { + g.excludeOSDs = slices.Clone(g.state.downOSDs) + resetPDBConfig(pdbStateMap, keys) + } + case osdDown && !pgClean: + logger.Infof("group %q: OSD(s) %v are down and PGs are not clean. PGs Status: %q", name, g.state.downOSDs, pgHealthMsg) + setPDBConfig(pdbStateMap, keys, g.state.osdDownFailureDomains, g.state.nodeDrainFailureDomains) + case !osdDown && !pgClean && pdbStateMap.Data[keys.draining] != "": + logger.Infof("group %q: OSDs are up but PGs are not clean from previous drain event. PGs Status: %q", name, pgHealthMsg) } return nil } -func (r *ReconcileClusterDisruption) handleInactiveDrains(allFailureDomains []string, failureDomainType, namespace string, excludeOSDs []int) error { - err := r.createDefaultPDBforOSD(namespace, excludeOSDs) - if err != nil { - return errors.Wrap(err, "failed to create default pdb") - } - for _, failureDomainName := range allFailureDomains { - err := r.deleteBlockingPDBForOSD(namespace, failureDomainType, failureDomainName) - if err != nil { - return errors.Wrapf(err, "failed to delete pdb for %q failure domain %q. %v", failureDomainType, failureDomainName, err) +// resetPDBConfig clears a group's drain-state keys (absent reads as "" everywhere, +// confirmed by findReferences on drainingFailureDomainKey). +func resetPDBConfig(pdbStateMap *corev1.ConfigMap, keys pdbDrainKeys) { + delete(pdbStateMap.Data, keys.draining) + delete(pdbStateMap.Data, keys.setNoOut) + delete(pdbStateMap.Data, keys.duration) +} + +// setPDBConfig records the draining failure domain and noout intent for one group. +// If there are unschedulable nodes (a node drain) those failure domains take +// precedence over failure domains where OSDs are merely down; noout is set only +// for node drains. +func setPDBConfig(pdbStateMap *corev1.ConfigMap, keys pdbDrainKeys, osdDownFailureDomains, nodeDrainFailureDomains []string) { + if pdbStateMap.Data[keys.draining] == "" { + if len(nodeDrainFailureDomains) > 0 { + pdbStateMap.Data[keys.draining] = nodeDrainFailureDomains[0] + pdbStateMap.Data[keys.setNoOut] = "true" + } else if len(osdDownFailureDomains) > 0 { + pdbStateMap.Data[keys.draining] = osdDownFailureDomains[0] + pdbStateMap.Data[keys.setNoOut] = "" + } + pdbStateMap.Data[keys.duration] = time.Now().Format(time.RFC3339) + } else { + // Update if the previously drained failure domain is back but another is down. + if len(nodeDrainFailureDomains) > 0 && !slices.Contains(nodeDrainFailureDomains, pdbStateMap.Data[keys.draining]) { + pdbStateMap.Data[keys.draining] = nodeDrainFailureDomains[0] + pdbStateMap.Data[keys.setNoOut] = "true" + } else if len(osdDownFailureDomains) > 0 && !slices.Contains(osdDownFailureDomains, pdbStateMap.Data[keys.draining]) { + pdbStateMap.Data[keys.draining] = osdDownFailureDomains[0] + pdbStateMap.Data[keys.setNoOut] = "" } - logger.Debugf("deleted temporary blocking pdb for %q failure domain %q.", failureDomainType, failureDomainName) + pdbStateMap.Data[keys.duration] = time.Now().Format(time.RFC3339) } - return nil } -func (r *ReconcileClusterDisruption) updateNoout(clusterInfo *cephclient.ClusterInfo, pdbStateMap *corev1.ConfigMap, allFailureDomains []string) error { +// updateNoout sets noout on the union of all groups' draining failure domains +// (subject to the maintenance timeout) and unsets it everywhere else. The union +// avoids two groups that share a failure-domain bucket name fighting over the flag. +func (r *ReconcileClusterDisruption) updateNoout(clusterInfo *cephclient.ClusterInfo, pdbStateMap *corev1.ConfigMap, groups []*pdbGroup) error { osdDump, err := cephclient.GetOSDDump(r.context.ClusterdContext, clusterInfo) if err != nil { return errors.Wrapf(err, "failed to get osddump for reconciling maintenance noout in namespace %s", clusterInfo.Namespace) } - for _, failureDomainName := range allFailureDomains { - drainingFailureDomainTimeStampKey := fmt.Sprintf("%s-noout-last-set-at", failureDomainName) - if pdbStateMap.Data[drainingFailureDomainKey] == failureDomainName { - if pdbStateMap.Data[setNoOut] == "true" { - // get the time stamp - nooutSetTimeString, ok := pdbStateMap.Data[drainingFailureDomainTimeStampKey] - if !ok || len(nooutSetTimeString) == 0 { - // initialize it if it's not set - pdbStateMap.Data[drainingFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339) + + nooutOn := map[string]bool{} + allFailureDomains := sets.New[string]() + for _, g := range groups { + keys := g.keys() + draining := pdbStateMap.Data[keys.draining] + holdNoout := pdbStateMap.Data[keys.setNoOut] == "true" + for _, failureDomainName := range g.state.allFailureDomains { + allFailureDomains.Insert(failureDomainName) + timestampKey := g.nooutTimestampKey(failureDomainName) + if failureDomainName == draining && holdNoout { + if pdbStateMap.Data[timestampKey] == "" { + pdbStateMap.Data[timestampKey] = time.Now().Format(time.RFC3339) } - // parse the timestamp - nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[drainingFailureDomainTimeStampKey]) + nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[timestampKey]) if err != nil { - return errors.Wrapf(err, "failed to parse timestamp %s for failureDomain %s", pdbStateMap.Data[drainingFailureDomainTimeStampKey], nooutSetTime) + return errors.Wrapf(err, "failed to parse noout timestamp %q for failure domain %q", pdbStateMap.Data[timestampKey], failureDomainName) } - if time.Since(nooutSetTime) >= r.maintenanceTimeout { - // noout expired - if _, err := osdDump.UpdateFlagOnCrushUnit(r.context.ClusterdContext, clusterInfo, false, failureDomainName, nooutFlag); err != nil { - return errors.Wrapf(err, "failed to update flag on crush unit when noout expired.") - } - } else { - // set noout - if _, err := osdDump.UpdateFlagOnCrushUnit(r.context.ClusterdContext, clusterInfo, true, failureDomainName, nooutFlag); err != nil { - return errors.Wrapf(err, "failed to update flag on crush unit while setting noout.") - } + if time.Since(nooutSetTime) < r.maintenanceTimeout { + nooutOn[failureDomainName] = true } } else { - if _, err := osdDump.UpdateFlagOnCrushUnit(r.context.ClusterdContext, clusterInfo, false, failureDomainName, nooutFlag); err != nil { - return errors.Wrapf(err, "failed to update flag on crush unit when ensuring noout is unset.") + delete(pdbStateMap.Data, timestampKey) + } + } + } + + for _, failureDomainName := range sets.List(allFailureDomains) { + if _, err := osdDump.UpdateFlagOnCrushUnit(r.context.ClusterdContext, clusterInfo, nooutOn[failureDomainName], failureDomainName, nooutFlag); err != nil { + return errors.Wrapf(err, "failed to update noout flag on crush unit %q", failureDomainName) + } + } + return nil +} + +// gcStaleDrainKeys deletes drain-state keys owned by groups that are not active: +// dc..* keys for classes not among the current groups (a cluster-wide group +// clears every dc.* key), and when there is no cluster-wide group, the bare keys a +// previous cluster-wide reconcile (including pre-upgrade) left behind. Without the +// latter, switching back to a cluster-wide group could resume a long-gone drain. +func gcStaleDrainKeys(pdbStateMap *corev1.ConfigMap, groups []*pdbGroup) { + hasGlobal := false + activePrefixes := make([]string, 0, len(groups)) + for _, g := range groups { + if g.deviceClass == "" { + hasGlobal = true + continue + } + activePrefixes = append(activePrefixes, fmt.Sprintf("%s%s.", dcKeyPrefix, g.deviceClass)) + } + for k := range pdbStateMap.Data { + if strings.HasPrefix(k, dcKeyPrefix) { + keep := false + for _, prefix := range activePrefixes { + if strings.HasPrefix(k, prefix) { + keep = true + break } - // delete the timestamp - delete(pdbStateMap.Data, drainingFailureDomainTimeStampKey) } - } else { - // ensure noout unset - if _, err := osdDump.UpdateFlagOnCrushUnit(r.context.ClusterdContext, clusterInfo, false, failureDomainName, nooutFlag); err != nil { - return errors.Wrapf(err, "failed to update flag on crush unit when ensuring noout is unset.") + if !keep { + delete(pdbStateMap.Data, k) } - // delete the timestamp - delete(pdbStateMap.Data, drainingFailureDomainTimeStampKey) + continue + } + if hasGlobal { + continue + } + // global keys: draining-failure-domain, set-no-out, the drain timestamp, + // and the per-failure-domain "-noout-last-set-at" noout timestamps + if k == drainingFailureDomainKey || k == setNoOut || k == drainingFailureDomainDurationKey || strings.HasSuffix(k, "-noout-last-set-at") { + delete(pdbStateMap.Data, k) } } - return nil } -func (r *ReconcileClusterDisruption) getOSDFailureDomains(clusterInfo *cephclient.ClusterInfo, request reconcile.Request, poolFailureDomain string) ([]string, []string, []string, []int, error) { +// requeuePDBController requeues while any group is actively draining or has a down +// OSD. Polling on a down OSD preserves the baseline self-heal: it drives the drain +// state machine forward as PG status changes (e.g. the post-drain 60s settle and +// the transition to a blocking layout) and removes a down OSD's default-PDB +// exclusion once it recovers — an OSD-deployment recovery raises no watch event, so +// without this poll the exclusion could linger. It keys off per-group drain state rather +// than the rook-ceph-osd default's DisruptionsAllowed, which reads 0 when it selects no OSDs. +func (r *ReconcileClusterDisruption) requeuePDBController(pdbStateMap *corev1.ConfigMap, groups []*pdbGroup) reconcile.Result { + for _, g := range groups { + if pdbStateMap.Data[g.keys().draining] != "" || len(g.state.downOSDs) > 0 { + logger.Info("reconciling osd pdb controller, a failure domain is draining or an OSD is down") + return reconcile.Result{Requeue: true, RequeueAfter: 30 * time.Second} + } + } + logger.Info("successfully reconciled OSD PDB controller") + return reconcile.Result{} +} + +// populateOSDFailureDomains buckets OSD deployments into their group and fills each +// group's failure-domain state. A cluster-wide group matches every OSD; per-class +// groups match by the device-class label. An OSD that no group matches is left for +// the rook-ceph-osd default. +func (r *ReconcileClusterDisruption) populateOSDFailureDomains(clusterInfo *cephclient.ClusterInfo, request reconcile.Request, groups []*pdbGroup) error { osdDeploymentList := &appsv1.DeploymentList{} namespaceListOpts := client.InNamespace(request.Namespace) - topologyLocationLabel := fmt.Sprintf(osd.TopologyLocationLabel, poolFailureDomain) - err := r.client.List(clusterInfo.Context, osdDeploymentList, client.MatchingLabels{k8sutil.AppAttr: osd.AppName}, namespaceListOpts) - if err != nil { - return nil, nil, nil, nil, errors.Wrap(err, "failed to list osd deployments") + if err := r.client.List(clusterInfo.Context, osdDeploymentList, client.MatchingLabels{k8sutil.AppAttr: osd.AppName}, namespaceListOpts); err != nil { + return errors.Wrap(err, "failed to list osd deployments") } - allFailureDomains := sets.New[string]() - nodeDrainFailureDomains := sets.New[string]() - osdDownFailureDomains := sets.New[string]() - downOSDs := []int{} - osdMetadata, err := cephclient.GetOSDMetadata(r.context.ClusterdContext, clusterInfo) if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "failed to get OSD status") + return errors.Wrapf(err, "failed to get OSD status") } - for _, deployment := range osdDeploymentList.Items { + type fdSets struct { + all sets.Set[string] + nodeDrain sets.Set[string] + osdDown sets.Set[string] + downOSDs []int + degraded bool + } + fdByClass := make(map[string]*fdSets, len(groups)) + for _, g := range groups { + fdByClass[g.deviceClass] = &fdSets{all: sets.New[string](), nodeDrain: sets.New[string](), osdDown: sets.New[string](), downOSDs: []int{}} + } + + for i := range osdDeploymentList.Items { + deployment := osdDeploymentList.Items[i] labels := deployment.GetLabels() + g := matchGroup(groups, labels[osd.DeviceClassLabelKey]) + if g == nil { + // no group matches this OSD's class: the rook-ceph-osd default covers it + continue + } + c := fdByClass[g.deviceClass] + topologyLocationLabel := fmt.Sprintf(osd.TopologyLocationLabel, g.failureDomainType) failureDomainName := labels[topologyLocationLabel] if failureDomainName == "" { - return nil, nil, nil, nil, errors.Errorf("failed to get the topology location label %q in OSD deployment %q", - topologyLocationLabel, deployment.Name) + // The group's failure-domain type does not resolve to a label on this OSD. + // Degrade the group to default-only rather than abort the whole reconcile + // (which would block every other group); its maxUnavailable=1 default still + // covers all its OSDs. + logger.Warningf("OSD deployment %q has no %q label; degrading group %q to a default-only PDB", deployment.Name, topologyLocationLabel, g.defaultPDBName()) + c.degraded = true + continue } + c.all.Insert(failureDomainName) // Assume node drain if osd deployment ReadyReplicas count is 0 and OSD pod is not scheduled on a node if deployment.Status.ReadyReplicas < 1 { - if !osdDownFailureDomains.Has(failureDomainName) { - osdDownFailureDomains.Insert(failureDomainName) - } + c.osdDown.Insert(failureDomainName) osdID, err := osd.GetOSDID(&deployment) if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "failed to get ID for the OSD deployment %q", deployment.Name) + return errors.Wrapf(err, "failed to get ID for the OSD deployment %q", deployment.Name) } - downOSDs = append(downOSDs, osdID) + c.downOSDs = append(c.downOSDs, osdID) - // check if OSD is down on unscheduleable node var osdNodeName string for _, metadata := range *osdMetadata { if metadata.Id == osdID { osdNodeName = metadata.HostName } } - if osdNodeName != "" { - isDrained, err := hasOSDNodeDrained(clusterInfo.Context, r.client, osdNodeName) - if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "failed to check if osd %q node is drained", deployment.Name) - } - if isDrained { - logger.Infof("osd %q is down on node %q and a possible node drain is detected", deployment.Name, osdNodeName) - if !nodeDrainFailureDomains.Has(failureDomainName) { - nodeDrainFailureDomains.Insert(failureDomainName) - } - } else { - if !strings.HasSuffix(deployment.Name, "-debug") { - logger.Infof("osd %q is down on node %q but no node drain is detected", deployment.Name, osdNodeName) - } - } - } else { + if osdNodeName == "" { logger.Warningf("failed to get the node name for the OSD %d", osdID) continue } + isDrained, err := hasOSDNodeDrained(clusterInfo.Context, r.client, osdNodeName) + if err != nil { + return errors.Wrapf(err, "failed to check if osd %q node is drained", deployment.Name) + } + if isDrained { + logger.Infof("osd %q is down on node %q and a possible node drain is detected", deployment.Name, osdNodeName) + c.nodeDrain.Insert(failureDomainName) + } else if !strings.HasSuffix(deployment.Name, "-debug") { + logger.Infof("osd %q is down on node %q but no node drain is detected", deployment.Name, osdNodeName) + } + } + } + for _, g := range groups { + c := fdByClass[g.deviceClass] + if c.degraded { + // Force the group idle: empty state means computeDesiredPDBs emits only the + // maxUnavailable=1 default, and updateNoout/requeue see no failure domains. + g.degraded = true + g.state = groupDrainState{} + continue + } + g.state = groupDrainState{ + allFailureDomains: sets.List(c.all), + nodeDrainFailureDomains: sets.List(c.nodeDrain), + osdDownFailureDomains: sets.List(c.osdDown), + downOSDs: c.downOSDs, } + } + return nil +} - if !allFailureDomains.Has(failureDomainName) { - allFailureDomains.Insert(failureDomainName) +// matchGroup returns the group an OSD with the given device class belongs to. A +// cluster-wide group (deviceClass "") matches every OSD; a class group matches only its +// class. Returns nil for an OSD that no group matches (left for the rook-ceph-osd default). +func matchGroup(groups []*pdbGroup, deviceClass string) *pdbGroup { + var clusterWide *pdbGroup + for _, g := range groups { + if g.deviceClass == "" { + clusterWide = g + continue + } + if g.deviceClass == deviceClass { + return g } } - return sets.List(allFailureDomains), sets.List(nodeDrainFailureDomains), sets.List(osdDownFailureDomains), downOSDs, nil + return clusterWide } // hasOSDNodeDrained returns true if OSD pod is not assigned to any node or if the OSD node is not schedulable @@ -529,69 +745,6 @@ func getNode(ctx context.Context, c client.Client, nodeName string) (*corev1.Nod return node, nil } -func getPDBName(failureDomainType, failureDomainName string) string { - return k8sutil.TruncateNodeName(fmt.Sprintf("%s-%s-%s", osdPDBAppName, failureDomainType, "%s"), failureDomainName) -} - -func resetPDBConfig(pdbStateMap *corev1.ConfigMap) { - pdbStateMap.Data[drainingFailureDomainKey] = "" - delete(pdbStateMap.Data, drainingFailureDomainDurationKey) - // reset `set-no-out` flag on the configMap - pdbStateMap.Data[setNoOut] = "" -} - -// setPDBConfig updates the OSD PDB config map. If there are unschedulable nodes (that is, a node drain event) -// then those failureDomains are given higher precedence than the failureDomains where OSDs might be down -// due to some reason but node is schedulable. `Noout` is set only if nodes are unscheduleable. -func setPDBConfig(pdbStateMap *corev1.ConfigMap, osdDownFailureDomains, nodeDrainFailureDomains []string) { - if len(pdbStateMap.Data[drainingFailureDomainKey]) == 0 { - if len(nodeDrainFailureDomains) > 0 { - pdbStateMap.Data[drainingFailureDomainKey] = nodeDrainFailureDomains[0] - pdbStateMap.Data[setNoOut] = "true" - } else if len(osdDownFailureDomains) > 0 { - pdbStateMap.Data[drainingFailureDomainKey] = osdDownFailureDomains[0] - pdbStateMap.Data[setNoOut] = "" - } - pdbStateMap.Data[drainingFailureDomainDurationKey] = time.Now().Format(time.RFC3339) - } else { - // Update the PDB configmap if the previously drained node is back but some other nodes are down. - if len(nodeDrainFailureDomains) > 0 && !slices.Contains(nodeDrainFailureDomains, pdbStateMap.Data[drainingFailureDomainKey]) { - pdbStateMap.Data[drainingFailureDomainKey] = nodeDrainFailureDomains[0] - pdbStateMap.Data[setNoOut] = "true" - } else if len(osdDownFailureDomains) > 0 && !slices.Contains(osdDownFailureDomains, pdbStateMap.Data[drainingFailureDomainKey]) { - pdbStateMap.Data[drainingFailureDomainKey] = osdDownFailureDomains[0] - pdbStateMap.Data[setNoOut] = "" - } - pdbStateMap.Data[drainingFailureDomainDurationKey] = time.Now().Format(time.RFC3339) - } -} - -// requeuePDBController returns requeue request with timeout if: -// - allowedDisruption in main PDB is 0, that is, One or more OSD went down. -// - MaxUnavailable in the main PDB is > 1, that is, OSDs are down but PGs might be clean. -// - default OSD PDB is not available. -func (r *ReconcileClusterDisruption) requeuePDBController(request reconcile.Request) (reconcile.Result, error) { - defaultPDB := &policyv1.PodDisruptionBudget{} - err := r.client.Get(r.context.OpManagerContext, types.NamespacedName{Name: osdPDBAppName, Namespace: request.Namespace}, defaultPDB) - if err != nil { - if apierrors.IsNotFound(err) { - logger.Debugf("default osd pdb %q not found", osdPDBAppName) - logger.Info("reconciling osd pdb controller") - return reconcile.Result{Requeue: true, RequeueAfter: 15 * time.Second}, nil - } else { - return reconcile.Result{}, errors.Wrapf(err, "failed to get allowed disruptions count from default osd pdb %q.", osdPDBAppName) - } - } - - if defaultPDB.Status.DisruptionsAllowed == 0 || pdbExcludesOSDs(defaultPDB) { - logger.Info("reconciling osd pdb controller") - return reconcile.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil - } - - logger.Info("successfully reconciled OSD PDB controller") - return reconcile.Result{}, nil -} - func getLastNodeDrainTimeStamp(pdbStateMap *corev1.ConfigMap, key string) (time.Time, error) { var err error var lastDrainTimeStamp time.Time @@ -608,21 +761,3 @@ func getLastNodeDrainTimeStamp(pdbStateMap *corev1.ConfigMap, key string) (time. } return lastDrainTimeStamp, nil } - -func pdbExcludesOSDs(pdb *policyv1.PodDisruptionBudget) bool { - if pdb == nil { - return false - } - if pdb.Spec.Selector == nil { - return false - } - if pdb.Spec.Selector.MatchExpressions == nil { - return false - } - for _, matchExpression := range pdb.Spec.Selector.MatchExpressions { - if matchExpression.Key == osdPDBOsdIdLabel { - return true - } - } - return false -} diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass.go new file mode 100644 index 000000000000..aa3943227fd5 --- /dev/null +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass.go @@ -0,0 +1,187 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterdisruption + +import ( + "fmt" + "slices" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology" + "github.com/rook/rook/pkg/operator/k8sutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// dcKeyPrefix marks the per-class ConfigMap keys for bulk cleanup. ConfigMap keys +// must match [-._a-zA-Z0-9]+, so "/" is invalid; the class-aware keys use "." as +// the separator: dc... +const dcKeyPrefix = "dc." + +func dcDrainingKey(class string) string { + return fmt.Sprintf("%s%s.%s", dcKeyPrefix, class, drainingFailureDomainKey) +} + +func dcSetNoOutKey(class string) string { + return fmt.Sprintf("%s%s.%s", dcKeyPrefix, class, setNoOut) +} + +func dcDurationKey(class string) string { + return fmt.Sprintf("%s%s.%s", dcKeyPrefix, class, drainingFailureDomainDurationKey) +} + +func dcNooutTimestampKey(class, failureDomainName string) string { + return fmt.Sprintf("%s%s.%s.noout-last-set-at", dcKeyPrefix, class, failureDomainName) +} + +// classDrainKeys is the per-class constructor for pdbDrainKeys. +func classDrainKeys(class string) pdbDrainKeys { + return pdbDrainKeys{ + draining: dcDrainingKey(class), + setNoOut: dcSetNoOutKey(class), + duration: dcDurationKey(class), + } +} + +func perClassDefaultPDBName(class string) string { + return fmt.Sprintf("%s-%s", osdPDBAppName, class) +} + +func perClassBlockingPDBName(class, failureDomainType, failureDomainName string) string { + return k8sutil.TruncateNodeName(fmt.Sprintf("%s-%s-%s-%s", osdPDBAppName, class, failureDomainType, "%s"), failureDomainName) +} + +// globalFailureDomain reduces the in-use pools' failure-domain types to the finest one in +// the CRUSH hierarchy — the cluster-wide failure domain for the cluster-wide OSD PDB group. +// Defaults to cephv1.DefaultFailureDomain when no in-use pool resolves a known level. +func globalFailureDomain(layout *cephclient.DeviceClassPDBLayout) string { + minIndex := -1 + for _, fdType := range layout.FailureDomainTypes { + for i, level := range topology.CRUSHMapLevelsOrdered { + if level == fdType { + if minIndex == -1 || i < minIndex { + minIndex = i + } + break + } + } + } + if minIndex == -1 { + return cephv1.DefaultFailureDomain + } + return topology.CRUSHMapLevelsOrdered[minIndex] +} + +// sortedClasses returns the device class names from the layout in sorted order. +func sortedClasses(layout *cephclient.DeviceClassPDBLayout) []string { + classes := make([]string, 0, len(layout.Classes)) + for class := range layout.Classes { + classes = append(classes, class) + } + slices.Sort(classes) + return classes +} + +// buildOSDPDBGroups reads the live CRUSH map and returns the OSD PDB groups to reconcile, +// each with its OSD failure-domain state populated: one group per device class when the +// in-use pools partition into more than one device class, otherwise a single cluster-wide +// group. Every error here is a ceph/api read failure the caller treats as "not ready yet". +func (r *ReconcileClusterDisruption) buildOSDPDBGroups(clusterInfo *cephclient.ClusterInfo, request reconcile.Request) ([]*pdbGroup, error) { + layout, err := cephclient.GetDeviceClassPDBLayout(r.context.ClusterdContext, clusterInfo) + if err != nil { + return nil, err + } + + var groups []*pdbGroup + classes := sortedClasses(layout) + if len(classes) <= 1 { + groups = []*pdbGroup{newPDBGroup("", globalFailureDomain(layout), nil)} + } else { + logger.Debugf("using device-class-aware OSD PDB groups for device classes %v", classes) + groups = make([]*pdbGroup, 0, len(classes)) + for _, class := range classes { + info := layout.Classes[class] + groups = append(groups, newPDBGroup(class, info.FailureDomainType, info.Pools)) + } + } + + if err := r.populateOSDFailureDomains(clusterInfo, request, groups); err != nil { + return nil, err + } + return groups, nil +} + +// newPDBGroup builds an OSD-PDB group. An empty deviceClass yields the cluster-wide group; +// a non-empty one yields a device-class group over the given pools. +func newPDBGroup(deviceClass, failureDomainType string, pools []string) *pdbGroup { + return &pdbGroup{ + deviceClass: deviceClass, + failureDomainType: failureDomainType, + pools: pools, + } +} + +// The cluster-wide group ("") keeps the bare rook-ceph-osd names, keys, and cluster-wide +// PG health, so existing clusters need no migration; a device-class group uses the +// dc..* names, keys, and per-class PG health. These derivations are why the +// reconcile never has to branch on deviceClass itself. + +func (g *pdbGroup) defaultPDBName() string { + if g.deviceClass == "" { + return osdPDBAppName + } + return perClassDefaultPDBName(g.deviceClass) +} + +// deviceClassIn is the device-class "In" selector clause for the group's default PDB: +// nil for the cluster-wide group, []string{deviceClass} for a device-class group. +func (g *pdbGroup) deviceClassIn() []string { + if g.deviceClass == "" { + return nil + } + return []string{g.deviceClass} +} + +func (g *pdbGroup) keys() pdbDrainKeys { + if g.deviceClass == "" { + return globalDrainKeys + } + return classDrainKeys(g.deviceClass) +} + +func (g *pdbGroup) blockingPDBName(failureDomainName string) string { + if g.deviceClass == "" { + return getPDBName(g.failureDomainType, failureDomainName) + } + return perClassBlockingPDBName(g.deviceClass, g.failureDomainType, failureDomainName) +} + +func (g *pdbGroup) nooutTimestampKey(failureDomainName string) string { + if g.deviceClass == "" { + return fmt.Sprintf("%s-noout-last-set-at", failureDomainName) + } + return dcNooutTimestampKey(g.deviceClass, failureDomainName) +} + +// groupPGsClean reports whether the group's PGs are healthy: cluster-wide for the +// cluster-wide group, scoped to the class's pools for a device-class group. +func (r *ReconcileClusterDisruption) groupPGsClean(clusterInfo *cephclient.ClusterInfo, g *pdbGroup, pgHealthyRegex string) (string, bool, error) { + if g.deviceClass == "" { + return cephclient.IsClusterClean(r.context.ClusterdContext, clusterInfo, pgHealthyRegex) + } + return cephclient.IsDeviceClassClean(r.context.ClusterdContext, clusterInfo, g.pools, pgHealthyRegex) +} diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass_test.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass_test.go new file mode 100644 index 000000000000..cb9e7b604ada --- /dev/null +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_deviceclass_test.go @@ -0,0 +1,760 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterdisruption + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// fakeClassedOSD builds an OSD deployment carrying the device-class and host +// topology labels the class-aware path reads. host is used both as the +// topology-location-host failure domain and as the node/metadata hostname. +func fakeClassedOSD(id, readyReplicas int, deviceClass, host string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("rook-ceph-osd-%d", id), + Namespace: namespace, + Labels: map[string]string{ + "app": "rook-ceph-osd", + "device-class": deviceClass, + "topology-location-host": host, + "ceph-osd-id": fmt.Sprintf("%d", id), + }, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: int32(readyReplicas), // nolint:gosec // G115 small test value + }, + } +} + +func osdPodLabels(deviceClass, host string, id int) map[string]string { + l := map[string]string{ + "app": "rook-ceph-osd", + "topology-location-host": host, + "ceph-osd-id": fmt.Sprintf("%d", id), + } + if deviceClass != "" { + l["device-class"] = deviceClass + } + return l +} + +// deviceClassExecutor mocks the ceph commands the class-aware reconcile issues. +// uncleanPools lists pools whose `pg ls-by-pool` returns a non-clean PG. +func deviceClassExecutor(osdMetadata string, uncleanPools map[string]bool) (*exectest.MockExecutor, *[]string) { + setGroupCalls := &[]string{} + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + switch { + case args[0] == "status": + // global-group IsClusterClean (fallback path); class groups use pg ls-by-pool + return healthyCephStatus, nil + case args[0] == "osd" && args[1] == "metadata": + return osdMetadata, nil + case args[0] == "osd" && args[1] == "dump": + return `{"OSDs":[]}`, nil + case args[0] == "pg" && args[1] == "ls-by-pool": + pool := args[2] + if uncleanPools[pool] { + return `{"pg_stats":[{"state":"active+clean"},{"state":"active+recovering"}]}`, nil + } + return `{"pg_stats":[{"state":"active+clean"},{"state":"active+clean"}]}`, nil + case args[0] == "osd" && args[1] == "set-group": + *setGroupCalls = append(*setGroupCalls, args[3]) // crush unit (failure domain) + return "", nil + case args[0] == "osd" && args[1] == "unset-group": + return "", nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + return executor, setGroupCalls +} + +func twoClassLayout() *cephclient.DeviceClassPDBLayout { + return &cephclient.DeviceClassPDBLayout{ + Classes: map[string]*cephclient.DeviceClassInfo{ + "ssd": {FailureDomainType: "host", Pools: []string{"ssd-pool"}}, + "hdd": {FailureDomainType: "host", Pools: []string{"hdd-pool"}}, + }, + } +} + +func pdbNames(t *testing.T, r *ReconcileClusterDisruption) []string { + list := &policyv1.PodDisruptionBudgetList{} + assert.NoError(t, r.client.List(context.TODO(), list)) + names := make([]string, 0, len(list.Items)) + for i := range list.Items { + names = append(names, list.Items[i].Name) + } + sort.Strings(names) + return names +} + +func getPDB(r *ReconcileClusterDisruption, name string) *policyv1.PodDisruptionBudget { + pdb := &policyv1.PodDisruptionBudget{} + err := r.client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, pdb) + if err != nil { + return nil + } + return pdb +} + +// countMatchingPDBs returns how many of the namespace's PDBs select a pod with +// the given labels — the eviction-time count that must be exactly 1 in steady state. +func countMatchingPDBs(t *testing.T, r *ReconcileClusterDisruption, podLabels map[string]string) int { + list := &policyv1.PodDisruptionBudgetList{} + assert.NoError(t, r.client.List(context.TODO(), list)) + count := 0 + for i := range list.Items { + sel, err := metav1.LabelSelectorAsSelector(list.Items[i].Spec.Selector) + assert.NoError(t, err) + if sel.Matches(labels.Set(podLabels)) { + count++ + } + } + return count +} + +func newClassAwareReconciler(t *testing.T, executor *exectest.MockExecutor, objs ...runtime.Object) *ReconcileClusterDisruption { + r := getFakeReconciler(t, objs...) + r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}, OpManagerContext: context.TODO()} + r.maintenanceTimeout = 30 * time.Minute + return r +} + +func runClassAware(t *testing.T, r *ReconcileClusterDisruption, cm *corev1.ConfigMap, layout *cephclient.DeviceClassPDBLayout) reconcile.Result { + clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} + + var groups []*pdbGroup + for _, class := range sortedClasses(layout) { + info := layout.Classes[class] + groups = append(groups, newPDBGroup(class, info.FailureDomainType, info.Pools)) + } + assert.NoError(t, r.populateOSDFailureDomains(clusterInfo, request, groups)) + + result, err := r.reconcilePDBsForOSDs(clusterInfo, request, cm, groups, "") + assert.NoError(t, err) + return result +} + +// runFallback drives the single global-group path (the fallback) through the +// unified reconcile, the way reconcile() does when a cluster is not class-eligible. +func runFallback(t *testing.T, r *ReconcileClusterDisruption, cm *corev1.ConfigMap, failureDomainType string) reconcile.Result { + clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} + + groups := []*pdbGroup{newPDBGroup("", failureDomainType, nil)} + assert.NoError(t, r.populateOSDFailureDomains(clusterInfo, request, groups)) + + result, err := r.reconcilePDBsForOSDs(clusterInfo, request, cm, groups, "") + assert.NoError(t, err) + return result +} + +func TestClassAwareIdle(t *testing.T) { + // Two classes, all OSDs up, all PGs clean: catch-all + one default per class. + cm := fakePDBConfigMap("") + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "ssd", "node-a"), + fakeClassedOSD(1, 1, "hdd", "node-b"), + } + executor, _ := deviceClassExecutor(`[{"id":0,"hostname":"node-a"},{"id":1,"hostname":"node-b"}]`, nil) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-a", false), getNodeObject("node-b", false))...) + + result := runClassAware(t, r, cm, twoClassLayout()) + assert.Zero(t, result.RequeueAfter) + + assert.Equal(t, []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd"}, pdbNames(t, r)) + + // catch-all carries a device-class NotIn clause covering both classes + catchAll := getPDB(r, "rook-ceph-osd") + assert.True(t, pdbHasDeviceClassSelector(catchAll)) + assert.Equal(t, int32(1), catchAll.Spec.MaxUnavailable.IntVal) + + // every OSD pod (and a classless one) is matched by exactly one PDB + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("ssd", "node-a", 0))) + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("hdd", "node-b", 1))) + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("", "node-c", 9))) + + // no class is draining + for k, v := range cm.Data { + assert.False(t, k == dcDrainingKey("ssd") && v != "", "ssd should not be draining") + assert.False(t, k == dcDrainingKey("hdd") && v != "", "hdd should not be draining") + } +} + +func TestClassAwareOneClassDraining(t *testing.T) { + // ssd node-a1 is drained (osd down, node unschedulable) with unclean ssd PGs. + // hdd stays healthy. ssd gets a blocking PDB for its other failure domain and + // no per-class default; hdd keeps its default. + cm := fakePDBConfigMap("") + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-a1"), // down + drained + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 1, "hdd", "node-b"), + } + executor, setGroup := deviceClassExecutor( + `[{"id":0,"hostname":"node-a1"},{"id":1,"hostname":"node-a2"},{"id":2,"hostname":"node-b"}]`, + map[string]bool{"ssd-pool": true}) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-a1", true), getNodeObject("node-a2", false), getNodeObject("node-b", false))...) + + result := runClassAware(t, r, cm, twoClassLayout()) + assert.NotZero(t, result.RequeueAfter) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd", + "rook-ceph-osd-ssd-host-node-a2", + }, pdbNames(t, r)) + + // ssd recorded as draining node-a1 with noout set + assert.Equal(t, "node-a1", cm.Data[dcDrainingKey("ssd")]) + assert.Equal(t, "true", cm.Data[dcSetNoOutKey("ssd")]) + assert.Equal(t, "", cm.Data[dcDrainingKey("hdd")]) + assert.Contains(t, *setGroup, "node-a1") // noout set on the drained failure domain + + // the draining ssd OSD on node-a1 matches no PDB (free to evict) + assert.Equal(t, 0, countMatchingPDBs(t, r, osdPodLabels("ssd", "node-a1", 0))) + // the non-draining ssd OSD is blocked by exactly one PDB + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("ssd", "node-a2", 1))) + // hdd is unaffected + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("hdd", "node-b", 2))) +} + +func TestClassAwareTwoClassesDrainingDifferentDomains(t *testing.T) { + // ssd drains node-a1, hdd drains node-b1 concurrently. Each class gets its own + // blocking PDB for its surviving domain; neither blocks the other. + cm := fakePDBConfigMap("") + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-a1"), + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 0, "hdd", "node-b1"), + fakeClassedOSD(3, 1, "hdd", "node-b2"), + } + executor, _ := deviceClassExecutor( + `[{"id":0,"hostname":"node-a1"},{"id":1,"hostname":"node-a2"},{"id":2,"hostname":"node-b1"},{"id":3,"hostname":"node-b2"}]`, + map[string]bool{"ssd-pool": true, "hdd-pool": true}) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-a1", true), getNodeObject("node-a2", false), + getNodeObject("node-b1", true), getNodeObject("node-b2", false))...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd-host-node-b2", + "rook-ceph-osd-ssd-host-node-a2", + }, pdbNames(t, r)) + assert.Equal(t, "node-a1", cm.Data[dcDrainingKey("ssd")]) + assert.Equal(t, "node-b1", cm.Data[dcDrainingKey("hdd")]) +} + +func TestClassAwareTwoClassesDrainingSameDomain(t *testing.T) { + // Both classes share node-x and drain it. noout is set once (union) and each + // class keeps a blocking PDB for the shared surviving domain node-y. + cm := fakePDBConfigMap("") + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-x"), + fakeClassedOSD(1, 1, "ssd", "node-y"), + fakeClassedOSD(2, 0, "hdd", "node-x"), + fakeClassedOSD(3, 1, "hdd", "node-y"), + } + executor, setGroup := deviceClassExecutor( + `[{"id":0,"hostname":"node-x"},{"id":1,"hostname":"node-y"},{"id":2,"hostname":"node-x"},{"id":3,"hostname":"node-y"}]`, + map[string]bool{"ssd-pool": true, "hdd-pool": true}) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-x", true), getNodeObject("node-y", false))...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd-host-node-y", + "rook-ceph-osd-ssd-host-node-y", + }, pdbNames(t, r)) + assert.Equal(t, "node-x", cm.Data[dcDrainingKey("ssd")]) + assert.Equal(t, "node-x", cm.Data[dcDrainingKey("hdd")]) + + // noout set exactly once on the shared node-x bucket (no per-class fighting) + nodeXCount := 0 + for _, fd := range *setGroup { + if fd == "node-x" { + nodeXCount++ + } + } + assert.Equal(t, 1, nodeXCount) +} + +func TestClassAwareIndependentCompletion(t *testing.T) { + // ssd finished draining (up + clean) while hdd is still draining node-b1. + // ssd returns to its per-class default without waiting for hdd. + cm := fakePDBConfigMap("") + cm.Data[dcDrainingKey("ssd")] = "node-a1" + cm.Data[dcSetNoOutKey("ssd")] = "true" + cm.Data[dcDrainingKey("hdd")] = "node-b1" + cm.Data[dcSetNoOutKey("hdd")] = "true" + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "ssd", "node-a1"), // back up + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 0, "hdd", "node-b1"), // still draining + fakeClassedOSD(3, 1, "hdd", "node-b2"), + } + executor, _ := deviceClassExecutor( + `[{"id":0,"hostname":"node-a1"},{"id":1,"hostname":"node-a2"},{"id":2,"hostname":"node-b1"},{"id":3,"hostname":"node-b2"}]`, + map[string]bool{"hdd-pool": true}) // only hdd still unclean + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-a1", false), getNodeObject("node-a2", false), + getNodeObject("node-b1", true), getNodeObject("node-b2", false))...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd-host-node-b2", + "rook-ceph-osd-ssd", + }, pdbNames(t, r)) + assert.Equal(t, "", cm.Data[dcDrainingKey("ssd")], "ssd should have completed") + assert.Equal(t, "node-b1", cm.Data[dcDrainingKey("hdd")], "hdd should still be draining") +} + +func TestClassAwareApplyBeforePruneOnDrainStart(t *testing.T) { + // Start from the settled idle state (per-class defaults present), then begin a + // drain. The ssd per-class default must be replaced by blocking PDBs, and the + // end state never leaves the surviving ssd OSD uncovered. + cm := fakePDBConfigMap("") + // pre-seed the idle per-class defaults and catch-all + preexisting := []runtime.Object{ + defaultOSDPDB(namespace, "rook-ceph-osd", nil, []string{"hdd", "ssd"}, nil), + defaultOSDPDB(namespace, "rook-ceph-osd-ssd", []string{"ssd"}, nil, nil), + defaultOSDPDB(namespace, "rook-ceph-osd-hdd", []string{"hdd"}, nil, nil), + } + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-a1"), + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 1, "hdd", "node-b"), + } + executor, _ := deviceClassExecutor( + `[{"id":0,"hostname":"node-a1"},{"id":1,"hostname":"node-a2"},{"id":2,"hostname":"node-b"}]`, + map[string]bool{"ssd-pool": true}) + allObjs := append(osds, cephCluster, cm, + getNodeObject("node-a1", true), getNodeObject("node-a2", false), getNodeObject("node-b", false)) + allObjs = append(allObjs, preexisting...) + r := newClassAwareReconciler(t, executor, allObjs...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd", + "rook-ceph-osd-ssd-host-node-a2", + }, pdbNames(t, r)) + // the stale ssd per-class default was pruned + assert.Nil(t, getPDB(r, "rook-ceph-osd-ssd")) + // the surviving ssd OSD remains covered by exactly one PDB throughout + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("ssd", "node-a2", 1))) +} + +func TestClassAwareUpgradeIdleAdoptsDefault(t *testing.T) { + // An old plain rook-ceph-osd default and an old classless blocking PDB exist. + // On an idle class-aware reconcile the default is overwritten to the catch-all + // and the classless blocking PDB is pruned. + cm := fakePDBConfigMap("") + oldDefault := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: "rook-ceph-osd", Namespace: namespace}, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{IntVal: 1}, + Selector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpIn, Values: []string{"rook-ceph-osd"}}, + }}, + }, + } + oldBlocking := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: "rook-ceph-osd-host-node-a", Namespace: namespace}, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{IntVal: 0}, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"topology-location-host": "node-a"}}, + }, + } + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "ssd", "node-a"), + fakeClassedOSD(1, 1, "hdd", "node-b"), + } + executor, _ := deviceClassExecutor(`[{"id":0,"hostname":"node-a"},{"id":1,"hostname":"node-b"}]`, nil) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, oldDefault, oldBlocking, + getNodeObject("node-a", false), getNodeObject("node-b", false))...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd"}, pdbNames(t, r)) + // rook-ceph-osd now carries the catch-all device-class NotIn clause + assert.True(t, pdbHasDeviceClassSelector(getPDB(r, "rook-ceph-osd"))) +} + +func TestClassAwareUpgradeMidDrain(t *testing.T) { + // Mid-drain upgrade: an old classless blocking PDB is present while ssd node-a1 + // is being drained. The class-aware path recomputes the drain from live state, + // prunes the old classless PDB, and keeps the surviving OSDs covered. + // The pre-upgrade global drain keys recorded for the same drain must be cleared. + cm := fakePDBConfigMap("node-a1") + cm.Data[setNoOut] = "true" + cm.Data[drainingFailureDomainDurationKey] = "2026-01-01T00:00:00Z" + cm.Data["node-a1-noout-last-set-at"] = "2026-01-01T00:00:00Z" + oldBlocking := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: "rook-ceph-osd-host-node-a2", Namespace: namespace}, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{IntVal: 0}, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"topology-location-host": "node-a2"}}, + }, + } + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-a1"), + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 1, "hdd", "node-b"), + } + executor, _ := deviceClassExecutor( + `[{"id":0,"hostname":"node-a1"},{"id":1,"hostname":"node-a2"},{"id":2,"hostname":"node-b"}]`, + map[string]bool{"ssd-pool": true}) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, oldBlocking, + getNodeObject("node-a1", true), getNodeObject("node-a2", false), getNodeObject("node-b", false))...) + + runClassAware(t, r, cm, twoClassLayout()) + + assert.Equal(t, []string{ + "rook-ceph-osd", + "rook-ceph-osd-hdd", + "rook-ceph-osd-ssd-host-node-a2", + }, pdbNames(t, r)) + // the old classless blocking PDB is gone + assert.Nil(t, getPDB(r, "rook-ceph-osd-host-node-a2")) + assert.Equal(t, "node-a1", cm.Data[dcDrainingKey("ssd")]) + // stale global drain keys are cleared so a later fallback cannot resume them + for _, k := range []string{drainingFailureDomainKey, setNoOut, drainingFailureDomainDurationKey, "node-a1-noout-last-set-at"} { + assert.NotContains(t, cm.Data, k) + } +} + +func countMatchInSet(t *testing.T, pdbs []policyv1.PodDisruptionBudget, podLabels map[string]string) int { + count := 0 + for i := range pdbs { + sel, err := metav1.LabelSelectorAsSelector(pdbs[i].Spec.Selector) + assert.NoError(t, err) + if sel.Matches(labels.Set(podLabels)) { + count++ + } + } + return count +} + +func TestClassAwareNoZeroCoverageOnEligibilityTransition(t *testing.T) { + // Transition fallback -> class-aware: the live rook-ceph-osd is the plain default + // covering all OSDs and is narrowed in place to the NotIn catch-all. An + // interceptor asserts that after EVERY PDB write, a classed OSD pod still matches + // at least one PDB — i.e. the catch-all is never narrowed before the per-class + // defaults that replace its coverage exist (the apply-catch-all-last fix). + cm := fakePDBConfigMap("") + plainDefault := defaultOSDPDB(namespace, osdPDBAppName, nil, nil, nil) // covers all OSDs (fallback state) + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "hdd", "node-a"), + fakeClassedOSD(1, 1, "ssd", "node-b"), + } + executor, _ := deviceClassExecutor(`[{"id":0,"hostname":"node-a"},{"id":1,"hostname":"node-b"}]`, nil) + + hddPod := osdPodLabels("hdd", "node-a", 0) + ssdPod := osdPodLabels("ssd", "node-b", 1) + + // On each PDB write, check coverage of the post-write set (current store overlaid + // with the object being written). + checkCoverage := func(ctx context.Context, c client.WithWatch, obj client.Object) { + pdb, ok := obj.(*policyv1.PodDisruptionBudget) + if !ok { + return + } + list := &policyv1.PodDisruptionBudgetList{} + assert.NoError(t, c.List(ctx, list, client.InNamespace(namespace))) + set := map[string]policyv1.PodDisruptionBudget{} + for i := range list.Items { + set[list.Items[i].Name] = list.Items[i] + } + set[pdb.Name] = *pdb + pdbs := make([]policyv1.PodDisruptionBudget, 0, len(set)) + for _, p := range set { + pdbs = append(pdbs, p) + } + assert.GreaterOrEqualf(t, countMatchInSet(t, pdbs, hddPod), 1, "hdd OSD left uncovered after writing %q", pdb.Name) + assert.GreaterOrEqualf(t, countMatchInSet(t, pdbs, ssdPod), 1, "ssd OSD left uncovered after writing %q", pdb.Name) + } + funcs := interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + checkCoverage(ctx, c, obj) + return c.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + checkCoverage(ctx, c, obj) + return c.Update(ctx, obj, opts...) + }, + } + + s := scheme.Scheme + assert.NoError(t, policyv1.AddToScheme(s)) + assert.NoError(t, appsv1.AddToScheme(s)) + assert.NoError(t, corev1.AddToScheme(s)) + objs := append(osds, cephCluster, cm, plainDefault, + getNodeObject("node-a", false), getNodeObject("node-b", false)) + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).WithInterceptorFuncs(funcs).Build() + r := &ReconcileClusterDisruption{ + client: cl, + scheme: s, + clusterMap: &ClusterMap{clusterMap: map[string]*cephv1.CephCluster{namespace: cephCluster}}, + context: &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}, OpManagerContext: context.TODO()}, + maintenanceTimeout: 30 * time.Minute, + } + + runClassAware(t, r, cm, twoClassLayout()) + + // end state is the catch-all + per-class defaults, each OSD matched by exactly one + assert.Equal(t, []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd"}, pdbNames(t, r)) + assert.Equal(t, 1, countMatchingPDBs(t, r, hddPod)) + assert.Equal(t, 1, countMatchingPDBs(t, r, ssdPod)) +} + +func TestFallbackConvergenceFromClassAware(t *testing.T) { + // Eligible-to-ineligible transition: the single global-group reconcile prunes + // the class-scoped PDBs and dc.* keys while (re)establishing the plain + // rook-ceph-osd default that covers all OSDs (apply before prune). + cm := fakePDBConfigMap("") + cm.Data[dcDrainingKey("ssd")] = "node-a1" + cm.Data[dcSetNoOutKey("ssd")] = "true" + cm.Data["dc.ssd.node-a1.noout-last-set-at"] = "2026-01-01T00:00:00Z" + + classScopedPDBs := []runtime.Object{ + defaultOSDPDB(namespace, "rook-ceph-osd", nil, []string{"hdd", "ssd"}, nil), + defaultOSDPDB(namespace, "rook-ceph-osd-hdd", []string{"hdd"}, nil, nil), + blockingOSDPDB(namespace, "rook-ceph-osd-ssd-host-node-a2", "ssd", "topology-location-host", "node-a2"), + } + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "hdd", "node-a"), + fakeClassedOSD(1, 1, "ssd", "node-b"), + } + executor, _ := deviceClassExecutor(`[{"id":0,"hostname":"node-a"},{"id":1,"hostname":"node-b"}]`, nil) + objs := append(classScopedPDBs, cephCluster, cm, + getNodeObject("node-a", false), getNodeObject("node-b", false)) + objs = append(objs, osds...) + r := newClassAwareReconciler(t, executor, objs...) + + runFallback(t, r, cm, "host") + + // only the plain default remains; class-scoped PDBs were pruned + assert.Equal(t, []string{"rook-ceph-osd"}, pdbNames(t, r)) + // rook-ceph-osd is now the plain default (no device-class selector) + assert.False(t, pdbHasDeviceClassSelector(getPDB(r, "rook-ceph-osd"))) + + // all dc.* keys cleared + for k := range cm.Data { + assert.NotContains(t, k, "dc.") + } +} + +func TestClassAwareDegradesWhenFDLabelMissing(t *testing.T) { + // The ssd class's CRUSH-derived failure domain is "rack", but the OSD deployments are + // only labeled topology-location-host. The ssd group must degrade to a default-only PDB + // (covering its OSDs) instead of aborting the whole reconcile; the hdd group (FD host, + // resolvable) proceeds normally. + cm := fakePDBConfigMap("") + layout := &cephclient.DeviceClassPDBLayout{ + Classes: map[string]*cephclient.DeviceClassInfo{ + "ssd": {FailureDomainType: "rack", Pools: []string{"ssd-pool"}}, + "hdd": {FailureDomainType: "host", Pools: []string{"hdd-pool"}}, + }, + } + osds := []runtime.Object{ + fakeClassedOSD(0, 1, "ssd", "node-a"), // only topology-location-host, no rack + fakeClassedOSD(1, 1, "hdd", "node-b"), + } + executor, _ := deviceClassExecutor(`[{"id":0,"hostname":"node-a"},{"id":1,"hostname":"node-b"}]`, nil) + r := newClassAwareReconciler(t, executor, append(osds, cephCluster, cm, + getNodeObject("node-a", false), getNodeObject("node-b", false))...) + + // runClassAware asserts populateOSDFailureDomains and reconcilePDBsForOSDs return no error. + runClassAware(t, r, cm, layout) + + // the ssd group degraded to its plain per-class default (no blocking PDBs) + assert.Equal(t, []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd"}, pdbNames(t, r)) + // the ssd OSD with no rack label is still covered by exactly one PDB + assert.Equal(t, 1, countMatchingPDBs(t, r, osdPodLabels("ssd", "node-a", 0))) + // no ssd drain state was written + assert.Equal(t, "", cm.Data[dcDrainingKey("ssd")]) +} + +func TestReconcileRequeuesOnCrushReadErrorMidDrain(t *testing.T) { + // A transient CRUSH read error during an in-flight class-aware drain must requeue and + // leave the dc.* drain state and class-aware PDBs untouched, never flip to the global + // group (which would run GC/noout with an empty global group and wipe the dc.* state). + managedCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-cluster", Namespace: namespace}, + Spec: cephv1.ClusterSpec{ + DisruptionManagement: cephv1.DisruptionManagementSpec{ManagePodBudgets: true}, + }, + } + // a pool CR so the poolCount<1 gate does not early-return before the CRUSH read + pool := &cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: namespace}} + + // in-flight ssd drain state and the class-aware PDBs that go with it + cm := fakePDBConfigMap("") + cm.Data[dcDrainingKey("ssd")] = "node-a1" + cm.Data[dcSetNoOutKey("ssd")] = "true" + cm.Data[dcNooutTimestampKey("ssd", "node-a1")] = "2026-01-01T00:00:00Z" + preexisting := []runtime.Object{ + defaultOSDPDB(namespace, "rook-ceph-osd", nil, []string{"hdd", "ssd"}, nil), + defaultOSDPDB(namespace, "rook-ceph-osd-hdd", []string{"hdd"}, nil, nil), + blockingOSDPDB(namespace, "rook-ceph-osd-ssd-host-node-a2", "ssd", "topology-location-host", "node-a2"), + } + osds := []runtime.Object{ + fakeClassedOSD(0, 0, "ssd", "node-a1"), + fakeClassedOSD(1, 1, "ssd", "node-a2"), + fakeClassedOSD(2, 1, "hdd", "node-b"), + } + + // executor that fails the crush dump (the first ceph call GetDeviceClassPDBLayout makes) + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if args[0] == "osd" && args[1] == "crush" && args[2] == "dump" { + return "", errors.New("transient mon read error") + } + return "", errors.Errorf("unexpected ceph command '%v'", args) + } + + objs := append(osds, managedCluster, pool, cm, + getNodeObject("node-a1", true), getNodeObject("node-a2", false), getNodeObject("node-b", false)) + objs = append(objs, preexisting...) + r := getFakeReconciler(t, objs...) + r.clusterMap = &ClusterMap{clusterMap: map[string]*cephv1.CephCluster{namespace: managedCluster}} + r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}, OpManagerContext: context.TODO()} + + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} + result, err := r.reconcile(request) + assert.NoError(t, err) + assert.Equal(t, opcontroller.WaitForRequeueIfCephClusterNotReady, result) + + // dc.* drain state preserved (no GC churn) + gotCM := &corev1.ConfigMap{} + assert.NoError(t, r.client.Get(context.TODO(), types.NamespacedName{Name: pdbStateMapName, Namespace: namespace}, gotCM)) + assert.Equal(t, "node-a1", gotCM.Data[dcDrainingKey("ssd")]) + assert.Equal(t, "true", gotCM.Data[dcSetNoOutKey("ssd")]) + assert.Equal(t, "2026-01-01T00:00:00Z", gotCM.Data[dcNooutTimestampKey("ssd", "node-a1")]) + + // class-aware PDBs left in place (no prune) + assert.NotNil(t, getPDB(r, "rook-ceph-osd-ssd-host-node-a2")) + assert.NotNil(t, getPDB(r, "rook-ceph-osd-hdd")) + assert.NotNil(t, getPDB(r, "rook-ceph-osd")) +} + +func TestComputeDesiredPDBs(t *testing.T) { + // computeDesiredPDBs is pure over the groups' state and the drain-state ConfigMap, so + // the groups are built directly with their failure-domain state pre-filled. No ceph mocking. + ssd := func() *pdbGroup { + g := newPDBGroup("ssd", "host", []string{"ssd-pool"}) + g.state = groupDrainState{allFailureDomains: []string{"node-a", "node-b"}} + return g + } + hdd := func() *pdbGroup { + g := newPDBGroup("hdd", "host", []string{"hdd-pool"}) + g.state = groupDrainState{allFailureDomains: []string{"node-c", "node-d"}} + return g + } + global := func() *pdbGroup { + g := newPDBGroup("", "host", nil) + g.state = groupDrainState{allFailureDomains: []string{"node-a", "node-b"}} + return g + } + + tests := []struct { + name string + groups []*pdbGroup + drainState map[string]string + wantNames []string + }{ + { + name: "global idle yields the plain default", + groups: []*pdbGroup{global()}, + wantNames: []string{"rook-ceph-osd"}, + }, + { + name: "global draining yields a blocking pdb for each surviving failure domain", + groups: []*pdbGroup{global()}, + drainState: map[string]string{drainingFailureDomainKey: "node-a"}, + wantNames: []string{"rook-ceph-osd-host-node-b"}, + }, + { + // the catch-all is derived from the class groups, not passed in + name: "class-aware idle yields per-class defaults plus the catch-all", + groups: []*pdbGroup{ssd(), hdd()}, + wantNames: []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd"}, + }, + { + name: "one class draining keeps the other class default and the catch-all", + groups: []*pdbGroup{ssd(), hdd()}, + drainState: map[string]string{dcDrainingKey("ssd"): "node-a"}, + wantNames: []string{"rook-ceph-osd", "rook-ceph-osd-hdd", "rook-ceph-osd-ssd-host-node-b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm := &corev1.ConfigMap{Data: tt.drainState} + if cm.Data == nil { + cm.Data = map[string]string{} + } + desired := computeDesiredPDBs(namespace, tt.groups, cm) + got := make([]string, 0, len(desired)) + for name := range desired { + got = append(got, name) + } + sort.Strings(got) + assert.Equal(t, tt.wantNames, got) + }) + } +} diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go index fc339710ffeb..e47569ad1c10 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go @@ -115,6 +115,15 @@ func getFakeClusterInfo() *client.ClusterInfo { return sharedClusterMap.GetClusterInfo(namespace) } +// enumerateGlobalGroup runs populateOSDFailureDomains for a single cluster-wide group +// and returns its state as the baseline tuple, so the baseline enumeration tests can +// assert on the group-parameterized implementation unchanged. +func enumerateGlobalGroup(r *ReconcileClusterDisruption, clusterInfo *client.ClusterInfo, request reconcile.Request, failureDomainType string) ([]string, []string, []string, []int, error) { + g := &pdbGroup{failureDomainType: failureDomainType} + err := r.populateOSDFailureDomains(clusterInfo, request, []*pdbGroup{g}) + return g.state.allFailureDomains, g.state.nodeDrainFailureDomains, g.state.osdDownFailureDomains, g.state.downOSDs, err +} + func TestGetOSDFailureDomains(t *testing.T) { testcases := []struct { name string @@ -239,7 +248,7 @@ func TestGetOSDFailureDomains(t *testing.T) { clusterInfo.Context = context.TODO() r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}} request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} - allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, downOSDs, err := r.getOSDFailureDomains(clusterInfo, request, "zone") + allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, downOSDs, err := enumerateGlobalGroup(r, clusterInfo, request, "zone") assert.NoError(t, err) assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains) assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains) @@ -249,54 +258,35 @@ func TestGetOSDFailureDomains(t *testing.T) { } } -func TestGetOSDFailureDomainsError(t *testing.T) { - testcases := []struct { - name string - osds []appsv1.Deployment - expectedAllFailureDomains []string - expectedDrainingFailureDomains []string - expectedOsdDownFailureDomains []string - expectedDownOSDs []int - }{ - { - name: "case 1: one or more OSD deployment is missing crush location label", - osds: []appsv1.Deployment{ - fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1), - fakeOSDDeployment(3, 1), - }, - expectedAllFailureDomains: nil, - expectedDrainingFailureDomains: nil, - expectedOsdDownFailureDomains: nil, - expectedDownOSDs: nil, - }, +func TestGetOSDFailureDomainsDegradesOnMissingLabel(t *testing.T) { + // An OSD deployment missing the group's failure-domain label must degrade the + // group (default-only), not abort the whole reconcile. + osds := []appsv1.Deployment{ + fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1), } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - executor := &exectest.MockExecutor{} - executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "osd" && args[1] == "metadata" { - return `[{"id": 1, "hostname": "node-1"}, {"id": 2, "hostname": "node-2"}, {"id": 3, "hostname": "node-3"}]`, nil - } - return "", errors.Errorf("unexpected ceph command '%v'", args) - } - osd := tc.osds[0].DeepCopy() - osd.Labels["topology-location-zone"] = "" - r := getFakeReconciler(t, cephCluster, &corev1.ConfigMap{}, - tc.osds[1].DeepCopy(), tc.osds[2].DeepCopy(), osd) - r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}} - clusterInfo := getFakeClusterInfo() - clusterInfo.Context = context.TODO() - request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} - allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, downOSDs, err := r.getOSDFailureDomains(clusterInfo, request, "zone") - assert.Error(t, err) - assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains) - assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains) - assert.Equal(t, tc.expectedOsdDownFailureDomains, osdDownFailureDomains) - assert.Equal(t, tc.expectedDownOSDs, downOSDs) - }) + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if args[0] == "osd" && args[1] == "metadata" { + return `[{"id": 1, "hostname": "node-1"}, {"id": 2, "hostname": "node-2"}, {"id": 3, "hostname": "node-3"}]`, nil + } + return "", errors.Errorf("unexpected ceph command '%v'", args) } + osd := osds[0].DeepCopy() + osd.Labels["topology-location-zone"] = "" + r := getFakeReconciler(t, cephCluster, &corev1.ConfigMap{}, + osds[1].DeepCopy(), osds[2].DeepCopy(), osd) + r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor}} + clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} + + g := &pdbGroup{failureDomainType: "zone"} + err := r.populateOSDFailureDomains(clusterInfo, request, []*pdbGroup{g}) + assert.NoError(t, err) + assert.True(t, g.degraded) + assert.Empty(t, g.state.allFailureDomains) + assert.Empty(t, g.state.downOSDs) } func TestReconcilePDBForOSD(t *testing.T) { @@ -427,7 +417,15 @@ func TestReconcilePDBForOSD(t *testing.T) { test.SetFakeKubernetesVersion(clientset, "v1.21.0") r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor, Clientset: clientset}} - _, err := r.reconcilePDBsForOSDs(clusterInfo, request, tc.configMap, "zone", tc.allFailureDomains, tc.osdDownFailureDomains, tc.activeNodeDrains, tc.downOSDs, tc.pgHealthyRegex) + // single global group reproduces the baseline fallback behavior + g := newPDBGroup("", "zone", nil) + g.state = groupDrainState{ + allFailureDomains: tc.allFailureDomains, + osdDownFailureDomains: tc.osdDownFailureDomains, + nodeDrainFailureDomains: tc.activeNodeDrains, + downOSDs: tc.downOSDs, + } + _, err := r.reconcilePDBsForOSDs(clusterInfo, request, tc.configMap, []*pdbGroup{g}, tc.pgHealthyRegex) assert.NoError(t, err) // assert that pdb for osd are created correctly @@ -489,6 +487,50 @@ func TestReconcilePDBForOSD(t *testing.T) { } } +func TestInitializePDBStateNilData(t *testing.T) { + // A ConfigMap whose drain-state keys were all deleted is stored with no data + // field and reloads with a nil Data map. initializePDBState must return a + // writable (non-nil) map so the subsequent setPDBConfig writes do not panic. + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: pdbStateMapName, Namespace: namespace}, + // Data intentionally nil, as reloaded from the apiserver after all keys were deleted + } + r := getFakeReconciler(t, cephCluster, cm) + r.context = &controllerconfig.Context{OpManagerContext: context.TODO()} + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} + + pdbStateMap, err := r.initializePDBState(request) + assert.NoError(t, err) + assert.NotNil(t, pdbStateMap.Data) + // writing drain state must not panic on the returned map + setPDBConfig(pdbStateMap, globalDrainKeys, []string{"zone-1"}, nil) + assert.Equal(t, "zone-1", pdbStateMap.Data[drainingFailureDomainKey]) +} + +func TestRequeuePDBController(t *testing.T) { + r := &ReconcileClusterDisruption{} + + // idle: no drain, no down OSD -> success, no requeue + idle := &pdbGroup{} + assert.Zero(t, r.requeuePDBController(&corev1.ConfigMap{Data: map[string]string{}}, []*pdbGroup{idle}).RequeueAfter) + + // down OSD but not draining (e.g. down+clean, excluded) -> requeue so the + // exclusion is removed when the OSD recovers (no deployment watch event fires) + down := &pdbGroup{state: groupDrainState{downOSDs: []int{3}}} + assert.NotZero(t, r.requeuePDBController(&corev1.ConfigMap{Data: map[string]string{}}, []*pdbGroup{down}).RequeueAfter) + + // actively draining -> requeue + draining := &pdbGroup{} + cm := &corev1.ConfigMap{Data: map[string]string{drainingFailureDomainKey: "zone-1"}} + assert.NotZero(t, r.requeuePDBController(cm, []*pdbGroup{draining}).RequeueAfter) + + // one class draining requeues even if another class is idle + idleClass := &pdbGroup{deviceClass: "ssd"} + drainingClass := &pdbGroup{deviceClass: "hdd"} + cm2 := &corev1.ConfigMap{Data: map[string]string{dcDrainingKey("hdd"): "node-1"}} + assert.NotZero(t, r.requeuePDBController(cm2, []*pdbGroup{idleClass, drainingClass}).RequeueAfter) +} + func TestHasNodeDrained(t *testing.T) { osdDeployment := fakeOSDDeployment(1, 1) ctx := context.TODO() @@ -575,7 +617,7 @@ func TestSetPDBConfig(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - setPDBConfig(tc.pdbConfig, tc.osdDownFailureDomains, tc.drainingFailureDomains) + setPDBConfig(tc.pdbConfig, globalDrainKeys, tc.osdDownFailureDomains, tc.drainingFailureDomains) assert.Equal(t, tc.expectedFailureDomainKeyValue, tc.pdbConfig.Data[drainingFailureDomainKey]) assert.Equal(t, tc.expecteNoOutSetting, tc.pdbConfig.Data[setNoOut]) }) diff --git a/pkg/operator/ceph/disruption/clusterdisruption/pools.go b/pkg/operator/ceph/disruption/clusterdisruption/pools.go index 7933ed602b48..5fb733b64850 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/pools.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/pools.go @@ -20,7 +20,6 @@ import ( "fmt" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology" "github.com/pkg/errors" policyv1 "k8s.io/api/policy/v1" @@ -31,81 +30,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func (r *ReconcileClusterDisruption) processPools(request reconcile.Request) (*cephv1.CephObjectStoreList, *cephv1.CephFilesystemList, string, int, error) { - namespaceListOpt := client.InNamespace(request.Namespace) - poolSpecs := make([]cephv1.PoolSpec, 0) - poolCount := 0 +// cephBlockPoolsExist reports whether any CephBlockPool CR exists in the namespace. +func (r *ReconcileClusterDisruption) cephBlockPoolsExist(request reconcile.Request) (bool, error) { cephBlockPoolList := &cephv1.CephBlockPoolList{} - err := r.client.List(r.context.OpManagerContext, cephBlockPoolList, namespaceListOpt) - if err != nil { - return nil, nil, "", poolCount, errors.Wrapf(err, "could not list the CephBlockpools %v", request.NamespacedName) + if err := r.client.List(r.context.OpManagerContext, cephBlockPoolList, client.InNamespace(request.Namespace)); err != nil { + return false, errors.Wrapf(err, "could not list the CephBlockPools %v", request.NamespacedName) } - poolCount += len(cephBlockPoolList.Items) - for _, cephBlockPool := range cephBlockPoolList.Items { - poolSpecs = append(poolSpecs, cephBlockPool.Spec.PoolSpec) - } - - cephFilesystemList := &cephv1.CephFilesystemList{} - err = r.client.List(r.context.OpManagerContext, cephFilesystemList, namespaceListOpt) - if err != nil { - return nil, nil, "", poolCount, errors.Wrapf(err, "could not list the CephFilesystems %v", request.NamespacedName) - } - poolCount += len(cephFilesystemList.Items) - for _, cephFilesystem := range cephFilesystemList.Items { - poolSpecs = append(poolSpecs, cephFilesystem.Spec.MetadataPool.PoolSpec) - for _, pool := range cephFilesystem.Spec.DataPools { - poolSpecs = append(poolSpecs, pool.PoolSpec) - } - - } - - cephObjectStoreList := &cephv1.CephObjectStoreList{} - err = r.client.List(r.context.OpManagerContext, cephObjectStoreList, namespaceListOpt) - if err != nil { - return nil, nil, "", poolCount, errors.Wrapf(err, "could not list the CephObjectStores %v", request.NamespacedName) - } - poolCount += len(cephObjectStoreList.Items) - for _, cephObjectStore := range cephObjectStoreList.Items { - poolSpecs = append(poolSpecs, cephObjectStore.Spec.MetadataPool) - poolSpecs = append(poolSpecs, cephObjectStore.Spec.DataPool) - - } - minFailureDomain := getMinimumFailureDomain(poolSpecs) - - return cephObjectStoreList, cephFilesystemList, minFailureDomain, poolCount, nil + return len(cephBlockPoolList.Items) > 0, nil } -func getMinimumFailureDomain(poolList []cephv1.PoolSpec) string { - if len(poolList) == 0 { - return cephv1.DefaultFailureDomain - } - - // start with max as the min - minfailureDomainIndex := len(topology.CRUSHMapLevelsOrdered) - 1 - matched := false - - for _, pool := range poolList { - for index, failureDomain := range topology.CRUSHMapLevelsOrdered { - if index == minfailureDomainIndex { - // index is higher-than/equal-to the min - break - } - if pool.FailureDomain == failureDomain { - // new min found - matched = true - minfailureDomainIndex = index - } - } - } - if !matched { - logger.Debugf("could not match failure domain. defaulting to %q", cephv1.DefaultFailureDomain) - return cephv1.DefaultFailureDomain +// reconcileCephObjectStore reconciles the RGW PDB (naive minAvailable n-1) for every +// CephObjectStore in the namespace and reports whether any exist. +func (r *ReconcileClusterDisruption) reconcileCephObjectStore(request reconcile.Request) (bool, error) { + cephObjectStoreList := &cephv1.CephObjectStoreList{} + if err := r.client.List(r.context.OpManagerContext, cephObjectStoreList, client.InNamespace(request.Namespace)); err != nil { + return false, errors.Wrapf(err, "could not list the CephObjectStores %v", request.NamespacedName) } - return topology.CRUSHMapLevelsOrdered[minfailureDomainIndex] -} - -// Setting naive minAvailable for RGW at: n - 1 -func (r *ReconcileClusterDisruption) reconcileCephObjectStore(cephObjectStoreList *cephv1.CephObjectStoreList) error { for _, objectStore := range cephObjectStoreList.Items { storeName := objectStore.ObjectMeta.Name namespace := objectStore.ObjectMeta.Namespace @@ -140,18 +80,22 @@ func (r *ReconcileClusterDisruption) reconcileCephObjectStore(cephObjectStoreLis MinAvailable: minAvailable, }, } - request := types.NamespacedName{Name: pdbName, Namespace: namespace} - err := r.reconcileStaticPDB(request, pdb) - if err != nil { - return errors.Wrapf(err, "failed to reconcile cephobjectstore pdb %v", request) + pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace} + if err := r.reconcileStaticPDB(pdbRequest, pdb); err != nil { + return false, errors.Wrapf(err, "failed to reconcile cephobjectstore pdb %v", pdbRequest) } } - return nil + return len(cephObjectStoreList.Items) > 0, nil } -// Setting naive minAvailable for MDS at: n -1 -// getting n from the cephfilesystem.spec.metadataserver.activecount -func (r *ReconcileClusterDisruption) reconcileCephFilesystem(cephFilesystemList *cephv1.CephFilesystemList) error { +// reconcileCephFilesystem reconciles the MDS PDB (naive minAvailable n-1, from +// spec.metadataServer.activeCount) for every CephFilesystem in the namespace and +// reports whether any exist. +func (r *ReconcileClusterDisruption) reconcileCephFilesystem(request reconcile.Request) (bool, error) { + cephFilesystemList := &cephv1.CephFilesystemList{} + if err := r.client.List(r.context.OpManagerContext, cephFilesystemList, client.InNamespace(request.Namespace)); err != nil { + return false, errors.Wrapf(err, "could not list the CephFilesystems %v", request.NamespacedName) + } for _, filesystem := range cephFilesystemList.Items { fsName := filesystem.ObjectMeta.Name namespace := filesystem.ObjectMeta.Namespace @@ -189,11 +133,10 @@ func (r *ReconcileClusterDisruption) reconcileCephFilesystem(cephFilesystemList MinAvailable: minAvailable, }, } - request := types.NamespacedName{Name: pdbName, Namespace: namespace} - err := r.reconcileStaticPDB(request, pdb) - if err != nil { - return errors.Wrapf(err, "failed to reconcile cephfs pdb %v", request) + pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace} + if err := r.reconcileStaticPDB(pdbRequest, pdb); err != nil { + return false, errors.Wrapf(err, "failed to reconcile cephfs pdb %v", pdbRequest) } } - return nil + return len(cephFilesystemList.Items) > 0, nil } diff --git a/pkg/operator/ceph/disruption/clusterdisruption/pools_test.go b/pkg/operator/ceph/disruption/clusterdisruption/pools_test.go index 3417d30a7c1d..847041f8b7cc 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/pools_test.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/pools_test.go @@ -21,31 +21,21 @@ import ( "github.com/stretchr/testify/assert" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" ) -func TestGetMinimumFailureDomain(t *testing.T) { - poolList := []cephv1.PoolSpec{ - {FailureDomain: "region"}, - {FailureDomain: "zone"}, - } +func TestGlobalFailureDomain(t *testing.T) { + // the finest (lowest in the CRUSH hierarchy) of the in-use pools' failure domains + layout := &cephclient.DeviceClassPDBLayout{FailureDomainTypes: []string{"region", "zone"}} + assert.Equal(t, "zone", globalFailureDomain(layout)) - assert.Equal(t, "zone", getMinimumFailureDomain(poolList)) + layout = &cephclient.DeviceClassPDBLayout{FailureDomainTypes: []string{"region", "zone", "host"}} + assert.Equal(t, "host", globalFailureDomain(layout)) - poolList = []cephv1.PoolSpec{ - {FailureDomain: "region"}, - {FailureDomain: "zone"}, - {FailureDomain: "host"}, - } + // unknown levels and an empty list fall back to the default failure domain + layout = &cephclient.DeviceClassPDBLayout{FailureDomainTypes: []string{"aaa", "bbb", "ccc"}} + assert.Equal(t, "host", globalFailureDomain(layout)) - assert.Equal(t, "host", getMinimumFailureDomain(poolList)) - - // test default - poolList = []cephv1.PoolSpec{ - {FailureDomain: "aaa"}, - {FailureDomain: "bbb"}, - {FailureDomain: "ccc"}, - } - - assert.Equal(t, "host", getMinimumFailureDomain(poolList)) + layout = &cephclient.DeviceClassPDBLayout{} + assert.Equal(t, "host", globalFailureDomain(layout)) } diff --git a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go index 3027e90f1555..912e838bafaf 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go @@ -136,43 +136,48 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon logger.Debugf("Using default maintenance timeout: %v", r.maintenanceTimeout) } - // reconcile the pools and get the failure domain - cephObjectStoreList, cephFilesystemList, poolFailureDomain, poolCount, err := r.processPools(request) + // reconcile the RGW and MDS PDBs; each lists its own CRs and reports whether any exist + objectStorePoolsExist, err := r.reconcileCephObjectStore(request) if err != nil { return reconcile.Result{}, err } - - // reconcile the pdbs for objectstores - err = r.reconcileCephObjectStore(cephObjectStoreList) + filesystemPoolsExist, err := r.reconcileCephFilesystem(request) if err != nil { return reconcile.Result{}, err } - - // reconcile the pdbs for filesystems - err = r.reconcileCephFilesystem(cephFilesystemList) + blockPoolsExist, err := r.cephBlockPoolsExist(request) if err != nil { return reconcile.Result{}, err } - // no pools, no need to reconcile OSD PDB - if poolCount < 1 { + // No pool-bearing CRs, no need to reconcile OSD PDBs. NOTE (pre-existing gap): this + // sees only pool CRs, so a cluster whose pools were all created directly via the + // toolbox (no CR) gets no OSD PDB protection. Closing it means deriving pool existence + // from the CRUSH/pool-list read instead; out of scope here. + if !objectStorePoolsExist && !filesystemPoolsExist && !blockPoolsExist { return reconcile.Result{}, nil } - // get a list of all the failure domains, failure domains with failed OSDs and failure domains with drained nodes - allFailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, downOSDs, err := r.getOSDFailureDomains(clusterInfo, request, poolFailureDomain) + pgHealthyRegex := cephCluster.Spec.DisruptionManagement.PGHealthyRegex + + // Build the OSD PDB groups from the live CRUSH map, each with its failure-domain state + // (one group per device class when the cluster qualifies, else a single cluster-wide group). + groups, err := r.buildOSDPDBGroups(clusterInfo, request) if err != nil { - return reconcile.Result{}, err + // Any read here is a transient "ceph not ready" condition. Requeue and leave the + // existing PDBs and drain state untouched; proceeding with a different group set + // mid-reconcile would churn an in-flight drain's state. + logger.Debugf("failed to build OSD PDB groups, requeuing: %v", err) + return opcontroller.WaitForRequeueIfCephClusterNotReady, nil } - // get the map that stores currently draining failure domain + // initialize the map that tracks each group's draining failure domain pdbStateMap, err := r.initializePDBState(request) if err != nil { return reconcile.Result{}, err } - pgHealthyRegex := cephCluster.Spec.DisruptionManagement.PGHealthyRegex - return r.reconcilePDBsForOSDs(clusterInfo, request, pdbStateMap, poolFailureDomain, allFailureDomains, osdDownFailureDomains, nodeDrainFailureDomains, downOSDs, pgHealthyRegex) + return r.reconcilePDBsForOSDs(clusterInfo, request, pdbStateMap, groups, pgHealthyRegex) } // ClusterMap maintains the association between namespace and clusername