From a878327ddf7016a90830698d53d550301c46edb6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 31 Oct 2026 06:51:49 +0800 Subject: [PATCH 1/2] fix(ci): configure HF dataset I/O deadlines and clean up temporary repos --- .github/actions/hf-temp-repo/cleanup.js | 4 +- .github/actions/hf-temp-repo/cleanup.test.js | 79 +++++++++++++++++++ .../actions/test_behavior_core/action.yaml | 2 +- .github/services/hf/hf_dataset/action.yml | 1 + .github/workflows/ci_check.yml | 11 +++ core/testkit/README.md | 6 ++ core/testkit/src/utils.rs | 19 ++++- 7 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 .github/actions/hf-temp-repo/cleanup.test.js diff --git a/.github/actions/hf-temp-repo/cleanup.js b/.github/actions/hf-temp-repo/cleanup.js index 48073d5a7a42..2dfca475ed18 100644 --- a/.github/actions/hf-temp-repo/cleanup.js +++ b/.github/actions/hf-temp-repo/cleanup.js @@ -32,9 +32,11 @@ async function run() { if (repoType === "bucket") { await hfRequest("DELETE", `/api/buckets/${repoId}`, token); } else { + const [organization, name] = repoId.split("/"); await hfRequest("DELETE", "/api/repos/delete", token, { type: repoType, - name: repoId, + name, + organization, }); } diff --git a/.github/actions/hf-temp-repo/cleanup.test.js b/.github/actions/hf-temp-repo/cleanup.test.js new file mode 100644 index 000000000000..174d7d78cccb --- /dev/null +++ b/.github/actions/hf-temp-repo/cleanup.test.js @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const assert = require("node:assert/strict"); +const { readFileSync } = require("node:fs"); +const test = require("node:test"); +const vm = require("node:vm"); + +const source = readFileSync(require.resolve("./cleanup.js"), "utf8"); + +async function runCleanup(state) { + const requests = []; + await vm.runInNewContext(source, { + require(name) { + assert.equal(name, "./common"); + return { + getState: (key) => state[key] || "", + async hfRequest(method, path, token, body) { + requests.push({ method, path, token, body: body && { ...body } }); + }, + }; + }, + console: { + log() {}, + warn(message) { + throw new Error(message); + }, + }, + }); + return requests; +} + +test("dataset cleanup sends the namespace and bare repository name", async () => { + const requests = await runCleanup({ + repo_id: "opendal/test-dataset-123-abcd", + repo_type: "dataset", + token: "test-token", + }); + assert.deepEqual(requests, [{ + method: "DELETE", + path: "/api/repos/delete", + token: "test-token", + body: { type: "dataset", name: "test-dataset-123-abcd", organization: "opendal" }, + }]); +}); + +test("bucket cleanup uses the bucket endpoint", async () => { + const requests = await runCleanup({ + repo_id: "opendal/test-bucket-123-abcd", + repo_type: "bucket", + token: "test-token", + }); + assert.deepEqual(requests, [{ + method: "DELETE", + path: "/api/buckets/opendal/test-bucket-123-abcd", + token: "test-token", + body: undefined, + }]); +}); + +test("cleanup skips deletion when creation saved no repository", async () => { + assert.deepEqual(await runCleanup({}), []); +}); diff --git a/.github/actions/test_behavior_core/action.yaml b/.github/actions/test_behavior_core/action.yaml index a26cfb3b46bf..d509cdebd3ee 100644 --- a/.github/actions/test_behavior_core/action.yaml +++ b/.github/actions/test_behavior_core/action.yaml @@ -102,7 +102,7 @@ runs: service_prefix=\$(printf '%s' "$TEST_SERVICE" | tr '[:lower:]-' '[:upper:]_') while IFS='=' read -r name _; do case "\$name" in - "OPENDAL_\${service_prefix}_"* | OPENDAL_DISABLE_RANDOM_ROOT | OPENDAL_TEST_CAPABILITY_OVERRIDES) + "OPENDAL_\${service_prefix}_"* | OPENDAL_DISABLE_RANDOM_ROOT | OPENDAL_TEST_CAPABILITY_OVERRIDES | OPENDAL_TEST_IO_TIMEOUT) echo "\$name=__OPENDAL_TEST_UNSET__" >>"\$GITHUB_ENV" ;; esac diff --git a/.github/services/hf/hf_dataset/action.yml b/.github/services/hf/hf_dataset/action.yml index bc40e13f06d1..5eef60da61ce 100644 --- a/.github/services/hf/hf_dataset/action.yml +++ b/.github/services/hf/hf_dataset/action.yml @@ -40,5 +40,6 @@ runs: cat << EOF >> $GITHUB_ENV OPENDAL_HF_REPO_TYPE=dataset OPENDAL_HF_FORCE_RESOLVE=true + OPENDAL_TEST_IO_TIMEOUT=30 RUST_TEST_THREADS=1 EOF diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 4e4f1eea724c..079313d38afa 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -33,6 +33,17 @@ env: RUST_BACKTRACE: 1 jobs: + hf-temp-repo: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - name: Test temporary HF repository cleanup + run: node --test .github/actions/hf-temp-repo/cleanup.test.js + labels: runs-on: ubuntu-slim timeout-minutes: 5 diff --git a/core/testkit/README.md b/core/testkit/README.md index d43e7786bdca..81745cb799cb 100644 --- a/core/testkit/README.md +++ b/core/testkit/README.md @@ -20,6 +20,12 @@ The `opendal` facade also re-exports the testkit behind its `tests` feature. - [Behavior test guide](https://github.com/apache/opendal/tree/main/core/tests/behavior) - [Apache OpenDAL documentation](https://opendal.apache.org/docs/) +## Test operator configuration + +`OPENDAL_TEST_IO_TIMEOUT` sets the behavior-test I/O timeout in whole seconds. +When unset, the test operator uses the default from `TimeoutLayer`. Service +fixtures can set this variable when their operations need a larger I/O budget. + ## License Licensed under the Apache License, Version 2.0. diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index c1ce9b2fa05c..e7016beff8fa 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -18,7 +18,10 @@ use std::collections::HashMap; use std::env; use std::sync::LazyLock; +use std::time::Duration; +use opendal_core::Error; +use opendal_core::ErrorKind; use opendal_core::Operator; use opendal_core::Result; use opendal_core::layers::CapabilityOverrideLayer; @@ -108,9 +111,23 @@ pub fn init_test_service() -> Result> { op = op.layer(CapabilityOverrideLayer::from_overrides(&overrides)?); } + let mut timeout = TimeoutLayer::new(); + if let Ok(value) = env::var("OPENDAL_TEST_IO_TIMEOUT") + && value != OPENDAL_TEST_UNSET_VALUE + { + let seconds = value.parse::().map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + "OPENDAL_TEST_IO_TIMEOUT must be a whole number of seconds", + ) + .set_source(err) + })?; + timeout = timeout.with_io_timeout(Duration::from_secs(seconds)); + } + let op = op .layer(LoggingLayer::default()) - .layer(TimeoutLayer::new()) + .layer(timeout) .layer(RetryLayer::new().with_max_times(4)); Ok(Some(op)) From 4c5d89b76611b09af4c1f41bf349dccccec8ee22 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 31 Oct 2026 07:05:04 +0800 Subject: [PATCH 2/2] fix(ci): reject a zero behavior test I/O budget --- core/testkit/README.md | 2 +- core/testkit/src/utils.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/testkit/README.md b/core/testkit/README.md index 81745cb799cb..067257b47098 100644 --- a/core/testkit/README.md +++ b/core/testkit/README.md @@ -22,7 +22,7 @@ The `opendal` facade also re-exports the testkit behind its `tests` feature. ## Test operator configuration -`OPENDAL_TEST_IO_TIMEOUT` sets the behavior-test I/O timeout in whole seconds. +`OPENDAL_TEST_IO_TIMEOUT` sets the behavior-test I/O timeout in positive whole seconds. When unset, the test operator uses the default from `TimeoutLayer`. Service fixtures can set this variable when their operations need a larger I/O budget. diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index e7016beff8fa..8d6a8b335940 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::env; +use std::num::NonZeroU64; use std::sync::LazyLock; use std::time::Duration; @@ -115,14 +116,14 @@ pub fn init_test_service() -> Result> { if let Ok(value) = env::var("OPENDAL_TEST_IO_TIMEOUT") && value != OPENDAL_TEST_UNSET_VALUE { - let seconds = value.parse::().map_err(|err| { + let seconds = value.parse::().map_err(|err| { Error::new( ErrorKind::ConfigInvalid, - "OPENDAL_TEST_IO_TIMEOUT must be a whole number of seconds", + "OPENDAL_TEST_IO_TIMEOUT must be a positive whole number of seconds", ) .set_source(err) })?; - timeout = timeout.with_io_timeout(Duration::from_secs(seconds)); + timeout = timeout.with_io_timeout(Duration::from_secs(seconds.get())); } let op = op