diff --git a/.github/actions/hf-temp-repo/cleanup.js b/.github/actions/hf-temp-repo/cleanup.js index 48073d5a7a42..2dfca475ed18 100644 --- a/.github/actions/hf-temp-repo/cleanup.js +++ b/.github/actions/hf-temp-repo/cleanup.js @@ -32,9 +32,11 @@ async function run() { if (repoType === "bucket") { await hfRequest("DELETE", `/api/buckets/${repoId}`, token); } else { + const [organization, name] = repoId.split("/"); await hfRequest("DELETE", "/api/repos/delete", token, { type: repoType, - name: repoId, + name, + organization, }); } diff --git a/.github/actions/hf-temp-repo/cleanup.test.js b/.github/actions/hf-temp-repo/cleanup.test.js new file mode 100644 index 000000000000..174d7d78cccb --- /dev/null +++ b/.github/actions/hf-temp-repo/cleanup.test.js @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const assert = require("node:assert/strict"); +const { readFileSync } = require("node:fs"); +const test = require("node:test"); +const vm = require("node:vm"); + +const source = readFileSync(require.resolve("./cleanup.js"), "utf8"); + +async function runCleanup(state) { + const requests = []; + await vm.runInNewContext(source, { + require(name) { + assert.equal(name, "./common"); + return { + getState: (key) => state[key] || "", + async hfRequest(method, path, token, body) { + requests.push({ method, path, token, body: body && { ...body } }); + }, + }; + }, + console: { + log() {}, + warn(message) { + throw new Error(message); + }, + }, + }); + return requests; +} + +test("dataset cleanup sends the namespace and bare repository name", async () => { + const requests = await runCleanup({ + repo_id: "opendal/test-dataset-123-abcd", + repo_type: "dataset", + token: "test-token", + }); + assert.deepEqual(requests, [{ + method: "DELETE", + path: "/api/repos/delete", + token: "test-token", + body: { type: "dataset", name: "test-dataset-123-abcd", organization: "opendal" }, + }]); +}); + +test("bucket cleanup uses the bucket endpoint", async () => { + const requests = await runCleanup({ + repo_id: "opendal/test-bucket-123-abcd", + repo_type: "bucket", + token: "test-token", + }); + assert.deepEqual(requests, [{ + method: "DELETE", + path: "/api/buckets/opendal/test-bucket-123-abcd", + token: "test-token", + body: undefined, + }]); +}); + +test("cleanup skips deletion when creation saved no repository", async () => { + assert.deepEqual(await runCleanup({}), []); +}); diff --git a/.github/actions/test_behavior_core/action.yaml b/.github/actions/test_behavior_core/action.yaml index a26cfb3b46bf..d509cdebd3ee 100644 --- a/.github/actions/test_behavior_core/action.yaml +++ b/.github/actions/test_behavior_core/action.yaml @@ -102,7 +102,7 @@ runs: service_prefix=\$(printf '%s' "$TEST_SERVICE" | tr '[:lower:]-' '[:upper:]_') while IFS='=' read -r name _; do case "\$name" in - "OPENDAL_\${service_prefix}_"* | OPENDAL_DISABLE_RANDOM_ROOT | OPENDAL_TEST_CAPABILITY_OVERRIDES) + "OPENDAL_\${service_prefix}_"* | OPENDAL_DISABLE_RANDOM_ROOT | OPENDAL_TEST_CAPABILITY_OVERRIDES | OPENDAL_TEST_IO_TIMEOUT) echo "\$name=__OPENDAL_TEST_UNSET__" >>"\$GITHUB_ENV" ;; esac diff --git a/.github/services/hf/hf_dataset/action.yml b/.github/services/hf/hf_dataset/action.yml index bc40e13f06d1..5eef60da61ce 100644 --- a/.github/services/hf/hf_dataset/action.yml +++ b/.github/services/hf/hf_dataset/action.yml @@ -40,5 +40,6 @@ runs: cat << EOF >> $GITHUB_ENV OPENDAL_HF_REPO_TYPE=dataset OPENDAL_HF_FORCE_RESOLVE=true + OPENDAL_TEST_IO_TIMEOUT=30 RUST_TEST_THREADS=1 EOF diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 4e4f1eea724c..079313d38afa 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -33,6 +33,17 @@ env: RUST_BACKTRACE: 1 jobs: + hf-temp-repo: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - name: Test temporary HF repository cleanup + run: node --test .github/actions/hf-temp-repo/cleanup.test.js + labels: runs-on: ubuntu-slim timeout-minutes: 5 diff --git a/core/testkit/README.md b/core/testkit/README.md index d43e7786bdca..067257b47098 100644 --- a/core/testkit/README.md +++ b/core/testkit/README.md @@ -20,6 +20,12 @@ The `opendal` facade also re-exports the testkit behind its `tests` feature. - [Behavior test guide](https://github.com/apache/opendal/tree/main/core/tests/behavior) - [Apache OpenDAL documentation](https://opendal.apache.org/docs/) +## Test operator configuration + +`OPENDAL_TEST_IO_TIMEOUT` sets the behavior-test I/O timeout in positive whole seconds. +When unset, the test operator uses the default from `TimeoutLayer`. Service +fixtures can set this variable when their operations need a larger I/O budget. + ## License Licensed under the Apache License, Version 2.0. diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index c1ce9b2fa05c..8d6a8b335940 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -17,8 +17,12 @@ use std::collections::HashMap; use std::env; +use std::num::NonZeroU64; use std::sync::LazyLock; +use std::time::Duration; +use opendal_core::Error; +use opendal_core::ErrorKind; use opendal_core::Operator; use opendal_core::Result; use opendal_core::layers::CapabilityOverrideLayer; @@ -108,9 +112,23 @@ pub fn init_test_service() -> Result> { op = op.layer(CapabilityOverrideLayer::from_overrides(&overrides)?); } + let mut timeout = TimeoutLayer::new(); + if let Ok(value) = env::var("OPENDAL_TEST_IO_TIMEOUT") + && value != OPENDAL_TEST_UNSET_VALUE + { + let seconds = value.parse::().map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + "OPENDAL_TEST_IO_TIMEOUT must be a positive whole number of seconds", + ) + .set_source(err) + })?; + timeout = timeout.with_io_timeout(Duration::from_secs(seconds.get())); + } + let op = op .layer(LoggingLayer::default()) - .layer(TimeoutLayer::new()) + .layer(timeout) .layer(RetryLayer::new().with_max_times(4)); Ok(Some(op))