From 4af4f4f81c566b6a7650c0c67c19328c1820c5b2 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Fri, 25 Sep 2026 10:21:22 +0200 Subject: [PATCH 1/2] feat(services/hf): add disable_config_load Like s3's option of the same name, it stops the service from reading a token from the environment, so a server embedding OpenDAL does not send its users' requests with its own Hugging Face token. --- .../ServiceConfig/HuggingfaceServiceConfig.cs | 8 +++ .../org/apache/opendal/ServiceConfig.java | 9 +++ bindings/python/python/opendal/config.py | 2 + core/services/hf/src/backend.rs | 62 ++++++++++++++++++- core/services/hf/src/config.rs | 38 ++++++++++++ website/data/services.json | 17 +++-- 6 files changed, 130 insertions(+), 6 deletions(-) diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs index 41f03ec1a012..bb71a5e87738 100644 --- a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs +++ b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs @@ -28,6 +28,10 @@ namespace OpenDAL.ServiceConfig /// public sealed class HuggingfaceServiceConfig : IServiceConfig { + /// + /// Disable loading the token from the environment: `HF_TOKEN`, `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`. + /// + public bool? DisableConfigLoad { get; init; } /// /// Download mode. Either `xet` (default) or `http`. When unset, the mode is resolved from the `HF_HUB_DISABLE_XET` environment variable: a non-empty value forces `http`, otherwise it defaults to `xet`. An explicit value here takes precedence. See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>. /// @@ -66,6 +70,10 @@ public sealed class HuggingfaceServiceConfig : IServiceConfig public IReadOnlyDictionary ToOptions() { var map = new Dictionary(); + if (DisableConfigLoad is not null) + { + map["disable_config_load"] = Utilities.ToOptionString(DisableConfigLoad); + } if (DownloadMode is not null) { map["download_mode"] = Utilities.ToOptionString(DownloadMode); diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java index e915e1112625..93892d1e8a29 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java +++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java @@ -1687,6 +1687,12 @@ public Map configMap() { @Data @RequiredArgsConstructor(access = AccessLevel.PRIVATE) class Hf implements ServiceConfig { + /** + *

Disable loading the token from the environment: HF_TOKEN, + * HF_TOKEN_PATH, or the token file under HF_HOME, + * $XDG_CACHE_HOME/huggingface or ~/.cache/huggingface.

+ */ + public final Boolean disableConfigLoad; /** *

Download mode. Either xet (default) or http.

*

When unset, the mode is resolved from the HF_HUB_DISABLE_XET @@ -1743,6 +1749,9 @@ public String scheme() { @Override public Map configMap() { final HashMap map = new HashMap<>(); + if (disableConfigLoad != null) { + map.put("disable_config_load", String.valueOf(disableConfigLoad)); + } if (downloadMode != null) { map.put("download_mode", downloadMode); } diff --git a/bindings/python/python/opendal/config.py b/bindings/python/python/opendal/config.py index bfe1587c87e1..d19ee351855d 100644 --- a/bindings/python/python/opendal/config.py +++ b/bindings/python/python/opendal/config.py @@ -408,6 +408,8 @@ class HfConfig(TypedDict): scheme: Required[Literal["hf"]] """The service scheme; fixed to `"hf"`.""" + disable_config_load: NotRequired[bool] + """Disable loading the token from the environment: `HF_TOKEN`, `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.""" download_mode: NotRequired[str] """Download mode. Either `xet` (default) or `http`. When unset, the mode is resolved from the `HF_HUB_DISABLE_XET` environment variable: a non-empty value forces `http`, otherwise it defaults to `xet`. An explicit value here takes precedence. See .""" enable_resolve_cache: NotRequired[bool] diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs index a0f5638a773d..4542806faa73 100644 --- a/core/services/hf/src/backend.rs +++ b/core/services/hf/src/backend.rs @@ -167,6 +167,14 @@ impl HfBuilder { self } + /// Disable loading the token from the environment. + /// + /// See [`HfConfig::disable_config_load`] for details. + pub fn disable_config_load(mut self) -> Self { + self.config.disable_config_load = true; + self + } + /// Resolve the Hub base URL: an explicit config value wins, then /// `HF_ENDPOINT`, then the public Hub. A trailing slash is trimmed /// because every URL is built by appending `/api/...` to this, and HF @@ -207,11 +215,15 @@ impl HfBuilder { } /// Resolve the authentication token using the same priority order as hf-hub: - /// explicit config → HF_HUB_DISABLE_IMPLICIT_TOKEN check → HF_TOKEN env → token file. + /// explicit config → disable_config_load → HF_HUB_DISABLE_IMPLICIT_TOKEN check → + /// HF_TOKEN env → token file. fn hf_token(&self) -> Option { if let Some(t) = self.config.token.clone() { return Some(t); } + if self.config.disable_config_load { + return None; + } if let Ok(val) = std::env::var("HF_HUB_DISABLE_IMPLICIT_TOKEN") && !val.is_empty() { @@ -558,6 +570,54 @@ mod tests { assert_eq!(result, None); } + #[test] + fn hf_token_disable_config_load_ignores_env() { + let _guard = ENV_LOCK.lock().unwrap(); + let dir = std::env::temp_dir().join("opendal-hf-disable-config-load"); + std::fs::create_dir_all(dir.join("huggingface")).unwrap(); + std::fs::write(dir.join("token"), "file-token").unwrap(); + std::fs::write(dir.join("huggingface/token"), "xdg-token").unwrap(); + + // Each source alone yields a token, unless config loading is disabled. + let keys = ["HF_TOKEN", "HF_TOKEN_PATH", "HF_HOME", "XDG_CACHE_HOME"]; + let values = [ + std::ffi::OsString::from("env-token"), + dir.join("token").into(), + dir.clone().into(), + dir.clone().into(), + ]; + let saved = keys.map(std::env::var_os); + unsafe { std::env::remove_var("HF_HUB_DISABLE_IMPLICIT_TOKEN") }; + let mut results = Vec::new(); + for (key, value) in keys.iter().zip(&values) { + for key in keys { + unsafe { std::env::remove_var(key) }; + } + unsafe { std::env::set_var(key, value) }; + results.push(( + *key, + builder_no_token().hf_token(), + builder_no_token().disable_config_load().hf_token(), + )); + } + let explicit = builder_with_token("config-token") + .disable_config_load() + .hf_token(); + for (key, value) in keys.iter().zip(saved) { + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + std::fs::remove_dir_all(&dir).ok(); + + for (key, loaded, disabled) in results { + assert!(loaded.is_some(), "{key} should provide a token"); + assert_eq!(disabled, None, "{key} should be ignored"); + } + assert_eq!(explicit.as_deref(), Some("config-token")); + } + #[test] fn hf_token_reads_from_file_via_hf_token_path() { let _guard = ENV_LOCK.lock().unwrap(); diff --git a/core/services/hf/src/config.rs b/core/services/hf/src/config.rs index d503c510d884..8511d7c58970 100644 --- a/core/services/hf/src/config.rs +++ b/core/services/hf/src/config.rs @@ -69,6 +69,10 @@ pub struct HfConfig { /// option is disabled. /// See [`HfBuilder::enable_resolve_cache`] for freshness semantics. pub enable_resolve_cache: bool, + /// Disable loading the token from the environment: `HF_TOKEN`, + /// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, + /// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`. + pub disable_config_load: bool, } impl Debug for HfConfig { @@ -83,6 +87,7 @@ impl Debug for HfConfig { .field("root", &self.root) .field("download_mode", &self.download_mode) .field("enable_resolve_cache", &self.enable_resolve_cache) + .field("disable_config_load", &self.disable_config_load) .finish_non_exhaustive() } } @@ -115,6 +120,18 @@ impl opendal_core::Configurator for HfConfig { .get("download_mode") .map(|s| HfDownloadMode::parse(s)) .transpose()?; + let disable_config_load = opts + .get("disable_config_load") + .map(|value| value.parse::()) + .transpose() + .map_err(|err| { + opendal_core::Error::new( + opendal_core::ErrorKind::ConfigInvalid, + "disable_config_load must be true or false", + ) + .set_source(err) + })? + .unwrap_or_default(); let enable_resolve_cache = opts .get("enable_resolve_cache") @@ -141,6 +158,7 @@ impl opendal_core::Configurator for HfConfig { endpoint: opts.get("endpoint").cloned(), download_mode, enable_resolve_cache, + disable_config_load, }) } else { // Bare scheme from via_iter, all config is in options. @@ -163,6 +181,7 @@ impl opendal_core::Configurator for HfConfig { endpoint: opts.get("endpoint").cloned(), download_mode, enable_resolve_cache, + disable_config_load, }) } } @@ -217,6 +236,25 @@ mod tests { assert_eq!(cfg.root.as_deref(), Some("/testdata/")); } + #[test] + fn from_uri_disable_config_load() { + let parse = |value: &str| { + let uri = OperatorUri::new( + "huggingface", + [ + ("repo_type".to_string(), "dataset".to_string()), + ("disable_config_load".to_string(), value.to_string()), + ], + ) + .unwrap(); + HfConfig::from_uri(&uri).map(|cfg| cfg.disable_config_load) + }; + + assert!(parse("true").unwrap()); + assert!(!parse("false").unwrap()); + assert!(parse("yes").is_err()); + } + #[test] fn from_uri_download_mode_http() { let uri = OperatorUri::new( diff --git a/website/data/services.json b/website/data/services.json index 1156882f31f3..1649d69f2fe4 100644 --- a/website/data/services.json +++ b/website/data/services.json @@ -2322,6 +2322,13 @@ "required": false, "group": "General", "comments": "Enable caching of resolved HTTP download addresses and XET file metadata.\n\nDefaults to `false`. Set to `true` to share resolve results across readers\non the same backend. Changed files may remain invisible while cached\nresults are reused. A reader retains XET metadata from its first read for\nits lifetime. Create a new reader to resolve the path again when this\noption is disabled.\nSee [`HfBuilder::enable_resolve_cache`] for freshness semantics." + }, + { + "name": "disable_config_load", + "type": "bool", + "required": false, + "group": "General", + "comments": "Disable loading the token from the environment: `HF_TOKEN`,\n`HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n`$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`." } ], "examples": [ @@ -2329,31 +2336,31 @@ "binding": "rust", "language": "rust", "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n])?;", - "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // (\"root\".to_string(), \"...\".to_string()),\n\n // Token of this backend.\n //\n // This is optional.\n // (\"token\".to_string(), \"...\".to_string()),\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // (\"endpoint\".to_string(), \"...\".to_string()),\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // (\"download_mode\".to_string(), \"...\".to_string()),\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // (\"enable_resolve_cache\".to_string(), \"true\".to_string()),\n])?;" + "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // (\"root\".to_string(), \"...\".to_string()),\n\n // Token of this backend.\n //\n // This is optional.\n // (\"token\".to_string(), \"...\".to_string()),\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // (\"endpoint\".to_string(), \"...\".to_string()),\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // (\"download_mode\".to_string(), \"...\".to_string()),\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // (\"enable_resolve_cache\".to_string(), \"true\".to_string()),\n\n // Disable loading the token from the environment: `HF_TOKEN`,\n // `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // (\"disable_config_load\".to_string(), \"true\".to_string()),\n])?;" }, { "binding": "java", "language": "java", "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\nOperator operator = Operator.of(\"hf\", config);", - "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Default is \"/\".\n// config.put(\"root\", \"...\");\n\n// Token of this backend.\n//\n// This is optional.\n// config.put(\"token\", \"...\");\n\n// Endpoint of the Hugging Face Hub.\n//\n// The default is `https://huggingface.co`.\n// config.put(\"endpoint\", \"...\");\n\n// Download mode. Either `xet` (default) or `http`.\n//\n// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n// environment variable: a non-empty value forces `http`, otherwise it\n// defaults to `xet`. An explicit value here takes precedence.\n//\n// See .\n// config.put(\"download_mode\", \"...\");\n\n// Enable caching of resolved HTTP download addresses and XET file metadata.\n//\n// Defaults to `false`. Set to `true` to share resolve results across readers\n// on the same backend. Changed files may remain invisible while cached\n// results are reused. A reader retains XET metadata from its first read for\n// its lifetime. Create a new reader to resolve the path again when this\n// option is disabled.\n// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n// config.put(\"enable_resolve_cache\", \"true\");\nOperator operator = Operator.of(\"hf\", config);" + "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Default is \"/\".\n// config.put(\"root\", \"...\");\n\n// Token of this backend.\n//\n// This is optional.\n// config.put(\"token\", \"...\");\n\n// Endpoint of the Hugging Face Hub.\n//\n// The default is `https://huggingface.co`.\n// config.put(\"endpoint\", \"...\");\n\n// Download mode. Either `xet` (default) or `http`.\n//\n// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n// environment variable: a non-empty value forces `http`, otherwise it\n// defaults to `xet`. An explicit value here takes precedence.\n//\n// See .\n// config.put(\"download_mode\", \"...\");\n\n// Enable caching of resolved HTTP download addresses and XET file metadata.\n//\n// Defaults to `false`. Set to `true` to share resolve results across readers\n// on the same backend. Changed files may remain invisible while cached\n// results are reused. A reader retains XET metadata from its first read for\n// its lifetime. Create a new reader to resolve the path again when this\n// option is disabled.\n// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n// config.put(\"enable_resolve_cache\", \"true\");\n\n// Disable loading the token from the environment: `HF_TOKEN`,\n// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n// config.put(\"disable_config_load\", \"true\");\nOperator operator = Operator.of(\"hf\", config);" }, { "binding": "python", "language": "python", "minimal": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n)", - "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is optional.\n # token=\"...\",\n\n # Endpoint of the Hugging Face Hub.\n #\n # The default is `https://huggingface.co`.\n # endpoint=\"...\",\n\n # Download mode. Either `xet` (default) or `http`.\n #\n # When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n # environment variable: a non-empty value forces `http`, otherwise it\n # defaults to `xet`. An explicit value here takes precedence.\n #\n # See .\n # download_mode=\"...\",\n\n # Enable caching of resolved HTTP download addresses and XET file metadata.\n #\n # Defaults to `false`. Set to `true` to share resolve results across readers\n # on the same backend. Changed files may remain invisible while cached\n # results are reused. A reader retains XET metadata from its first read for\n # its lifetime. Create a new reader to resolve the path again when this\n # option is disabled.\n # See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n # enable_resolve_cache=\"true\",\n)" + "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is optional.\n # token=\"...\",\n\n # Endpoint of the Hugging Face Hub.\n #\n # The default is `https://huggingface.co`.\n # endpoint=\"...\",\n\n # Download mode. Either `xet` (default) or `http`.\n #\n # When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n # environment variable: a non-empty value forces `http`, otherwise it\n # defaults to `xet`. An explicit value here takes precedence.\n #\n # See .\n # download_mode=\"...\",\n\n # Enable caching of resolved HTTP download addresses and XET file metadata.\n #\n # Defaults to `false`. Set to `true` to share resolve results across readers\n # on the same backend. Changed files may remain invisible while cached\n # results are reused. A reader retains XET metadata from its first read for\n # its lifetime. Create a new reader to resolve the path again when this\n # option is disabled.\n # See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n # enable_resolve_cache=\"true\",\n\n # Disable loading the token from the environment: `HF_TOKEN`,\n # `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n # `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n # disable_config_load=\"true\",\n)" }, { "binding": "nodejs", "language": "javascript", "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n});", - "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // This is optional.\n // token: \"...\",\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // endpoint: \"...\",\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // download_mode: \"...\",\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // enable_resolve_cache: \"true\",\n});" + "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // This is optional.\n // token: \"...\",\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // endpoint: \"...\",\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // download_mode: \"...\",\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // enable_resolve_cache: \"true\",\n\n // Disable loading the token from the environment: `HF_TOKEN`,\n // `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // disable_config_load: \"true\",\n});" }, { "binding": "go", "language": "go", "minimal": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n})", - "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend. Can be \"/path/to/dir\".\n\t//\n\t// Default is \"/\".\n\t// \"root\": \"...\",\n\n\t// Token of this backend.\n\t//\n\t// This is optional.\n\t// \"token\": \"...\",\n\n\t// Endpoint of the Hugging Face Hub.\n\t//\n\t// The default is `https://huggingface.co`.\n\t// \"endpoint\": \"...\",\n\n\t// Download mode. Either `xet` (default) or `http`.\n\t//\n\t// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n\t// environment variable: a non-empty value forces `http`, otherwise it\n\t// defaults to `xet`. An explicit value here takes precedence.\n\t//\n\t// See .\n\t// \"download_mode\": \"...\",\n\n\t// Enable caching of resolved HTTP download addresses and XET file metadata.\n\t//\n\t// Defaults to `false`. Set to `true` to share resolve results across readers\n\t// on the same backend. Changed files may remain invisible while cached\n\t// results are reused. A reader retains XET metadata from its first read for\n\t// its lifetime. Create a new reader to resolve the path again when this\n\t// option is disabled.\n\t// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n\t// \"enable_resolve_cache\": \"true\",\n})" + "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend. Can be \"/path/to/dir\".\n\t//\n\t// Default is \"/\".\n\t// \"root\": \"...\",\n\n\t// Token of this backend.\n\t//\n\t// This is optional.\n\t// \"token\": \"...\",\n\n\t// Endpoint of the Hugging Face Hub.\n\t//\n\t// The default is `https://huggingface.co`.\n\t// \"endpoint\": \"...\",\n\n\t// Download mode. Either `xet` (default) or `http`.\n\t//\n\t// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n\t// environment variable: a non-empty value forces `http`, otherwise it\n\t// defaults to `xet`. An explicit value here takes precedence.\n\t//\n\t// See .\n\t// \"download_mode\": \"...\",\n\n\t// Enable caching of resolved HTTP download addresses and XET file metadata.\n\t//\n\t// Defaults to `false`. Set to `true` to share resolve results across readers\n\t// on the same backend. Changed files may remain invisible while cached\n\t// results are reused. A reader retains XET metadata from its first read for\n\t// its lifetime. Create a new reader to resolve the path again when this\n\t// option is disabled.\n\t// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n\t// \"enable_resolve_cache\": \"true\",\n\n\t// Disable loading the token from the environment: `HF_TOKEN`,\n\t// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n\t// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n\t// \"disable_config_load\": \"true\",\n})" } ] }, From 93d03b54598c4592b590d464d64edb8dbdc722c6 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Tue, 27 Oct 2026 17:10:47 +0100 Subject: [PATCH 2/2] feat(services/hf): ignore endpoint and download mode env under disable_config_load Match s3, whose disable_config_load also skips AWS_ENDPOINT_URL: HF_ENDPOINT and HF_HUB_DISABLE_XET are no longer read when the option is set. --- .../ServiceConfig/HuggingfaceServiceConfig.cs | 2 +- .../org/apache/opendal/ServiceConfig.java | 5 ++- bindings/python/python/opendal/config.py | 2 +- core/services/hf/src/backend.rs | 38 +++++++++++++++---- core/services/hf/src/config.rs | 6 ++- website/data/services.json | 12 +++--- 6 files changed, 45 insertions(+), 20 deletions(-) diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs index bb71a5e87738..65df1d55fda4 100644 --- a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs +++ b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs @@ -29,7 +29,7 @@ namespace OpenDAL.ServiceConfig public sealed class HuggingfaceServiceConfig : IServiceConfig { ///

- /// Disable loading the token from the environment: `HF_TOKEN`, `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`. + /// Disable loading config from the environment. When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`, `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`. /// public bool? DisableConfigLoad { get; init; } /// diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java index 93892d1e8a29..036731a21490 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java +++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java @@ -1688,8 +1688,9 @@ public Map configMap() { @RequiredArgsConstructor(access = AccessLevel.PRIVATE) class Hf implements ServiceConfig { /** - *

Disable loading the token from the environment: HF_TOKEN, - * HF_TOKEN_PATH, or the token file under HF_HOME, + *

Disable loading config from the environment.

+ *

When set, the service ignores HF_ENDPOINT, HF_HUB_DISABLE_XET, + * HF_TOKEN, HF_TOKEN_PATH, and the token file under HF_HOME, * $XDG_CACHE_HOME/huggingface or ~/.cache/huggingface.

*/ public final Boolean disableConfigLoad; diff --git a/bindings/python/python/opendal/config.py b/bindings/python/python/opendal/config.py index d19ee351855d..2bc02ffd1439 100644 --- a/bindings/python/python/opendal/config.py +++ b/bindings/python/python/opendal/config.py @@ -409,7 +409,7 @@ class HfConfig(TypedDict): scheme: Required[Literal["hf"]] """The service scheme; fixed to `"hf"`.""" disable_config_load: NotRequired[bool] - """Disable loading the token from the environment: `HF_TOKEN`, `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.""" + """Disable loading config from the environment. When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`, `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`, `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.""" download_mode: NotRequired[str] """Download mode. Either `xet` (default) or `http`. When unset, the mode is resolved from the `HF_HUB_DISABLE_XET` environment variable: a non-empty value forces `http`, otherwise it defaults to `xet`. An explicit value here takes precedence. See .""" enable_resolve_cache: NotRequired[bool] diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs index 4542806faa73..32e25d72becd 100644 --- a/core/services/hf/src/backend.rs +++ b/core/services/hf/src/backend.rs @@ -167,7 +167,7 @@ impl HfBuilder { self } - /// Disable loading the token from the environment. + /// Disable loading config from the environment. /// /// See [`HfConfig::disable_config_load`] for details. pub fn disable_config_load(mut self) -> Self { @@ -176,26 +176,34 @@ impl HfBuilder { } /// Resolve the Hub base URL: an explicit config value wins, then - /// `HF_ENDPOINT`, then the public Hub. A trailing slash is trimmed - /// because every URL is built by appending `/api/...` to this, and HF - /// answers the resulting `//api/...` with a 404. + /// `HF_ENDPOINT` unless disable_config_load is set, then the public Hub. + /// A trailing slash is trimmed because every URL is built by appending + /// `/api/...` to this, and HF answers the resulting `//api/...` with a 404. fn hf_endpoint(&self) -> String { self.config .endpoint .clone() - .or_else(|| std::env::var("HF_ENDPOINT").ok()) + .or_else(|| { + if self.config.disable_config_load { + None + } else { + std::env::var("HF_ENDPOINT").ok() + } + }) .unwrap_or_else(|| "https://huggingface.co".to_string()) .trim_end_matches('/') .to_string() } - /// Resolve the download mode: an explicit config value wins; otherwise a set, - /// non-empty HF_HUB_DISABLE_XET (a huggingface_hub env var) forces http; default Xet. + /// Resolve the download mode: an explicit config value wins; otherwise, unless + /// disable_config_load is set, a set, non-empty HF_HUB_DISABLE_XET (a + /// huggingface_hub env var) forces http; default Xet. fn hf_download_mode(&self) -> HfDownloadMode { if let Some(mode) = self.config.download_mode { return mode; } - if let Ok(val) = std::env::var("HF_HUB_DISABLE_XET") + if !self.config.disable_config_load + && let Ok(val) = std::env::var("HF_HUB_DISABLE_XET") && !val.is_empty() { return HfDownloadMode::Http; @@ -706,6 +714,20 @@ mod tests { assert_eq!(mode, HfDownloadMode::Xet); } + #[test] + fn disable_config_load_ignores_endpoint_and_download_mode_env() { + let _guard = ENV_LOCK.lock().unwrap(); + unsafe { std::env::set_var("HF_ENDPOINT", "https://env.example.com") }; + unsafe { std::env::set_var("HF_HUB_DISABLE_XET", "1") }; + let builder = HfBuilder::default().disable_config_load(); + let endpoint = builder.hf_endpoint(); + let mode = builder.hf_download_mode(); + unsafe { std::env::remove_var("HF_ENDPOINT") }; + unsafe { std::env::remove_var("HF_HUB_DISABLE_XET") }; + assert_eq!(endpoint, "https://huggingface.co"); + assert_eq!(mode, HfDownloadMode::Xet); + } + #[tokio::test] async fn build_resolve_cache_requires_opt_in() -> Result<()> { use super::super::core::test_utils::MockHttpTransport; diff --git a/core/services/hf/src/config.rs b/core/services/hf/src/config.rs index 8511d7c58970..8c4626908acf 100644 --- a/core/services/hf/src/config.rs +++ b/core/services/hf/src/config.rs @@ -69,8 +69,10 @@ pub struct HfConfig { /// option is disabled. /// See [`HfBuilder::enable_resolve_cache`] for freshness semantics. pub enable_resolve_cache: bool, - /// Disable loading the token from the environment: `HF_TOKEN`, - /// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`, + /// Disable loading config from the environment. + /// + /// When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`, + /// `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`, /// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`. pub disable_config_load: bool, } diff --git a/website/data/services.json b/website/data/services.json index 1649d69f2fe4..50b778d526ab 100644 --- a/website/data/services.json +++ b/website/data/services.json @@ -2328,7 +2328,7 @@ "type": "bool", "required": false, "group": "General", - "comments": "Disable loading the token from the environment: `HF_TOKEN`,\n`HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n`$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`." + "comments": "Disable loading config from the environment.\n\nWhen set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n`HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n`$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`." } ], "examples": [ @@ -2336,31 +2336,31 @@ "binding": "rust", "language": "rust", "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n])?;", - "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // (\"root\".to_string(), \"...\".to_string()),\n\n // Token of this backend.\n //\n // This is optional.\n // (\"token\".to_string(), \"...\".to_string()),\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // (\"endpoint\".to_string(), \"...\".to_string()),\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // (\"download_mode\".to_string(), \"...\".to_string()),\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // (\"enable_resolve_cache\".to_string(), \"true\".to_string()),\n\n // Disable loading the token from the environment: `HF_TOKEN`,\n // `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // (\"disable_config_load\".to_string(), \"true\".to_string()),\n])?;" + "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // (\"root\".to_string(), \"...\".to_string()),\n\n // Token of this backend.\n //\n // This is optional.\n // (\"token\".to_string(), \"...\".to_string()),\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // (\"endpoint\".to_string(), \"...\".to_string()),\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // (\"download_mode\".to_string(), \"...\".to_string()),\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // (\"enable_resolve_cache\".to_string(), \"true\".to_string()),\n\n // Disable loading config from the environment.\n //\n // When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n // `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // (\"disable_config_load\".to_string(), \"true\".to_string()),\n])?;" }, { "binding": "java", "language": "java", "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\nOperator operator = Operator.of(\"hf\", config);", - "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Default is \"/\".\n// config.put(\"root\", \"...\");\n\n// Token of this backend.\n//\n// This is optional.\n// config.put(\"token\", \"...\");\n\n// Endpoint of the Hugging Face Hub.\n//\n// The default is `https://huggingface.co`.\n// config.put(\"endpoint\", \"...\");\n\n// Download mode. Either `xet` (default) or `http`.\n//\n// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n// environment variable: a non-empty value forces `http`, otherwise it\n// defaults to `xet`. An explicit value here takes precedence.\n//\n// See .\n// config.put(\"download_mode\", \"...\");\n\n// Enable caching of resolved HTTP download addresses and XET file metadata.\n//\n// Defaults to `false`. Set to `true` to share resolve results across readers\n// on the same backend. Changed files may remain invisible while cached\n// results are reused. A reader retains XET metadata from its first read for\n// its lifetime. Create a new reader to resolve the path again when this\n// option is disabled.\n// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n// config.put(\"enable_resolve_cache\", \"true\");\n\n// Disable loading the token from the environment: `HF_TOKEN`,\n// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n// config.put(\"disable_config_load\", \"true\");\nOperator operator = Operator.of(\"hf\", config);" + "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Default is \"/\".\n// config.put(\"root\", \"...\");\n\n// Token of this backend.\n//\n// This is optional.\n// config.put(\"token\", \"...\");\n\n// Endpoint of the Hugging Face Hub.\n//\n// The default is `https://huggingface.co`.\n// config.put(\"endpoint\", \"...\");\n\n// Download mode. Either `xet` (default) or `http`.\n//\n// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n// environment variable: a non-empty value forces `http`, otherwise it\n// defaults to `xet`. An explicit value here takes precedence.\n//\n// See .\n// config.put(\"download_mode\", \"...\");\n\n// Enable caching of resolved HTTP download addresses and XET file metadata.\n//\n// Defaults to `false`. Set to `true` to share resolve results across readers\n// on the same backend. Changed files may remain invisible while cached\n// results are reused. A reader retains XET metadata from its first read for\n// its lifetime. Create a new reader to resolve the path again when this\n// option is disabled.\n// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n// config.put(\"enable_resolve_cache\", \"true\");\n\n// Disable loading config from the environment.\n//\n// When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n// `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n// config.put(\"disable_config_load\", \"true\");\nOperator operator = Operator.of(\"hf\", config);" }, { "binding": "python", "language": "python", "minimal": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n)", - "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is optional.\n # token=\"...\",\n\n # Endpoint of the Hugging Face Hub.\n #\n # The default is `https://huggingface.co`.\n # endpoint=\"...\",\n\n # Download mode. Either `xet` (default) or `http`.\n #\n # When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n # environment variable: a non-empty value forces `http`, otherwise it\n # defaults to `xet`. An explicit value here takes precedence.\n #\n # See .\n # download_mode=\"...\",\n\n # Enable caching of resolved HTTP download addresses and XET file metadata.\n #\n # Defaults to `false`. Set to `true` to share resolve results across readers\n # on the same backend. Changed files may remain invisible while cached\n # results are reused. A reader retains XET metadata from its first read for\n # its lifetime. Create a new reader to resolve the path again when this\n # option is disabled.\n # See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n # enable_resolve_cache=\"true\",\n\n # Disable loading the token from the environment: `HF_TOKEN`,\n # `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n # `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n # disable_config_load=\"true\",\n)" + "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is optional.\n # token=\"...\",\n\n # Endpoint of the Hugging Face Hub.\n #\n # The default is `https://huggingface.co`.\n # endpoint=\"...\",\n\n # Download mode. Either `xet` (default) or `http`.\n #\n # When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n # environment variable: a non-empty value forces `http`, otherwise it\n # defaults to `xet`. An explicit value here takes precedence.\n #\n # See .\n # download_mode=\"...\",\n\n # Enable caching of resolved HTTP download addresses and XET file metadata.\n #\n # Defaults to `false`. Set to `true` to share resolve results across readers\n # on the same backend. Changed files may remain invisible while cached\n # results are reused. A reader retains XET metadata from its first read for\n # its lifetime. Create a new reader to resolve the path again when this\n # option is disabled.\n # See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n # enable_resolve_cache=\"true\",\n\n # Disable loading config from the environment.\n #\n # When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n # `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n # `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n # disable_config_load=\"true\",\n)" }, { "binding": "nodejs", "language": "javascript", "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n});", - "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // This is optional.\n // token: \"...\",\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // endpoint: \"...\",\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // download_mode: \"...\",\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // enable_resolve_cache: \"true\",\n\n // Disable loading the token from the environment: `HF_TOKEN`,\n // `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // disable_config_load: \"true\",\n});" + "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // This is optional.\n // token: \"...\",\n\n // Endpoint of the Hugging Face Hub.\n //\n // The default is `https://huggingface.co`.\n // endpoint: \"...\",\n\n // Download mode. Either `xet` (default) or `http`.\n //\n // When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n // environment variable: a non-empty value forces `http`, otherwise it\n // defaults to `xet`. An explicit value here takes precedence.\n //\n // See .\n // download_mode: \"...\",\n\n // Enable caching of resolved HTTP download addresses and XET file metadata.\n //\n // Defaults to `false`. Set to `true` to share resolve results across readers\n // on the same backend. Changed files may remain invisible while cached\n // results are reused. A reader retains XET metadata from its first read for\n // its lifetime. Create a new reader to resolve the path again when this\n // option is disabled.\n // See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n // enable_resolve_cache: \"true\",\n\n // Disable loading config from the environment.\n //\n // When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n // `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n // `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n // disable_config_load: \"true\",\n});" }, { "binding": "go", "language": "go", "minimal": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n})", - "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend. Can be \"/path/to/dir\".\n\t//\n\t// Default is \"/\".\n\t// \"root\": \"...\",\n\n\t// Token of this backend.\n\t//\n\t// This is optional.\n\t// \"token\": \"...\",\n\n\t// Endpoint of the Hugging Face Hub.\n\t//\n\t// The default is `https://huggingface.co`.\n\t// \"endpoint\": \"...\",\n\n\t// Download mode. Either `xet` (default) or `http`.\n\t//\n\t// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n\t// environment variable: a non-empty value forces `http`, otherwise it\n\t// defaults to `xet`. An explicit value here takes precedence.\n\t//\n\t// See .\n\t// \"download_mode\": \"...\",\n\n\t// Enable caching of resolved HTTP download addresses and XET file metadata.\n\t//\n\t// Defaults to `false`. Set to `true` to share resolve results across readers\n\t// on the same backend. Changed files may remain invisible while cached\n\t// results are reused. A reader retains XET metadata from its first read for\n\t// its lifetime. Create a new reader to resolve the path again when this\n\t// option is disabled.\n\t// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n\t// \"enable_resolve_cache\": \"true\",\n\n\t// Disable loading the token from the environment: `HF_TOKEN`,\n\t// `HF_TOKEN_PATH`, or the `token` file under `HF_HOME`,\n\t// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n\t// \"disable_config_load\": \"true\",\n})" + "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend. Can be \"/path/to/dir\".\n\t//\n\t// Default is \"/\".\n\t// \"root\": \"...\",\n\n\t// Token of this backend.\n\t//\n\t// This is optional.\n\t// \"token\": \"...\",\n\n\t// Endpoint of the Hugging Face Hub.\n\t//\n\t// The default is `https://huggingface.co`.\n\t// \"endpoint\": \"...\",\n\n\t// Download mode. Either `xet` (default) or `http`.\n\t//\n\t// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\n\t// environment variable: a non-empty value forces `http`, otherwise it\n\t// defaults to `xet`. An explicit value here takes precedence.\n\t//\n\t// See .\n\t// \"download_mode\": \"...\",\n\n\t// Enable caching of resolved HTTP download addresses and XET file metadata.\n\t//\n\t// Defaults to `false`. Set to `true` to share resolve results across readers\n\t// on the same backend. Changed files may remain invisible while cached\n\t// results are reused. A reader retains XET metadata from its first read for\n\t// its lifetime. Create a new reader to resolve the path again when this\n\t// option is disabled.\n\t// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.\n\t// \"enable_resolve_cache\": \"true\",\n\n\t// Disable loading config from the environment.\n\t//\n\t// When set, the service ignores `HF_ENDPOINT`, `HF_HUB_DISABLE_XET`,\n\t// `HF_TOKEN`, `HF_TOKEN_PATH`, and the `token` file under `HF_HOME`,\n\t// `$XDG_CACHE_HOME/huggingface` or `~/.cache/huggingface`.\n\t// \"disable_config_load\": \"true\",\n})" } ] },