You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/10 06:39:04 UTC
(opendal) branch main updated: feat(hf): auto-discover token from env and credential file (#7490)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new a1df28c86 feat(hf): auto-discover token from env and credential file (#7490)
a1df28c86 is described below
commit a1df28c86778cddda1074ef1ce4895f921c319a3
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Sun May 10 08:38:58 2026 +0200
feat(hf): auto-discover token from env and credential file (#7490)
feat(services/hf): auto-discover token from env and credential file
Read the HuggingFace token using the same priority order as hf-hub:
explicit config → HF_HUB_DISABLE_IMPLICIT_TOKEN → HF_TOKEN env var →
token file (HF_TOKEN_PATH or $HF_HOME/token or ~/.cache/huggingface/token).
Users logged in via `huggingface-cli login` no longer need to pass the
token explicitly when constructing an HfBuilder.
Co-authored-by: Claude Sonnet 4.6 <no...@anthropic.com>
---
core/services/hf/src/backend.rs | 151 ++++++++++++++++++++++++++++++++++++----
1 file changed, 136 insertions(+), 15 deletions(-)
diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs
index 78c6418f2..525aa4647 100644
--- a/core/services/hf/src/backend.rs
+++ b/core/services/hf/src/backend.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::path::PathBuf;
use std::sync::Arc;
use log::debug;
@@ -120,6 +121,52 @@ impl HfBuilder {
}
self
}
+
+ fn hf_endpoint(&self) -> String {
+ self.config
+ .endpoint
+ .clone()
+ .or_else(|| std::env::var("HF_ENDPOINT").ok())
+ .unwrap_or_else(|| "https://huggingface.co".to_string())
+ }
+
+ fn hf_home() -> Option<PathBuf> {
+ if let Ok(h) = std::env::var("HF_HOME") {
+ return Some(PathBuf::from(h));
+ }
+ if let Ok(xdg) = std::env::var("XDG_CACHE_HOME") {
+ return Some(PathBuf::from(xdg).join("huggingface"));
+ }
+ let home = std::env::var("HOME").ok()?;
+ Some(PathBuf::from(home).join(".cache/huggingface"))
+ }
+
+ /// Resolve the authentication token using the same priority order as hf-hub:
+ /// explicit config → HF_HUB_DISABLE_IMPLICIT_TOKEN check → HF_TOKEN env → token file.
+ fn hf_token(&self) -> Option<String> {
+ if let Some(t) = self.config.token.clone() {
+ return Some(t);
+ }
+ if let Ok(val) = std::env::var("HF_HUB_DISABLE_IMPLICIT_TOKEN") {
+ if !val.is_empty() {
+ return None;
+ }
+ }
+ if let Ok(t) = std::env::var("HF_TOKEN") {
+ if !t.is_empty() {
+ return Some(t);
+ }
+ }
+ let token_path = if let Ok(p) = std::env::var("HF_TOKEN_PATH") {
+ Some(PathBuf::from(p))
+ } else {
+ Self::hf_home().map(|h| h.join("token"))
+ };
+ token_path
+ .and_then(|p| std::fs::read_to_string(p).ok())
+ .map(|s| s.trim().to_string())
+ .filter(|s| !s.is_empty())
+ }
}
impl Builder for HfBuilder {
@@ -128,6 +175,9 @@ impl Builder for HfBuilder {
fn build(self) -> Result<impl Access> {
debug!("backend build started: {:?}", &self);
+ let token = self.hf_token();
+ let endpoint = self.hf_endpoint();
+
let repo_type = self.config.repo_type;
debug!("backend use repo_type: {:?}", &repo_type);
@@ -147,21 +197,7 @@ impl Builder for HfBuilder {
let root = normalize_root(&self.config.root.unwrap_or_default());
debug!("backend use root: {}", &root);
-
- let token = self.config.token.as_ref().cloned();
-
- let endpoint = match &self.config.endpoint {
- Some(endpoint) => endpoint.clone(),
- None => {
- // Try to read from HF_ENDPOINT env var which is used
- // by the official huggingface clients.
- if let Ok(env_endpoint) = std::env::var("HF_ENDPOINT") {
- env_endpoint
- } else {
- "https://huggingface.co".to_string()
- }
- }
- };
+ debug!("backend use token: {}", token.is_some());
debug!("backend use endpoint: {}", &endpoint);
let info: Arc<AccessorInfo> = {
@@ -307,6 +343,91 @@ pub(super) mod test_utils {
#[cfg(test)]
mod tests {
use super::*;
+ use std::sync::Mutex;
+
+ // Env vars are process-global; serialize all tests that mutate them.
+ static ENV_LOCK: Mutex<()> = Mutex::new(());
+
+ fn builder_with_token(token: &str) -> HfBuilder {
+ HfBuilder::default().token(token)
+ }
+
+ fn builder_no_token() -> HfBuilder {
+ HfBuilder::default()
+ }
+
+ #[test]
+ fn hf_token_config_takes_priority_over_env() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_TOKEN", "env-token") };
+ let result = builder_with_token("config-token").hf_token();
+ unsafe { std::env::remove_var("HF_TOKEN") };
+ assert_eq!(result.as_deref(), Some("config-token"));
+ }
+
+ #[test]
+ fn hf_token_reads_hf_token_env_var() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_IMPLICIT_TOKEN") };
+ unsafe { std::env::remove_var("HF_TOKEN_PATH") };
+ unsafe { std::env::set_var("HF_TOKEN", "my-env-token") };
+ let result = builder_no_token().hf_token();
+ unsafe { std::env::remove_var("HF_TOKEN") };
+ assert_eq!(result.as_deref(), Some("my-env-token"));
+ }
+
+ #[test]
+ fn hf_token_disable_flag_suppresses_discovery() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_HUB_DISABLE_IMPLICIT_TOKEN", "1") };
+ unsafe { std::env::set_var("HF_TOKEN", "my-env-token") };
+ let result = builder_no_token().hf_token();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_IMPLICIT_TOKEN") };
+ unsafe { std::env::remove_var("HF_TOKEN") };
+ assert_eq!(result, None);
+ }
+
+ #[test]
+ fn hf_token_reads_from_file_via_hf_token_path() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ let token_file = std::env::temp_dir().join("opendal-hf-token-test");
+ std::fs::write(&token_file, "file-token\n").unwrap();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_IMPLICIT_TOKEN") };
+ unsafe { std::env::remove_var("HF_TOKEN") };
+ unsafe { std::env::set_var("HF_TOKEN_PATH", &token_file) };
+ let result = builder_no_token().hf_token();
+ unsafe { std::env::remove_var("HF_TOKEN_PATH") };
+ std::fs::remove_file(&token_file).ok();
+ assert_eq!(result.as_deref(), Some("file-token"));
+ }
+
+ #[test]
+ fn hf_endpoint_returns_default() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::remove_var("HF_ENDPOINT") };
+ let result = HfBuilder::default().hf_endpoint();
+ assert_eq!(result, "https://huggingface.co");
+ }
+
+ #[test]
+ fn hf_endpoint_config_takes_priority_over_env() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_ENDPOINT", "https://env.example.com") };
+ let result = HfBuilder::default()
+ .endpoint("https://config.example.com")
+ .hf_endpoint();
+ unsafe { std::env::remove_var("HF_ENDPOINT") };
+ assert_eq!(result, "https://config.example.com");
+ }
+
+ #[test]
+ fn hf_endpoint_reads_hf_endpoint_env_var() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_ENDPOINT", "https://env.example.com") };
+ let result = HfBuilder::default().hf_endpoint();
+ unsafe { std::env::remove_var("HF_ENDPOINT") };
+ assert_eq!(result, "https://env.example.com");
+ }
#[test]
fn build_accepts_datasets_alias() {