You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by er...@apache.org on 2026/08/22 01:22:31 UTC

(opendal) branch main updated: fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)

This is an automated email from the ASF dual-hosted git repository.

erickguan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 9e779fb16 fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)
9e779fb16 is described below

commit 9e779fb160a6f00b676dbea19087cff8a0c0b79a
Author: Matthew Hambrecht <14...@users.noreply.github.com>
AuthorDate: Fri Aug 21 21:22:24 2026 -0400

    fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)
    
    * fix: Canonicalize HuggingFace id to avoid inconsistent API
    
    cleanup
    
    * switch to OnceCell
---
 core/Cargo.lock                |   1 +
 core/services/hf/Cargo.toml    |   1 +
 core/services/hf/src/core.rs   | 199 ++++++++++++++++++++++++++++++++++++++---
 core/services/hf/src/lister.rs |   2 +-
 core/services/hf/src/reader.rs |   6 +-
 core/services/hf/src/writer.rs |   2 +-
 6 files changed, 197 insertions(+), 14 deletions(-)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index 93c977392..68fa68fc1 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -7188,6 +7188,7 @@ dependencies = [
 name = "opendal-service-hf"
 version = "0.58.2"
 dependencies = [
+ "asyncband",
  "base64 0.23.0",
  "bytes",
  "futures",
diff --git a/core/services/hf/Cargo.toml b/core/services/hf/Cargo.toml
index b1949c63f..a14d28fbe 100644
--- a/core/services/hf/Cargo.toml
+++ b/core/services/hf/Cargo.toml
@@ -32,6 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
+asyncband = { workspace = true }
 bytes = { workspace = true }
 hf-xet = "1.6.0"
 http = { workspace = true }
diff --git a/core/services/hf/src/core.rs b/core/services/hf/src/core.rs
index 26ab4cdf1..c2ba2467e 100644
--- a/core/services/hf/src/core.rs
+++ b/core/services/hf/src/core.rs
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use asyncband::once::OnceCell;
 use bytes::Buf;
 use bytes::Bytes;
 use http::Request;
@@ -160,6 +161,11 @@ pub(super) struct XetFileResponse {
     pub size: u64,
 }
 
+#[derive(Deserialize)]
+pub(super) struct RepoInfoResponse {
+    pub id: String,
+}
+
 #[derive(serde::Deserialize, Debug)]
 pub(super) struct CommitResponse {
     #[allow(dead_code)]
@@ -244,6 +250,7 @@ pub struct HfCore {
     pub endpoint: String,
     pub xet_session: XetSession,
     pub download_mode: HfDownloadMode,
+    canonical_repo: OnceCell<HfRepo>,
 }
 
 impl Debug for HfCore {
@@ -277,6 +284,7 @@ impl HfCore {
             endpoint,
             xet_session,
             download_mode,
+            canonical_repo: OnceCell::new(),
         }
     }
 
@@ -318,8 +326,14 @@ impl HfCore {
     /// Create a new XET upload commit with token refresh configured.
     ///
     /// Each call creates a fresh upload commit from the shared XET session.
-    pub(super) async fn xet_upload_commit(&self) -> Result<XetUploadCommit> {
-        let refresh_url = self.repo.xet_token_url(&self.endpoint, "write");
+    pub(super) async fn xet_upload_commit(
+        &self,
+        ctx: &OperationContext,
+    ) -> Result<XetUploadCommit> {
+        let refresh_url = self
+            .canonical_repo(ctx)
+            .await?
+            .xet_token_url(&self.endpoint, "write");
         let refresh_headers = self.xet_token_refresh_headers();
         self.xet_session
             .new_upload_commit()
@@ -339,8 +353,14 @@ impl HfCore {
     /// Create a new XET download stream group with token refresh configured.
     ///
     /// Each call creates a fresh download group from the shared XET session.
-    pub(super) async fn xet_download_group(&self) -> Result<XetDownloadStreamGroup> {
-        let refresh_url = self.repo.xet_token_url(&self.endpoint, "read");
+    pub(super) async fn xet_download_group(
+        &self,
+        ctx: &OperationContext,
+    ) -> Result<XetDownloadStreamGroup> {
+        let refresh_url = self
+            .canonical_repo(ctx)
+            .await?
+            .xet_token_url(&self.endpoint, "read");
         let refresh_headers = self.xet_token_refresh_headers();
         self.xet_session
             .new_download_stream_group()
@@ -396,9 +416,47 @@ impl HfCore {
         Ok(req)
     }
 
-    /// Build an [`HfUri`] for the given operator-relative path.
-    pub(super) fn uri(&self, path: &str) -> HfUri {
-        self.repo.uri(&self.root, path)
+    /// Return the repo handle with the server's canonical repo id, resolving
+    /// and caching it on first use.
+    ///
+    /// HF resolves repo ids case-insensitively but replies 307 to any request
+    /// for a non-canonically-cased id (e.g. `user/repo` for `user/Repo`).
+    /// The transport cannot replay bodied requests (commit, paths-info)
+    /// through a redirect. Building every URL from the canonical id avoids
+    /// depending on redirect behavior entirely, removing the inconsistency across
+    /// operations.
+    pub(super) async fn canonical_repo(&self, ctx: &OperationContext) -> Result<HfRepo> {
+        // Buckets are addressed by opaque ids
+        if self.repo.is_bucket() {
+            return Ok(self.repo.clone());
+        }
+
+        self.canonical_repo
+            .get_or_try_init(|| async {
+                let url = format!(
+                    "{}/api/{}/{}",
+                    self.endpoint,
+                    self.repo.repo_type.as_plural_str(),
+                    self.repo.repo_id,
+                );
+                let req = self
+                    .request(http::Method::GET, &url, Operation::Stat, "RepoInfo")?
+                    .body(Buffer::new())
+                    .map_err(new_request_build_error)?;
+                let (_, info) = self.send_parse::<RepoInfoResponse>(ctx, req).await?;
+
+                let mut repo = self.repo.clone();
+                repo.repo_id = info.id;
+                Ok(repo)
+            })
+            .await
+            .cloned()
+    }
+
+    /// Build an [`HfUri`] for the given operator-relative path, using the
+    /// canonical repo id.
+    pub(super) async fn canonical_uri(&self, ctx: &OperationContext, path: &str) -> Result<HfUri> {
+        Ok(self.canonical_repo(ctx).await?.uri(&self.root, path))
     }
 
     /// Convert an operator-relative path to a repo-absolute path
@@ -445,7 +503,7 @@ impl HfCore {
     }
 
     pub(super) async fn path_info(&self, ctx: &OperationContext, path: &str) -> Result<PathInfo> {
-        let uri = self.uri(path);
+        let uri = self.canonical_uri(ctx, path).await?;
         let url = uri.paths_info_url(&self.endpoint);
         let form_body = format!("paths={}&expand=True", percent_encode_path(&uri.path));
 
@@ -476,7 +534,7 @@ impl HfCore {
         range: BytesRange,
         mode: HfDownloadMode,
     ) -> Result<Response<HttpBody>> {
-        let uri = self.uri(path);
+        let uri = self.canonical_uri(ctx, path).await?;
         let url = uri.resolve_url(&self.endpoint, self.repo.revision());
 
         let mut req = self.request(http::Method::GET, &url, Operation::Read, "Resolve")?;
@@ -519,7 +577,10 @@ impl HfCore {
         deleted_files: Vec<DeletedFile>,
         deleted_folders: Vec<DeletedFolder>,
     ) -> Result<CommitResponse> {
-        let url = self.repo.git_commit_url(&self.endpoint);
+        let url = self
+            .canonical_repo(ctx)
+            .await?
+            .git_commit_url(&self.endpoint);
 
         let payload = MixedCommitPayload {
             summary: "Commit via OpenDAL".to_string(),
@@ -637,6 +698,16 @@ pub(crate) mod test_utils {
                     HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
                     size,
                 )
+            } else if let Some(rest) = req.uri().path().strip_prefix("/api/") {
+                // Repo-info: echo the requested id back as the canonical id.
+                let id = rest.split_once('/').map(|(_, id)| id).unwrap_or(rest);
+                let data = Bytes::from(format!(r#"{{"id":"{id}"}}"#));
+                let size = data.len() as u64;
+                let buffer = Buffer::from(data);
+                (
+                    HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
+                    size,
+                )
             } else {
                 let data = Bytes::from_static(b"hello");
                 let size = data.len() as u64;
@@ -688,6 +759,8 @@ pub(crate) mod test_utils {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::{Arc, Mutex};
+
     use super::super::core::HfRepoType;
     use super::test_utils::create_test_core;
     use super::*;
@@ -771,6 +844,112 @@ mod tests {
 
         Ok(())
     }
+
+    /// A scripted transport mirroring how HF serves a repo whose configured id
+    /// is not canonically cased: repo-info returns the canonical id, and only
+    /// the canonical commit URL accepts the commit.
+    #[derive(Clone, Default)]
+    struct CanonicalCaseTransport {
+        requests: Arc<Mutex<Vec<(String, String, String)>>>,
+    }
+
+    impl HttpTransport for CanonicalCaseTransport {
+        async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
+            let uri = req.uri().to_string();
+            let auth = req
+                .headers()
+                .get(header::AUTHORIZATION)
+                .and_then(|v| v.to_str().ok())
+                .unwrap_or_default()
+                .to_string();
+            self.requests
+                .lock()
+                .unwrap()
+                .push((req.method().to_string(), uri.clone(), auth));
+
+            let body = match uri.as_str() {
+                "https://huggingface.co/api/models/test-user/uppercase-repo" => {
+                    Bytes::from_static(br#"{"id":"test-user/Uppercase-Repo"}"#)
+                }
+                "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main" => {
+                    Bytes::from_static(b"{}")
+                }
+                other => panic!("unexpected request to {other}"),
+            };
+            let len = body.len() as u64;
+            Ok(Response::builder()
+                .status(http::StatusCode::OK)
+                .header(header::CONTENT_LENGTH, len)
+                .body(HttpBody::new(
+                    futures::stream::iter(vec![Ok(Buffer::from(body))]),
+                    Some(len),
+                ))
+                .unwrap())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_commit_uses_canonical_repo_id() -> Result<()> {
+        let transport = CanonicalCaseTransport::default();
+        let ctx = OperationContext::from_parts(
+            HttpTransporter::new(transport.clone()),
+            Executor::default(),
+        );
+
+        let xet_session = XetSessionBuilder::new()
+            .build()
+            .expect("failed to create xet session");
+        let core = HfCore::new(
+            ServiceInfo::new("hf", "", ""),
+            Capability::default(),
+            HfRepo::new(
+                HfRepoType::Model,
+                "test-user/uppercase-repo".to_string(),
+                Some("main".to_string()),
+            ),
+            "/".to_string(),
+            Some("hf_dummy".to_string()),
+            "https://huggingface.co".to_string(),
+            xet_session,
+            HfDownloadMode::Xet,
+        );
+
+        let lfs_file = |path: &str| LfsFile {
+            path: path.to_string(),
+            oid: "deadbeef".to_string(),
+            algo: "sha256".to_string(),
+            size: 2812,
+        };
+        core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
+            .await?;
+        core.commit_git(&ctx, vec![], vec![lfs_file("b.md")], vec![], vec![])
+            .await?;
+
+        let requests = transport.requests.lock().unwrap();
+        let expected = [
+            (
+                "GET",
+                "https://huggingface.co/api/models/test-user/uppercase-repo",
+            ),
+            (
+                "POST",
+                "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
+            ),
+            // The canonical id is cached: no second repo-info request.
+            (
+                "POST",
+                "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
+            ),
+        ];
+        assert_eq!(requests.len(), expected.len());
+        for ((method, uri, auth), (exp_method, exp_uri)) in requests.iter().zip(expected) {
+            assert_eq!(method, exp_method);
+            assert_eq!(uri, exp_uri);
+            assert_eq!(auth, "Bearer hf_dummy");
+        }
+
+        Ok(())
+    }
 }
 
 mod error {
diff --git a/core/services/hf/src/lister.rs b/core/services/hf/src/lister.rs
index 0f30f1c0d..9c9300b76 100644
--- a/core/services/hf/src/lister.rs
+++ b/core/services/hf/src/lister.rs
@@ -84,7 +84,7 @@ impl HfLister {
         recursive: bool,
         cursor: Option<&str>,
     ) -> Result<FileTree> {
-        let uri = self.core.uri(path);
+        let uri = self.core.canonical_uri(&self.ctx, path).await?;
         let url = uri.file_tree_url(&self.core.endpoint, recursive, cursor);
 
         let req = self
diff --git a/core/services/hf/src/reader.rs b/core/services/hf/src/reader.rs
index bfb84614f..cd78de4c8 100644
--- a/core/services/hf/src/reader.rs
+++ b/core/services/hf/src/reader.rs
@@ -42,7 +42,8 @@ impl HfReadStream {
                 serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
             let metadata = Metadata::new(EntryMode::FILE).with_content_length(info.size);
             let reader =
-                Self::try_new_xet(core, &XetFileInfo::new(info.hash, info.size), range).await?;
+                Self::try_new_xet(core, ctx, &XetFileInfo::new(info.hash, info.size), range)
+                    .await?;
             Ok((RpRead::new(metadata), reader))
         } else {
             let metadata = parse_into_metadata(path, resp.headers())?;
@@ -52,10 +53,11 @@ impl HfReadStream {
 
     async fn try_new_xet(
         core: &HfCore,
+        ctx: &OperationContext,
         file_info: &XetFileInfo,
         range: BytesRange,
     ) -> Result<Self> {
-        let group = core.xet_download_group().await?;
+        let group = core.xet_download_group(ctx).await?;
 
         let xet_range = if range.is_full() {
             None
diff --git a/core/services/hf/src/writer.rs b/core/services/hf/src/writer.rs
index fce8b2bc3..70d9aa4e1 100644
--- a/core/services/hf/src/writer.rs
+++ b/core/services/hf/src/writer.rs
@@ -89,7 +89,7 @@ impl HfWriter {
     /// subsequent [`write`](oio::Write::write) calls can stream data
     /// directly into the XET CAS.
     pub async fn try_new(core: Arc<HfCore>, ctx: OperationContext, path: String) -> Result<Self> {
-        let commit = core.xet_upload_commit().await?;
+        let commit = core.xet_upload_commit(&ctx).await?;
         let stream = commit
             .upload_stream(None, Sha256Policy::Compute)
             .await