You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by er...@apache.org on 2026/08/22 01:22:31 UTC
(opendal) branch main updated: fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)
This is an automated email from the ASF dual-hosted git repository.
erickguan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 9e779fb16 fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)
9e779fb16 is described below
commit 9e779fb160a6f00b676dbea19087cff8a0c0b79a
Author: Matthew Hambrecht <14...@users.noreply.github.com>
AuthorDate: Fri Aug 21 21:22:24 2026 -0400
fix: Canonicalize HuggingFace IDs to avoid inconsistent API (#8108)
* fix: Canonicalize HuggingFace id to avoid inconsistent API
cleanup
* switch to OnceCell
---
core/Cargo.lock | 1 +
core/services/hf/Cargo.toml | 1 +
core/services/hf/src/core.rs | 199 ++++++++++++++++++++++++++++++++++++++---
core/services/hf/src/lister.rs | 2 +-
core/services/hf/src/reader.rs | 6 +-
core/services/hf/src/writer.rs | 2 +-
6 files changed, 197 insertions(+), 14 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 93c977392..68fa68fc1 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -7188,6 +7188,7 @@ dependencies = [
name = "opendal-service-hf"
version = "0.58.2"
dependencies = [
+ "asyncband",
"base64 0.23.0",
"bytes",
"futures",
diff --git a/core/services/hf/Cargo.toml b/core/services/hf/Cargo.toml
index b1949c63f..a14d28fbe 100644
--- a/core/services/hf/Cargo.toml
+++ b/core/services/hf/Cargo.toml
@@ -32,6 +32,7 @@ version = { workspace = true }
all-features = true
[dependencies]
+asyncband = { workspace = true }
bytes = { workspace = true }
hf-xet = "1.6.0"
http = { workspace = true }
diff --git a/core/services/hf/src/core.rs b/core/services/hf/src/core.rs
index 26ab4cdf1..c2ba2467e 100644
--- a/core/services/hf/src/core.rs
+++ b/core/services/hf/src/core.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use asyncband::once::OnceCell;
use bytes::Buf;
use bytes::Bytes;
use http::Request;
@@ -160,6 +161,11 @@ pub(super) struct XetFileResponse {
pub size: u64,
}
+#[derive(Deserialize)]
+pub(super) struct RepoInfoResponse {
+ pub id: String,
+}
+
#[derive(serde::Deserialize, Debug)]
pub(super) struct CommitResponse {
#[allow(dead_code)]
@@ -244,6 +250,7 @@ pub struct HfCore {
pub endpoint: String,
pub xet_session: XetSession,
pub download_mode: HfDownloadMode,
+ canonical_repo: OnceCell<HfRepo>,
}
impl Debug for HfCore {
@@ -277,6 +284,7 @@ impl HfCore {
endpoint,
xet_session,
download_mode,
+ canonical_repo: OnceCell::new(),
}
}
@@ -318,8 +326,14 @@ impl HfCore {
/// Create a new XET upload commit with token refresh configured.
///
/// Each call creates a fresh upload commit from the shared XET session.
- pub(super) async fn xet_upload_commit(&self) -> Result<XetUploadCommit> {
- let refresh_url = self.repo.xet_token_url(&self.endpoint, "write");
+ pub(super) async fn xet_upload_commit(
+ &self,
+ ctx: &OperationContext,
+ ) -> Result<XetUploadCommit> {
+ let refresh_url = self
+ .canonical_repo(ctx)
+ .await?
+ .xet_token_url(&self.endpoint, "write");
let refresh_headers = self.xet_token_refresh_headers();
self.xet_session
.new_upload_commit()
@@ -339,8 +353,14 @@ impl HfCore {
/// Create a new XET download stream group with token refresh configured.
///
/// Each call creates a fresh download group from the shared XET session.
- pub(super) async fn xet_download_group(&self) -> Result<XetDownloadStreamGroup> {
- let refresh_url = self.repo.xet_token_url(&self.endpoint, "read");
+ pub(super) async fn xet_download_group(
+ &self,
+ ctx: &OperationContext,
+ ) -> Result<XetDownloadStreamGroup> {
+ let refresh_url = self
+ .canonical_repo(ctx)
+ .await?
+ .xet_token_url(&self.endpoint, "read");
let refresh_headers = self.xet_token_refresh_headers();
self.xet_session
.new_download_stream_group()
@@ -396,9 +416,47 @@ impl HfCore {
Ok(req)
}
- /// Build an [`HfUri`] for the given operator-relative path.
- pub(super) fn uri(&self, path: &str) -> HfUri {
- self.repo.uri(&self.root, path)
+ /// Return the repo handle with the server's canonical repo id, resolving
+ /// and caching it on first use.
+ ///
+ /// HF resolves repo ids case-insensitively but replies 307 to any request
+ /// for a non-canonically-cased id (e.g. `user/repo` for `user/Repo`).
+ /// The transport cannot replay bodied requests (commit, paths-info)
+ /// through a redirect. Building every URL from the canonical id avoids
+ /// depending on redirect behavior entirely, removing the inconsistency across
+ /// operations.
+ pub(super) async fn canonical_repo(&self, ctx: &OperationContext) -> Result<HfRepo> {
+ // Buckets are addressed by opaque ids
+ if self.repo.is_bucket() {
+ return Ok(self.repo.clone());
+ }
+
+ self.canonical_repo
+ .get_or_try_init(|| async {
+ let url = format!(
+ "{}/api/{}/{}",
+ self.endpoint,
+ self.repo.repo_type.as_plural_str(),
+ self.repo.repo_id,
+ );
+ let req = self
+ .request(http::Method::GET, &url, Operation::Stat, "RepoInfo")?
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+ let (_, info) = self.send_parse::<RepoInfoResponse>(ctx, req).await?;
+
+ let mut repo = self.repo.clone();
+ repo.repo_id = info.id;
+ Ok(repo)
+ })
+ .await
+ .cloned()
+ }
+
+ /// Build an [`HfUri`] for the given operator-relative path, using the
+ /// canonical repo id.
+ pub(super) async fn canonical_uri(&self, ctx: &OperationContext, path: &str) -> Result<HfUri> {
+ Ok(self.canonical_repo(ctx).await?.uri(&self.root, path))
}
/// Convert an operator-relative path to a repo-absolute path
@@ -445,7 +503,7 @@ impl HfCore {
}
pub(super) async fn path_info(&self, ctx: &OperationContext, path: &str) -> Result<PathInfo> {
- let uri = self.uri(path);
+ let uri = self.canonical_uri(ctx, path).await?;
let url = uri.paths_info_url(&self.endpoint);
let form_body = format!("paths={}&expand=True", percent_encode_path(&uri.path));
@@ -476,7 +534,7 @@ impl HfCore {
range: BytesRange,
mode: HfDownloadMode,
) -> Result<Response<HttpBody>> {
- let uri = self.uri(path);
+ let uri = self.canonical_uri(ctx, path).await?;
let url = uri.resolve_url(&self.endpoint, self.repo.revision());
let mut req = self.request(http::Method::GET, &url, Operation::Read, "Resolve")?;
@@ -519,7 +577,10 @@ impl HfCore {
deleted_files: Vec<DeletedFile>,
deleted_folders: Vec<DeletedFolder>,
) -> Result<CommitResponse> {
- let url = self.repo.git_commit_url(&self.endpoint);
+ let url = self
+ .canonical_repo(ctx)
+ .await?
+ .git_commit_url(&self.endpoint);
let payload = MixedCommitPayload {
summary: "Commit via OpenDAL".to_string(),
@@ -637,6 +698,16 @@ pub(crate) mod test_utils {
HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
size,
)
+ } else if let Some(rest) = req.uri().path().strip_prefix("/api/") {
+ // Repo-info: echo the requested id back as the canonical id.
+ let id = rest.split_once('/').map(|(_, id)| id).unwrap_or(rest);
+ let data = Bytes::from(format!(r#"{{"id":"{id}"}}"#));
+ let size = data.len() as u64;
+ let buffer = Buffer::from(data);
+ (
+ HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
+ size,
+ )
} else {
let data = Bytes::from_static(b"hello");
let size = data.len() as u64;
@@ -688,6 +759,8 @@ pub(crate) mod test_utils {
#[cfg(test)]
mod tests {
+ use std::sync::{Arc, Mutex};
+
use super::super::core::HfRepoType;
use super::test_utils::create_test_core;
use super::*;
@@ -771,6 +844,112 @@ mod tests {
Ok(())
}
+
+ /// A scripted transport mirroring how HF serves a repo whose configured id
+ /// is not canonically cased: repo-info returns the canonical id, and only
+ /// the canonical commit URL accepts the commit.
+ #[derive(Clone, Default)]
+ struct CanonicalCaseTransport {
+ requests: Arc<Mutex<Vec<(String, String, String)>>>,
+ }
+
+ impl HttpTransport for CanonicalCaseTransport {
+ async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
+ let uri = req.uri().to_string();
+ let auth = req
+ .headers()
+ .get(header::AUTHORIZATION)
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or_default()
+ .to_string();
+ self.requests
+ .lock()
+ .unwrap()
+ .push((req.method().to_string(), uri.clone(), auth));
+
+ let body = match uri.as_str() {
+ "https://huggingface.co/api/models/test-user/uppercase-repo" => {
+ Bytes::from_static(br#"{"id":"test-user/Uppercase-Repo"}"#)
+ }
+ "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main" => {
+ Bytes::from_static(b"{}")
+ }
+ other => panic!("unexpected request to {other}"),
+ };
+ let len = body.len() as u64;
+ Ok(Response::builder()
+ .status(http::StatusCode::OK)
+ .header(header::CONTENT_LENGTH, len)
+ .body(HttpBody::new(
+ futures::stream::iter(vec![Ok(Buffer::from(body))]),
+ Some(len),
+ ))
+ .unwrap())
+ }
+ }
+
+ #[tokio::test]
+ async fn test_commit_uses_canonical_repo_id() -> Result<()> {
+ let transport = CanonicalCaseTransport::default();
+ let ctx = OperationContext::from_parts(
+ HttpTransporter::new(transport.clone()),
+ Executor::default(),
+ );
+
+ let xet_session = XetSessionBuilder::new()
+ .build()
+ .expect("failed to create xet session");
+ let core = HfCore::new(
+ ServiceInfo::new("hf", "", ""),
+ Capability::default(),
+ HfRepo::new(
+ HfRepoType::Model,
+ "test-user/uppercase-repo".to_string(),
+ Some("main".to_string()),
+ ),
+ "/".to_string(),
+ Some("hf_dummy".to_string()),
+ "https://huggingface.co".to_string(),
+ xet_session,
+ HfDownloadMode::Xet,
+ );
+
+ let lfs_file = |path: &str| LfsFile {
+ path: path.to_string(),
+ oid: "deadbeef".to_string(),
+ algo: "sha256".to_string(),
+ size: 2812,
+ };
+ core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
+ .await?;
+ core.commit_git(&ctx, vec![], vec![lfs_file("b.md")], vec![], vec![])
+ .await?;
+
+ let requests = transport.requests.lock().unwrap();
+ let expected = [
+ (
+ "GET",
+ "https://huggingface.co/api/models/test-user/uppercase-repo",
+ ),
+ (
+ "POST",
+ "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
+ ),
+ // The canonical id is cached: no second repo-info request.
+ (
+ "POST",
+ "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
+ ),
+ ];
+ assert_eq!(requests.len(), expected.len());
+ for ((method, uri, auth), (exp_method, exp_uri)) in requests.iter().zip(expected) {
+ assert_eq!(method, exp_method);
+ assert_eq!(uri, exp_uri);
+ assert_eq!(auth, "Bearer hf_dummy");
+ }
+
+ Ok(())
+ }
}
mod error {
diff --git a/core/services/hf/src/lister.rs b/core/services/hf/src/lister.rs
index 0f30f1c0d..9c9300b76 100644
--- a/core/services/hf/src/lister.rs
+++ b/core/services/hf/src/lister.rs
@@ -84,7 +84,7 @@ impl HfLister {
recursive: bool,
cursor: Option<&str>,
) -> Result<FileTree> {
- let uri = self.core.uri(path);
+ let uri = self.core.canonical_uri(&self.ctx, path).await?;
let url = uri.file_tree_url(&self.core.endpoint, recursive, cursor);
let req = self
diff --git a/core/services/hf/src/reader.rs b/core/services/hf/src/reader.rs
index bfb84614f..cd78de4c8 100644
--- a/core/services/hf/src/reader.rs
+++ b/core/services/hf/src/reader.rs
@@ -42,7 +42,8 @@ impl HfReadStream {
serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
let metadata = Metadata::new(EntryMode::FILE).with_content_length(info.size);
let reader =
- Self::try_new_xet(core, &XetFileInfo::new(info.hash, info.size), range).await?;
+ Self::try_new_xet(core, ctx, &XetFileInfo::new(info.hash, info.size), range)
+ .await?;
Ok((RpRead::new(metadata), reader))
} else {
let metadata = parse_into_metadata(path, resp.headers())?;
@@ -52,10 +53,11 @@ impl HfReadStream {
async fn try_new_xet(
core: &HfCore,
+ ctx: &OperationContext,
file_info: &XetFileInfo,
range: BytesRange,
) -> Result<Self> {
- let group = core.xet_download_group().await?;
+ let group = core.xet_download_group(ctx).await?;
let xet_range = if range.is_full() {
None
diff --git a/core/services/hf/src/writer.rs b/core/services/hf/src/writer.rs
index fce8b2bc3..70d9aa4e1 100644
--- a/core/services/hf/src/writer.rs
+++ b/core/services/hf/src/writer.rs
@@ -89,7 +89,7 @@ impl HfWriter {
/// subsequent [`write`](oio::Write::write) calls can stream data
/// directly into the XET CAS.
pub async fn try_new(core: Arc<HfCore>, ctx: OperationContext, path: String) -> Result<Self> {
- let commit = core.xet_upload_commit().await?;
+ let commit = core.xet_upload_commit(&ctx).await?;
let stream = commit
.upload_stream(None, Sha256Policy::Compute)
.await